diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..688d29877eeeed1bda99e315f7f5ae6afc3f8d6b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +save_model/best_model.pth filter=lfs diff=lfs merge=lfs -text diff --git a/__pycache__/data.cpython-38.pyc b/__pycache__/data.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1594d7fddef51dd9d0a946b378812f537bc42b46 Binary files /dev/null and b/__pycache__/data.cpython-38.pyc differ diff --git a/__pycache__/data.cpython-39.pyc b/__pycache__/data.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a1fd31c2f853a6d1f6a3054eb37554156d8c1c5 Binary files /dev/null and b/__pycache__/data.cpython-39.pyc differ diff --git a/__pycache__/engine.cpython-38.pyc b/__pycache__/engine.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65c570dc205fd702022476a1d6fe97a5de1e4f40 Binary files /dev/null and b/__pycache__/engine.cpython-38.pyc differ diff --git a/__pycache__/engine.cpython-39.pyc b/__pycache__/engine.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b90e46b3265091738df01bc6696d78a732672e36 Binary files /dev/null and b/__pycache__/engine.cpython-39.pyc differ diff --git a/__pycache__/train.cpython-38.pyc b/__pycache__/train.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..94bf93d19d293999fc877cdfa2ddce3aa526525a Binary files /dev/null and b/__pycache__/train.cpython-38.pyc differ diff --git a/__pycache__/train.cpython-39.pyc b/__pycache__/train.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53969238422d85e9b3089d857c74fb4321ba41a1 Binary files /dev/null and b/__pycache__/train.cpython-39.pyc differ diff --git a/__pycache__/utils.cpython-38.pyc b/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0a05e5a73b6174933051b12d9a46b54a7ae743a4 Binary files /dev/null and b/__pycache__/utils.cpython-38.pyc differ diff --git a/__pycache__/utils.cpython-39.pyc b/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2fa313df18e57e05db6d9066c05fce45c7640007 Binary files /dev/null and b/__pycache__/utils.cpython-39.pyc differ diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..1fe59b1ee5da6aa0c806d51f76a031132be1f2a5 --- /dev/null +++ b/app.py @@ -0,0 +1,91 @@ +import gradio as gr +import torch +import torch.nn as nn +import onnx +import engine, data, utils +from typing import Tuple, Dict +from train import NUM_CLASSES +from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights +import torchvision.transforms as T +import onnxruntime as ort +import numpy as np +from timeit import default_timer as timer +from pathlib import Path + +PATH = "save_model/food_cpu.onnx" + + +model = efficientnet_b0(weights=EfficientNet_B0_Weights.DEFAULT) +model.classifier = nn.Sequential( + nn.Dropout(p = 0.2, inplace = True), + nn.Linear(1280, NUM_CLASSES), + # nn.Softmax() + ) +model = utils.load_model(model, "save_model/best_model.pth") + +utils.onnx_inference(model, PATH, "cpu") +onnx_model = onnx.load(PATH) +onnx_check = onnx.checker.check_model(onnx_model) + +classes = data.train_datasets.classes + +trn = T.ToPILImage() + +def predict(img) -> Tuple[Dict, float]: + """Transforms and performs a prediction on img and return prediction and time take.""" + # Start the timer + start_time = timer() + + # transform the target image and add a batch dimension + img = data.transform(img).unsqueeze(dim = 0) + + # inference using onnx + ort_sess = ort.InferenceSession(PATH) + outputs = ort_sess.run(None, {'input': img.numpy()}) + + predicted = classes[outputs[0][0].argmax(0)] + + # print("\n", outputs[0][0], "\n") + outputs = np.array(torch.softmax(torch.from_numpy(outputs[0]), dim = 1)) + pred_labels_and_prob = {classes[i]: float(outputs[0][i]) for i in range(len(classes))} + # print(f'Predicted: "{predicted}"') + + # Calculate the predicion time + pred_time = round(timer() - start_time, 5) + + # Return the prediction dictionary and prediction time + return pred_labels_and_prob, pred_time + + +image = trn(data.test_datasets[3][0]) + +exp_dir = "./example_data/" +test_data_paths = list(Path(exp_dir).glob("*.jpg")) +# print(test_data_paths) +example_list = [[str(filepath)] for filepath in test_data_paths] +# print(example_list) + + +# pred_dict, pred_time = predict(img = image) +# print(f"Predicted label and probability: {pred_dict}") +# print(f"Prediction time: {pred_time}") + +title = "FoodVision 🍕🥩🍣" +description = "An EfficientNetB2 feature extractor computer vision model to classify images of food as pizza, steak or sushi." +article = "Created at [09. PyTorch Model Deployment](https://www.learnpytorch.io/09_pytorch_model_deployment/)." + + +# Create the Gradio demo +demo = gr.Interface(fn=predict, # mapping function from input to output + inputs=gr.Image(type="pil"), # what are the inputs? + outputs=[gr.Label(num_top_classes=101, label="Predictions"), # what are the outputs? + gr.Number(label="Prediction time (s)")], # our fn has two outputs, therefore we have two outputs + examples=example_list, + title=title, + description=description, + article=article) + +# Launch the demo! +# demo.launch(debug=False, # print errors locally? +# share=True) # generate a publically shareable URL? +demo.launch(share=True) \ No newline at end of file diff --git a/data.py b/data.py new file mode 100644 index 0000000000000000000000000000000000000000..975e4df542fd65e6ebf52b7b5c45d215bc7a252a --- /dev/null +++ b/data.py @@ -0,0 +1,35 @@ +import torch +import torchvision +from torchvision.datasets import Food101, MNIST +from torch.utils.data import Subset, DataLoader +import torchvision.transforms as transforms + +transform = transforms.Compose([ + transforms.Resize((224, 224)), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]), +]) + +split = 0.25 +batch_size = 32 + +train_datasets = Food101(root = "./data", split = "train", transform = transform, download = True) +test_datasets = Food101(root = "./data", split = "test", transform = transform, download = True) + +num_samples_train = int(split * len(train_datasets)) +train_indices = torch.randperm(len(train_datasets))[:num_samples_train] + +num_samples_valid = int(split * len(train_datasets)) +valid_indices = torch.randperm(len(train_datasets))[:num_samples_valid] + +num_samples_test = int(split * len(test_datasets)) +test_indices = torch.randperm(len(test_datasets))[:num_samples_test] + +split_tarin_data = Subset(train_datasets, train_indices) +split_valid_data = Subset(train_datasets, valid_indices) +split_test_data = Subset(test_datasets, test_indices) + +train_dataloader = DataLoader(split_tarin_data, batch_size = batch_size, shuffle = True, drop_last = True) +valid_dataloader = DataLoader(split_valid_data, batch_size = batch_size, shuffle = True, drop_last = True) +test_dataloader = DataLoader(split_test_data, batch_size = batch_size, shuffle = False, drop_last = True) \ No newline at end of file diff --git a/engine.py b/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..6222251b56b4a61ce2c94f20c65ec4c51f59c142 --- /dev/null +++ b/engine.py @@ -0,0 +1,203 @@ +import torch +import wandb +from tqdm.auto import tqdm + +API_KEY = "881252af31786a1cf813449b9b4124955f54703e" + + +def train_loop(model: torch.nn.Module, dataloader: torch.utils.data.DataLoader, loss_fn: torch.nn.Module, + optimizer: torch.optim.Optimizer, accuracy_fn, device: torch.device): + + """ + Function for Model Training + + Args: + model: A pytorch model you want to train + dataloader: A dataloader for intance for model training + loss_fn: A loss function for calculate model loss + accuracy_fn: A Accuracy function to check how model is accurate + device: A device on which model run i.e.: "cuda" or "cpu" + + Return: + list of train loss and accuracy and also model weights + + Example usage: + train_loop(model = mymodel, dataloader = train_dataloader, loss_fn = loss_fn, + accuracy_fn = accuracy_fn, device = device) + """ + + train_loss, train_acc = 0, 0 + + model.train() + + for batch, (x_train, y_train) in enumerate(dataloader): + x_train, y_train = x_train.to(device), y_train.to(device) + + # 1. Forwrad Pass + logits = model(x_train) + + # 2. Loss + loss = loss_fn(logits, y_train) + + # 3. Gradzero step + optimizer.zero_grad() + + # 4. Backward + loss.backward() + + # 5. Optimizer step + optimizer.step() + + acc = accuracy_fn(torch.argmax(logits, dim = 1), y_train) + + train_acc += acc + train_loss += loss + + train_loss /= len(dataloader) + train_acc /= len(dataloader) + + return model, train_loss, train_acc + + +def test_loop(model: torch.nn.Module, dataloader: torch.utils.data.DataLoader, loss_fn: torch.nn.Module, + accuracy_fn, device: torch.device): + + """ + A Funtion to test the model after traininig + + Args: + model: A model which you want to test on test intance + dataloader: A dataloader intance on which you test model + loss_fn: A loss function to calculate the model loss + accuracy_fn : A accuracy function to check model accuracy on dataloader intance + device: A device on whic you want to run model i.e.: "cuda" or "cpu" + + Return: + A list of test loss and Accuracy + + Example Usage: + test_loop(model = mymodel, dataloader = test_datloader, loss_fn = loss_fn, + accuracy_fn = accuracy+fn, device = device) + """ + + test_loss, test_acc = 0, 0 + + model.eval() + with torch.inference_mode(): + for x_test, y_test in dataloader: + x_test, y_test = x_test.to(device), y_test.to(device) + + # 1. Forward Pass + logits = model(x_test) + + # 2. Loss + test_loss += loss_fn(logits, y_test) + + test_acc += accuracy_fn(torch.argmax(logits, dim = 1), y_test) + + test_acc /= len(dataloader) + test_loss /= len(dataloader) + + return test_loss, test_acc + + +def log_image_table(images, predicted, labels, probs): + "Log a wandb.Table with (img, pred, target, scores)" + wandb.login(key=API_KEY) + print("[LOG]: Login Succesfull.") + # 🐝 Create a wandb Table to log images, labels and predictions to + table = wandb.Table(columns=["image", "pred", "target"]+[f"score_{i}" for i in range(10)]) + for img, pred, targ, prob in zip(images.to("cpu"), predicted.to("cpu"), labels.to("cpu"), probs.to("cpu")): + table.add_data(wandb.Image(img[0].numpy()*255), pred, targ, *prob.numpy()) + wandb.log({"predictions_table":table}, commit=False) + + + +def validation(model: torch.nn.Module, dataloader: torch.utils.data.DataLoader, loss_fn: torch.nn.Module, + accuracy_fn, log_images: bool, device: torch.device, batch_idx=0): + """ + A Function for hyperparameter tuning + + Args: + model: A model the you want tune its hyperparameter + dataloder: Adataloader intance for hyperparameter tuning + loss_fn: A loss funtion to calcualte model loss + Accuracy_fn: a accuracy function to calcultae accuracy for model perforamnce + device: A device on which model run i.e.: "cuda" or "cpu" + + Return: + A list of accuracy and loss + + Example usage: + validation(model = mymodel, dataloader = valid_dataloader, loss_fn = loss_fn, + accuracy_fn = accuracy_fn, device = device) + """ + + val_loss, val_acc = 0, 0 + + model.eval() + with torch.inference_mode(): + for i, (x_val, y_val) in enumerate(dataloader): + x_val, y_val = x_val.to(device), y_val.to(device) + + logits = model(x_val) + + val_loss += loss_fn(logits, y_val) + + val_acc += accuracy_fn(torch.argmax(logits, dim = 1), y_val) + + # Log one batch of images to the dashboard, always same batch_idx. + if i==batch_idx and log_images: + log_image_table(x_val, torch.max(logits.data, 1)[0], y_val, logits.softmax(dim=1)) + + val_loss /= len(dataloader) + val_acc /= len(dataloader) + + return val_loss, val_acc + +def train(model: torch.nn.Module, train_dataloader: torch.utils.data.DataLoader, test_dataloader: torch.utils.data.DataLoader, + optimizer: torch.optim.Optimizer, loss_fn: torch.nn.Module, accuracy_fn, epochs: int, device: torch.device): + + """ + A function to train and test the pytorch model + + Args: + model: A model to train and test + train_dataloader: A dataloader intance for train model + test_dataloader: A dataloader intance for test model + optimizer: A optimizer funtion to optimize the model + loss_fn: A loss function to calculate model loss + accuracy_fn: An accuracy to calculate model performance + epochs: number of iteration to run the loop + device: A device on which model run i.e.: "cuda" or "cpu" + + Return: + train model, List of train and test losses and accuracy + + Example usage: + train(model = mymodel, train_dataloader = train_dataloader, test_dataloader = test_dataloader, optimizer = optimizer, + loss_fn = loss_fn, acuuracy_fn = accuracy_fn, epochs = epochs, device = device) + """ + + train_losses, test_losses = [], [] + train_accs, test_accs = [], [] + + for epoch in tqdm(range(epochs)): + + print(f"\nEpoch: {epoch+1}") + + train_model, train_loss, train_acc = train_loop(model = model, dataloader = train_dataloader, + loss_fn = loss_fn, optimizer = optimizer, + accuracy_fn = accuracy_fn, device = device) + + test_loss, test_acc = test_loop(model = model, dataloader = test_dataloader, loss_fn = loss_fn, + accuracy_fn = accuracy_fn, device = device) + + print(f"Train Loss: {train_loss:.5f} | Test Loss: {test_loss:.5f} || Train Accuracy: {train_acc:.5f} | Test Accuracy: {test_acc:.5f}") + + train_losses.append(train_loss.item()) + test_losses.append(test_loss.item()) + train_accs.append(train_acc.item()) + test_accs.append(test_acc.item()) + + return train_losses, test_losses, train_accs, test_accs, train_model \ No newline at end of file diff --git a/example_data/1.jpg b/example_data/1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0654ba40b94f5abdf3e001919f9d7cd8ae770474 Binary files /dev/null and b/example_data/1.jpg differ diff --git a/example_data/2.jpg b/example_data/2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7b99699cff70648f831bd6dbb541460c9dfc60d9 Binary files /dev/null and b/example_data/2.jpg differ diff --git a/example_data/3.jpg b/example_data/3.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c595c8b62c22f02f5c36b5d7672b712edcd5de03 Binary files /dev/null and b/example_data/3.jpg differ diff --git a/hyperpara.py b/hyperpara.py new file mode 100644 index 0000000000000000000000000000000000000000..4ea4572b35527358cb7074b59baa19b544d382c4 --- /dev/null +++ b/hyperpara.py @@ -0,0 +1,127 @@ +import os +import torch +import torch.nn as nn +import optuna +from optuna.trial import TrialState +from torch import optim +import engine, data, utils +from train import device, LEARNIGN_RATE, NUM_EPOCH, NUM_CLASSES +from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights +from torchmetrics.classification import MulticlassAccuracy + +import wandb + +# WanDB login + +API_KEY = "881252af31786a1cf813449b9b4124955f54703e" + +wandb.login(key=API_KEY) +print("[LOG]: Login Succesfull.") + + + + +def objective(trial, n_trials=100): + """Objective function to be optimized by Optuna. + Hyperparameters chosen to be optimized: optimizer, learning rate, + dropout values, number of convolutional layers, number of filters of + convolutional layers, number of neurons of fully connected layers. + Inputs: + - trial (optuna.trial._trial.Trial): Optuna trial + Returns: + - accuracy(torch.Tensor): The test accuracy. Parameter to be maximized. + """ + + lr = trial.suggest_float("lr", LEARNIGN_RATE*1e-2, LEARNIGN_RATE, log=True) # Learning rates + n_epochs = trial.suggest_int('n_estimators', NUM_EPOCH//2, NUM_EPOCH) + + optimizer_name = trial.suggest_categorical("optimizer", ["Adam", "RMSprop", "SGD"]) + optimizer = getattr(optim, optimizer_name)(model.parameters(), lr=lr) #torch.optim.Adam(model.parameters(), lr = lr)# getattr(optim, optimizer_name)(model.parameters(), lr=lr) + + + loss_fn = torch.nn.CrossEntropyLoss() + accuracy_fn = MulticlassAccuracy(num_classes = NUM_CLASSES).to(device) + + wandb.init( + # set the wandb project where this run will be logged + project="food-app", + + # track hyperparameters and run metadata + config={ + "optimizer": trial.params["optimizer"], + "architecture": "Efficientnet B0", + "dataset": "Food101", + "epochs": trial.params["n_estimators"], + } + ) + + # Training of the model + best_loss = 100 + patience = 5 + early_stop = 0 + for epoch in range(n_epochs): + + train_model, train_loss, train_acc = engine.train_loop(model = model, dataloader = data.train_dataloader, + loss_fn = loss_fn, optimizer = optimizer, + accuracy_fn = accuracy_fn, device = device) + + val_loss, val_acc = engine.validation(model = model, dataloader = data.valid_dataloader, loss_fn = loss_fn, + accuracy_fn = accuracy_fn, log_images=(epoch==(wandb.config.epochs-1)), device = device) + + + # For pruning (stops trial early if not promising) + trial.report(val_acc, epoch) + # Handle pruning based on the intermediate value. + if trial.should_prune(): + raise optuna.exceptions.TrialPruned() + + if val_acc < best_loss: + early_stop = 0 + best_loss = val_acc + utils.save_model(model = train_model, target_dir = "./save_model", model_name = f"best_model.pth") + + else: + early_stop += 1 + + if early_stop == patience: + break + + wandb.log({"Train Loss": train_loss, "Train Accuracy": train_acc, "Validation Loss": val_loss, "Validation Accuracy": val_acc}) + + wandb.finish() + + return val_acc + + +model = efficientnet_b0(weights=EfficientNet_B0_Weights.DEFAULT) +model.classifier = nn.Sequential( + nn.Dropout(p = 0.2, inplace = True), + nn.Linear(1280, NUM_CLASSES), + # nn.Softmax() + ) + +# isExist = os.path.exists("save_model/train_model_4e-06.pth") +# print(isExist) +model = utils.load_model(model, "save_model/train_model_4e-06.pth").to(device) +# print(model) + +# Create an Optuna study to maximize test accuracy +study = optuna.create_study(direction="maximize") +study.optimize(objective, n_trials=11) + +# Find number of pruned and completed trials +pruned_trials = study.get_trials(deepcopy=False, states=[TrialState.PRUNED]) +complete_trials = study.get_trials(deepcopy=False, states=[TrialState.COMPLETE]) + +# Display the study statistics +print("\nStudy statistics: ") +print(" Number of finished trials: ", len(study.trials)) +print(" Number of pruned trials: ", len(pruned_trials)) +print(" Number of complete trials: ", len(complete_trials)) + +trial = study.best_trial +print("Best trial:") +print(" Value: ", trial.value) +print(" Params: ") +for key, value in trial.params.items(): + print(" {}: {}".format(key, value)) \ No newline at end of file diff --git a/inference.py b/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..f395b2f685981cc2c9f955d03b5869cece0e6d3c --- /dev/null +++ b/inference.py @@ -0,0 +1,36 @@ +import torch +import torch.nn as nn + +import onnx +import engine, data, utils +from train import device, NUM_CLASSES +from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights +from torchmetrics.classification import MulticlassAccuracy +import onnxruntime as ort +import numpy as np + + + +model = efficientnet_b0(weights=EfficientNet_B0_Weights.IMAGENET1K_V1) +model.classifier = nn.Sequential( + nn.Dropout(p = 0.2, inplace = True), + nn.Linear(1280, NUM_CLASSES), + # nn.Softmax() + ) +model = utils.load_model(model, "save_model/best_model.pth").to(device) + +PATH = "save_model/food_cpu.onnx" +# onnx inference +utils.onnx_inference(model, PATH, "cpu") +onnx_model = onnx.load(PATH) +onnx_check = onnx.checker.check_model(onnx_model) + +# print(onnx_check) +x, y = data.test_datasets[0][0], data.test_datasets[0][1] +ort_sess = ort.InferenceSession(PATH) +outputs = ort_sess.run(None, {'input': x.unsqueeze(dim = 0).numpy()}) + +# Result +classes = data.train_datasets.classes +predicted, actual = classes[outputs[0][0].argmax(0)], classes[y] +print(f'Predicted: "{predicted}", Actual: "{actual}"') \ No newline at end of file diff --git a/plots/cnn_train_Loss_and_accuracy_plot_3e-06.jpg b/plots/cnn_train_Loss_and_accuracy_plot_3e-06.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5ed287e4cdf9676f1554dc3d8017b3ed0633241a Binary files /dev/null and b/plots/cnn_train_Loss_and_accuracy_plot_3e-06.jpg differ diff --git a/plots/cnn_train_Loss_and_accuracy_plot_4e-06.jpg b/plots/cnn_train_Loss_and_accuracy_plot_4e-06.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8513367ddc1675c252946622db05cdd7fa03d59c Binary files /dev/null and b/plots/cnn_train_Loss_and_accuracy_plot_4e-06.jpg differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..fecc21b624cc23ef6c0c7eafca1b87f270175318 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +pytorch==2.0.0 +torchvision==0.15.0 +torchaudio==2.0.0 +torchmetrics +onnx +onnxruntime +optuna +wandb +torch-summary +matplotlib diff --git a/results/hyp_tune.txt b/results/hyp_tune.txt new file mode 100644 index 0000000000000000000000000000000000000000..c93b327f0665f5fcff7687fd6d74a9fc2121b82d --- /dev/null +++ b/results/hyp_tune.txt @@ -0,0 +1,606 @@ +[I 2023-10-22 01:37:08,918] A new study created in memory with name: no-name-c840de1b-897a-43af-bd8e-df563b179162 +[I 2023-10-22 02:46:09,630] Trial 0 finished with value: 0.5360714197158813 and parameters: {'lr': 7.489632455796532e-09, 'n_estimators': 34, 'optimizer': 'SGD'}. Best is trial 0 with value: 0.5360714197158813. +[I 2023-10-22 03:30:52,975] Trial 1 finished with value: 0.5364061594009399 and parameters: {'lr': 1.3698661828277252e-08, 'n_estimators': 25, 'optimizer': 'Adam'}. Best is trial 1 with value: 0.5364061594009399. +[I 2023-10-22 03:59:25,244] Trial 2 finished with value: 0.5400202870368958 and parameters: {'lr': 1.813677229052636e-07, 'n_estimators': 25, 'optimizer': 'Adam'}. Best is trial 2 with value: 0.5400202870368958. +[I 2023-10-22 04:32:04,511] Trial 3 finished with value: 0.5361357927322388 and parameters: {'lr': 8.336261616991292e-08, 'n_estimators': 44, 'optimizer': 'Adam'}. Best is trial 2 with value: 0.5400202870368958. +[I 2023-10-22 04:56:30,858] Trial 4 finished with value: 0.5398440361022949 and parameters: {'lr': 3.8570428767211705e-07, 'n_estimators': 14, 'optimizer': 'Adam'}. Best is trial 2 with value: 0.5400202870368958. +[I 2023-10-22 05:57:25,530] Trial 5 finished with value: 0.5384387969970703 and parameters: {'lr': 8.89697701761264e-09, 'n_estimators': 37, 'optimizer': 'SGD'}. Best is trial 2 with value: 0.5400202870368958. +[I 2023-10-22 06:33:16,334] Trial 6 finished with value: 0.540566623210907 and parameters: {'lr': 5.558156142085502e-08, 'n_estimators': 44, 'optimizer': 'Adam'}. Best is trial 6 with value: 0.540566623210907. +[I 2023-10-22 07:00:35,226] Trial 7 finished with value: 0.5460574626922607 and parameters: {'lr': 1.90286747029132e-07, 'n_estimators': 12, 'optimizer': 'Adam'}. Best is trial 7 with value: 0.5460574626922607. +[I 2023-10-22 07:23:51,994] Trial 8 finished with value: 0.5440860390663147 and parameters: {'lr': 9.886624025118481e-09, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 7 with value: 0.5460574626922607. +[I 2023-10-22 07:58:56,022] Trial 9 finished with value: 0.5461758971214294 and parameters: {'lr': 1.2078859717026416e-07, 'n_estimators': 26, 'optimizer': 'Adam'}. Best is trial 9 with value: 0.5461758971214294. +[I 2023-10-22 08:45:40,341] Trial 10 finished with value: 0.5450535416603088 and parameters: {'lr': 3.411775194205487e-08, 'n_estimators': 20, 'optimizer': 'RMSprop'}. Best is trial 9 with value: 0.5461758971214294. +[I 2023-10-22 09:08:56,407] Trial 11 finished with value: 0.5503671765327454 and parameters: {'lr': 1.2684791385173414e-07, 'n_estimators': 12, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454. +[I 2023-10-22 09:32:17,407] Trial 12 finished with value: 0.5460602045059204 and parameters: {'lr': 1.0571866651675711e-07, 'n_estimators': 19, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454. +[I 2023-10-22 09:59:36,467] Trial 13 finished with value: 0.5463402271270752 and parameters: {'lr': 3.143297264441004e-08, 'n_estimators': 50, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454. +[I 2023-10-22 10:26:49,809] Trial 14 finished with value: 0.5461093783378601 and parameters: {'lr': 3.0821157193882773e-08, 'n_estimators': 50, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454. +[I 2023-10-22 11:23:12,523] Trial 15 finished with value: 0.5502772331237793 and parameters: {'lr': 1.9747824446554313e-08, 'n_estimators': 32, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454. +[I 2023-10-22 12:08:08,005] Trial 16 finished with value: 0.5461443066596985 and parameters: {'lr': 1.8079689971330616e-08, 'n_estimators': 31, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454. +[I 2023-10-22 12:44:55,748] Trial 17 finished with value: 0.548389196395874 and parameters: {'lr': 1.8707046412210653e-08, 'n_estimators': 18, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454. +[I 2023-10-22 13:21:34,306] Trial 18 finished with value: 0.5481426119804382 and parameters: {'lr': 5.407206770131877e-08, 'n_estimators': 36, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454. +[I 2023-10-22 14:18:47,176] Trial 19 finished with value: 0.5486733317375183 and parameters: {'lr': 4.5994986446708456e-09, 'n_estimators': 30, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454. +[I 2023-10-22 15:03:22,294] Trial 20 finished with value: 0.5505406856536865 and parameters: {'lr': 4.3839971014050346e-08, 'n_estimators': 11, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-22 15:27:38,957] Trial 21 finished with value: 0.5479753613471985 and parameters: {'lr': 5.284735869059107e-08, 'n_estimators': 10, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-22 15:52:15,068] Trial 22 finished with value: 0.5478231310844421 and parameters: {'lr': 2.3240425242663824e-08, 'n_estimators': 15, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-22 16:16:38,410] Trial 23 finished with value: 0.5491703748703003 and parameters: {'lr': 4.4870264812671536e-08, 'n_estimators': 21, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-22 17:13:33,420] Trial 24 finished with value: 0.5481324195861816 and parameters: {'lr': 7.469682637842191e-08, 'n_estimators': 14, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-22 17:51:02,788] Trial 25 finished with value: 0.5498676300048828 and parameters: {'lr': 2.64698690852027e-08, 'n_estimators': 10, 'optimizer': 'RMSprop'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-22 18:28:07,513] Trial 26 finished with value: 0.5503433346748352 and parameters: {'lr': 3.70518809559498e-08, 'n_estimators': 29, 'optimizer': 'RMSprop'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-22 19:21:42,279] Trial 27 finished with value: 0.5504162907600403 and parameters: {'lr': 4.074495301401983e-08, 'n_estimators': 17, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-22 20:02:40,205] Trial 28 finished with value: 0.549625813961029 and parameters: {'lr': 4.287480774476896e-08, 'n_estimators': 16, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-22 20:52:10,303] Trial 29 finished with value: 0.5462378263473511 and parameters: {'lr': 6.806050743074082e-08, 'n_estimators': 12, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-22 22:02:28,677] Trial 30 finished with value: 0.5487064123153687 and parameters: {'lr': 9.595182919683205e-08, 'n_estimators': 17, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-22 22:27:17,590] Trial 31 finished with value: 0.5490727424621582 and parameters: {'lr': 3.789547094688714e-08, 'n_estimators': 22, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-22 23:04:22,026] Trial 32 finished with value: 0.5467727780342102 and parameters: {'lr': 6.407275318372976e-08, 'n_estimators': 27, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-23 00:31:00,911] Trial 33 finished with value: 0.5485678911209106 and parameters: {'lr': 3.542292563055157e-08, 'n_estimators': 23, 'optimizer': 'RMSprop'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-23 01:18:08,441] Trial 34 finished with value: 0.5498937964439392 and parameters: {'lr': 4.538312987041805e-08, 'n_estimators': 12, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-23 01:41:30,297] Trial 35 finished with value: 0.5497119426727295 and parameters: {'lr': 2.6255028232030936e-08, 'n_estimators': 14, 'optimizer': 'RMSprop'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-23 02:24:02,801] Trial 36 finished with value: 0.5490474700927734 and parameters: {'lr': 1.3889162318307476e-08, 'n_estimators': 28, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-23 02:51:04,062] Trial 37 finished with value: 0.5489190816879272 and parameters: {'lr': 1.557848733105634e-07, 'n_estimators': 23, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865. +[I 2023-10-23 03:18:20,869] Trial 38 finished with value: 0.550723135471344 and parameters: {'lr': 7.232390810885362e-08, 'n_estimators': 39, 'optimizer': 'RMSprop'}. Best is trial 38 with value: 0.550723135471344. +[I 2023-10-23 04:51:40,915] Trial 39 finished with value: 0.547078013420105 and parameters: {'lr': 8.156050875435005e-08, 'n_estimators': 40, 'optimizer': 'SGD'}. Best is trial 38 with value: 0.550723135471344. +[I 2023-10-23 05:42:10,595] Trial 40 finished with value: 0.5495409369468689 and parameters: {'lr': 8.915478213601037e-08, 'n_estimators': 34, 'optimizer': 'Adam'}. Best is trial 38 with value: 0.550723135471344. +[I 2023-10-23 06:21:10,896] Trial 41 finished with value: 0.5536542534828186 and parameters: {'lr': 5.934005291949064e-08, 'n_estimators': 38, 'optimizer': 'RMSprop'}. Best is trial 41 with value: 0.5536542534828186. +[I 2023-10-23 06:48:23,716] Trial 42 finished with value: 0.552158534526825 and parameters: {'lr': 6.448909468854163e-08, 'n_estimators': 40, 'optimizer': 'RMSprop'}. Best is trial 41 with value: 0.5536542534828186. +[I 2023-10-23 07:15:32,410] Trial 43 finished with value: 0.5526768565177917 and parameters: {'lr': 6.259029563811938e-08, 'n_estimators': 40, 'optimizer': 'RMSprop'}. Best is trial 41 with value: 0.5536542534828186. +[I 2023-10-23 07:50:36,379] Trial 44 finished with value: 0.5551037788391113 and parameters: {'lr': 5.640725954569493e-08, 'n_estimators': 39, 'optimizer': 'RMSprop'}. Best is trial 44 with value: 0.5551037788391113. +[I 2023-10-23 08:21:58,489] Trial 45 finished with value: 0.5539408922195435 and parameters: {'lr': 6.562450771563963e-08, 'n_estimators': 39, 'optimizer': 'RMSprop'}. Best is trial 44 with value: 0.5551037788391113. +[I 2023-10-23 08:53:04,595] Trial 46 finished with value: 0.5582337379455566 and parameters: {'lr': 6.502594269660963e-08, 'n_estimators': 46, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566. +[I 2023-10-23 09:20:24,889] Trial 47 finished with value: 0.5553371906280518 and parameters: {'lr': 5.355589591906474e-08, 'n_estimators': 47, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566. +[I 2023-10-23 09:43:43,726] Trial 48 finished with value: 0.556071400642395 and parameters: {'lr': 1.0555536719402554e-07, 'n_estimators': 47, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566. +[I 2023-10-23 10:07:21,421] Trial 49 finished with value: 0.5569140911102295 and parameters: {'lr': 1.0687358012554984e-07, 'n_estimators': 47, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566. +[I 2023-10-23 10:42:52,432] Trial 50 finished with value: 0.5552547574043274 and parameters: {'lr': 1.0462986496303203e-07, 'n_estimators': 47, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566. +[I 2023-10-23 11:10:10,852] Trial 51 finished with value: 0.5569719076156616 and parameters: {'lr': 1.0981543368796825e-07, 'n_estimators': 47, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566. +[I 2023-10-23 11:42:13,758] Trial 52 finished with value: 0.5563305020332336 and parameters: {'lr': 1.1460389239037368e-07, 'n_estimators': 47, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566. +[I 2023-10-23 12:15:05,177] Trial 53 finished with value: 0.5571174621582031 and parameters: {'lr': 1.3916428640610327e-07, 'n_estimators': 46, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566. +[I 2023-10-23 12:51:50,898] Trial 54 finished with value: 0.5606839060783386 and parameters: {'lr': 1.2238913630137383e-07, 'n_estimators': 45, 'optimizer': 'RMSprop'}. Best is trial 54 with value: 0.5606839060783386. +[I 2023-10-23 13:16:21,095] Trial 55 finished with value: 0.5585974454879761 and parameters: {'lr': 1.3523286029041088e-07, 'n_estimators': 44, 'optimizer': 'RMSprop'}. Best is trial 54 with value: 0.5606839060783386. +[I 2023-10-23 13:40:52,822] Trial 56 finished with value: 0.563145101070404 and parameters: {'lr': 1.321765003562829e-07, 'n_estimators': 44, 'optimizer': 'RMSprop'}. Best is trial 56 with value: 0.563145101070404. +[I 2023-10-23 14:17:34,503] Trial 57 finished with value: 0.5649504661560059 and parameters: {'lr': 1.417339501340407e-07, 'n_estimators': 44, 'optimizer': 'Adam'}. Best is trial 57 with value: 0.5649504661560059. +[I 2023-10-23 15:06:34,962] Trial 58 finished with value: 0.5648221969604492 and parameters: {'lr': 2.1270825073760548e-07, 'n_estimators': 44, 'optimizer': 'Adam'}. Best is trial 57 with value: 0.5649504661560059. +[I 2023-10-23 15:43:43,175] Trial 59 finished with value: 0.5651143193244934 and parameters: {'lr': 2.0658178964301883e-07, 'n_estimators': 43, 'optimizer': 'Adam'}. Best is trial 59 with value: 0.5651143193244934. +[I 2023-10-23 16:12:19,615] Trial 60 finished with value: 0.5670831203460693 and parameters: {'lr': 2.0425588190663883e-07, 'n_estimators': 43, 'optimizer': 'Adam'}. Best is trial 60 with value: 0.5670831203460693. +[I 2023-10-23 16:36:55,347] Trial 61 finished with value: 0.570649266242981 and parameters: {'lr': 2.1582322766042557e-07, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 61 with value: 0.570649266242981. +[I 2023-10-23 17:14:30,922] Trial 62 finished with value: 0.5675162076950073 and parameters: {'lr': 2.3392754826613693e-07, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 61 with value: 0.570649266242981. +[I 2023-10-23 18:07:46,984] Trial 63 finished with value: 0.5706519484519958 and parameters: {'lr': 2.2526583690421322e-07, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 63 with value: 0.5706519484519958. +[I 2023-10-23 18:32:20,406] Trial 64 finished with value: 0.572399377822876 and parameters: {'lr': 2.2729787427772587e-07, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 64 with value: 0.572399377822876. +[I 2023-10-23 19:34:18,729] Trial 65 finished with value: 0.5735474824905396 and parameters: {'lr': 2.4068578827383907e-07, 'n_estimators': 43, 'optimizer': 'Adam'}. Best is trial 65 with value: 0.5735474824905396. +[I 2023-10-23 19:58:59,725] Trial 66 finished with value: 0.5737140774726868 and parameters: {'lr': 2.507074965520075e-07, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 66 with value: 0.5737140774726868. +[I 2023-10-23 20:27:39,393] Trial 67 finished with value: 0.575024425983429 and parameters: {'lr': 2.7138721673308e-07, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 67 with value: 0.575024425983429. +[I 2023-10-23 20:56:36,970] Trial 68 finished with value: 0.5768953561782837 and parameters: {'lr': 2.8770118447252704e-07, 'n_estimators': 36, 'optimizer': 'Adam'}. Best is trial 68 with value: 0.5768953561782837. +[I 2023-10-23 21:21:26,705] Trial 69 finished with value: 0.578156590461731 and parameters: {'lr': 2.8443655413359384e-07, 'n_estimators': 36, 'optimizer': 'Adam'}. Best is trial 69 with value: 0.578156590461731. +[I 2023-10-23 21:46:06,503] Trial 70 finished with value: 0.5815342664718628 and parameters: {'lr': 2.9882598602116e-07, 'n_estimators': 36, 'optimizer': 'Adam'}. Best is trial 70 with value: 0.5815342664718628. +[I 2023-10-23 22:10:52,256] Trial 71 finished with value: 0.5806370377540588 and parameters: {'lr': 2.930792868424465e-07, 'n_estimators': 35, 'optimizer': 'Adam'}. Best is trial 70 with value: 0.5815342664718628. + +Model Loaded. + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth +[I 2023-10-23 22:40:28,127] Trial 72 finished with value: 0.584173321723938 and parameters: {'lr': 3.072183458986638e-07, 'n_estimators': 35, 'optimizer': 'Adam'}. Best is trial 72 with value: 0.584173321723938. +[I 2023-10-23 23:21:59,127] Trial 73 finished with value: 0.5868313312530518 and parameters: {'lr': 3.1505289704826233e-07, 'n_estimators': 35, 'optimizer': 'Adam'}. Best is trial 73 with value: 0.5868313312530518. +[I 2023-10-24 00:03:21,470] Trial 74 finished with value: 0.5852217674255371 and parameters: {'lr': 3.37944519125002e-07, 'n_estimators': 35, 'optimizer': 'Adam'}. Best is trial 73 with value: 0.5868313312530518. +[I 2023-10-24 00:28:20,202] Trial 75 finished with value: 0.5888196229934692 and parameters: {'lr': 3.4279797801428923e-07, 'n_estimators': 35, 'optimizer': 'Adam'}. Best is trial 75 with value: 0.5888196229934692. +[I 2023-10-24 01:10:32,472] Trial 76 finished with value: 0.5885568857192993 and parameters: {'lr': 3.7785727396953406e-07, 'n_estimators': 35, 'optimizer': 'Adam'}. Best is trial 75 with value: 0.5888196229934692. +[I 2023-10-24 01:47:41,292] Trial 77 finished with value: 0.5886940956115723 and parameters: {'lr': 3.730182606568301e-07, 'n_estimators': 33, 'optimizer': 'Adam'}. Best is trial 75 with value: 0.5888196229934692. +[I 2023-10-24 02:16:42,436] Trial 78 finished with value: 0.5928004384040833 and parameters: {'lr': 3.98740229310406e-07, 'n_estimators': 33, 'optimizer': 'Adam'}. Best is trial 78 with value: 0.5928004384040833. +[I 2023-10-24 02:54:09,980] Trial 79 finished with value: 0.596867561340332 and parameters: {'lr': 3.951220434815224e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 79 with value: 0.596867561340332. +[I 2023-10-24 03:18:56,660] Trial 80 finished with value: 0.5987726449966431 and parameters: {'lr': 3.9217337610956314e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 80 with value: 0.5987726449966431. +[I 2023-10-24 03:43:56,636] Trial 81 finished with value: 0.6010125279426575 and parameters: {'lr': 3.8983258726821844e-07, 'n_estimators': 33, 'optimizer': 'Adam'}. Best is trial 81 with value: 0.6010125279426575. +[I 2023-10-24 04:12:53,577] Trial 82 finished with value: 0.6015823483467102 and parameters: {'lr': 3.9805005014361763e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 82 with value: 0.6015823483467102. +[I 2023-10-24 05:14:20,760] Trial 83 finished with value: 0.6092749238014221 and parameters: {'lr': 3.988667591679828e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 83 with value: 0.6092749238014221. +[I 2023-10-24 05:38:45,088] Trial 84 finished with value: 0.6045148968696594 and parameters: {'lr': 3.801315324866347e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 83 with value: 0.6092749238014221. +[I 2023-10-24 06:07:41,056] Trial 85 finished with value: 0.6064762473106384 and parameters: {'lr': 3.9351790098082646e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 83 with value: 0.6092749238014221. +[I 2023-10-24 06:36:23,798] Trial 86 finished with value: 0.6090301871299744 and parameters: {'lr': 3.8907654722821625e-07, 'n_estimators': 31, 'optimizer': 'Adam'}. Best is trial 83 with value: 0.6092749238014221. +[I 2023-10-24 07:05:17,713] Trial 87 finished with value: 0.6128934025764465 and parameters: {'lr': 3.7610862993862156e-07, 'n_estimators': 31, 'optimizer': 'Adam'}. Best is trial 87 with value: 0.6128934025764465. +[I 2023-10-24 07:42:09,228] Trial 88 finished with value: 0.6093340516090393 and parameters: {'lr': 3.4982809838009586e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 87 with value: 0.6128934025764465. +[I 2023-10-24 08:19:16,153] Trial 89 finished with value: 0.6141061186790466 and parameters: {'lr': 3.3943441707865654e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 89 with value: 0.6141061186790466. +[I 2023-10-24 08:47:53,069] Trial 90 finished with value: 0.6181385517120361 and parameters: {'lr': 3.415242523802862e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 90 with value: 0.6181385517120361. +[I 2023-10-24 09:16:44,855] Trial 91 finished with value: 0.6147274374961853 and parameters: {'lr': 3.3757698574028605e-07, 'n_estimators': 29, 'optimizer': 'Adam'}. Best is trial 90 with value: 0.6181385517120361. +[I 2023-10-24 09:49:23,877] Trial 92 finished with value: 0.618079423904419 and parameters: {'lr': 3.369688207850333e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 90 with value: 0.6181385517120361. +[I 2023-10-24 10:47:19,401] Trial 93 finished with value: 0.6206730008125305 and parameters: {'lr': 3.333427116728784e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 93 with value: 0.6206730008125305. +[I 2023-10-24 11:24:06,673] Trial 94 finished with value: 0.6197251081466675 and parameters: {'lr': 3.461562208850868e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 93 with value: 0.6206730008125305. +[I 2023-10-24 12:08:51,353] Trial 95 finished with value: 0.6243206262588501 and parameters: {'lr': 3.3584644827281837e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 95 with value: 0.6243206262588501. +[I 2023-10-24 12:33:05,496] Trial 96 finished with value: 0.6249201893806458 and parameters: {'lr': 3.287811197213919e-07, 'n_estimators': 29, 'optimizer': 'Adam'}. Best is trial 96 with value: 0.6249201893806458. +[I 2023-10-24 13:02:04,516] Trial 97 finished with value: 0.6230003237724304 and parameters: {'lr': 3.354841731848597e-07, 'n_estimators': 29, 'optimizer': 'Adam'}. Best is trial 96 with value: 0.6249201893806458. +[I 2023-10-24 13:27:03,953] Trial 98 finished with value: 0.6289621591567993 and parameters: {'lr': 3.241147612310783e-07, 'n_estimators': 26, 'optimizer': 'Adam'}. Best is trial 98 with value: 0.6289621591567993. +[I 2023-10-24 13:52:14,918] Trial 99 finished with value: 0.625308096408844 and parameters: {'lr': 3.287683345694601e-07, 'n_estimators': 26, 'optimizer': 'Adam'}. Best is trial 98 with value: 0.6289621591567993. + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Study statistics: + Number of finished trials: 100 + Number of pruned trials: 0 + Number of complete trials: 100 +Best trial: + Value: 0.6289621591567993 + Params: + lr: 3.241147612310783e-07 + n_estimators: 26 + optimizer: Adam diff --git a/results/hyp_with_wandb.txt b/results/hyp_with_wandb.txt new file mode 100644 index 0000000000000000000000000000000000000000..39c4a04aa2cc436092f5e6f9ce1eeab20c15ebc8 --- /dev/null +++ b/results/hyp_with_wandb.txt @@ -0,0 +1,323 @@ +wandb: Currently logged in as: jashpatel8561 (maa_64). Use `wandb login --relogin` to force relogin +wandb: Appending key for api.wandb.ai to your netrc file: /root/.netrc +[I 2023-10-31 20:08:45,039] A new study created in memory with name: no-name-881464b3-8bd6-47d5-8e36-f91094c9f780 +wandb: Tracking run with wandb version 0.15.12 +wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run scary-treat-7 +wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app +wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/8vkss7as +wandb: Waiting for W&B process to finish... (success). +wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: | 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: / 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: - 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: Train Accuracy ▁ +wandb: Train Loss ▁ +wandb: Validation Accuracy ▁ +wandb: Validation Loss ▁ +wandb: +wandb: Run summary: +wandb: Train Accuracy 0.47619 +wandb: Train Loss 1.52292 +wandb: Validation Accuracy 0.53696 +wandb: Validation Loss 1.31054 +wandb: +wandb: 🚀 View run scary-treat-7 at: https://wandb.ai/maa_64/food-app/runs/8vkss7as +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20231031_200845-8vkss7as/logs +[I 2023-10-31 20:53:40,276] Trial 0 finished with value: 0.5369605422019958 and parameters: {'lr': 3.746494558899465e-06, 'n_estimators': 31, 'optimizer': 'SGD'}. Best is trial 0 with value: 0.5369605422019958. +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: Tracking run with wandb version 0.15.12 +wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run strange-cape-8 +wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app +wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/97cre3xd +wandb: Waiting for W&B process to finish... (success). +wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: | 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: / 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: | 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: Train Accuracy ▁ +wandb: Train Loss ▁ +wandb: Validation Accuracy ▁ +wandb: Validation Loss ▁ +wandb: +wandb: Run summary: +wandb: Train Accuracy 0.48417 +wandb: Train Loss 1.49974 +wandb: Validation Accuracy 0.53469 +wandb: Validation Loss 1.30942 +wandb: +wandb: 🚀 View run strange-cape-8 at: https://wandb.ai/maa_64/food-app/runs/97cre3xd +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20231031_205340-97cre3xd/logs +[I 2023-10-31 21:26:05,821] Trial 1 finished with value: 0.5346873998641968 and parameters: {'lr': 2.1962752768229064e-07, 'n_estimators': 31, 'optimizer': 'RMSprop'}. Best is trial 0 with value: 0.5369605422019958. +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: Tracking run with wandb version 0.15.12 +wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run mischievous-wendigo-9 +wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app +wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/2ork0ogz +wandb: Waiting for W&B process to finish... (success). +wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: | 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: / 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: Train Accuracy ▁ +wandb: Train Loss ▁ +wandb: Validation Accuracy ▁ +wandb: Validation Loss ▁ +wandb: +wandb: Run summary: +wandb: Train Accuracy 0.48504 +wandb: Train Loss 1.49989 +wandb: Validation Accuracy 0.5386 +wandb: Validation Loss 1.29999 +wandb: +wandb: 🚀 View run mischievous-wendigo-9 at: https://wandb.ai/maa_64/food-app/runs/2ork0ogz +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20231031_212605-2ork0ogz/logs +[I 2023-10-31 21:54:42,788] Trial 2 finished with value: 0.5385968685150146 and parameters: {'lr': 4.9567301150128566e-08, 'n_estimators': 43, 'optimizer': 'RMSprop'}. Best is trial 2 with value: 0.5385968685150146. +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: Tracking run with wandb version 0.15.12 +wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run masked-cape-10 +wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app +wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/pegj5p38 +wandb: Waiting for W&B process to finish... (success). +wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: | 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: / 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: - 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: \ 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: Train Accuracy ▁ +wandb: Train Loss ▁ +wandb: Validation Accuracy ▁ +wandb: Validation Loss ▁ +wandb: +wandb: Run summary: +wandb: Train Accuracy 0.48078 +wandb: Train Loss 1.49002 +wandb: Validation Accuracy 0.54252 +wandb: Validation Loss 1.30474 +wandb: +wandb: 🚀 View run masked-cape-10 at: https://wandb.ai/maa_64/food-app/runs/pegj5p38 +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20231031_215442-pegj5p38/logs +[I 2023-10-31 22:19:18,241] Trial 3 finished with value: 0.5425158739089966 and parameters: {'lr': 3.6384818250623903e-06, 'n_estimators': 33, 'optimizer': 'SGD'}. Best is trial 3 with value: 0.5425158739089966. +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: Tracking run with wandb version 0.15.12 +wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run strange-wraith-11 +wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app +wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/gr6ablo3 +wandb: Waiting for W&B process to finish... (success). +wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: | 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: / 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: - 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: \ 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: | 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: Train Accuracy ▁ +wandb: Train Loss ▁ +wandb: Validation Accuracy ▁ +wandb: Validation Loss ▁ +wandb: +wandb: Run summary: +wandb: Train Accuracy 0.48708 +wandb: Train Loss 1.49079 +wandb: Validation Accuracy 0.54065 +wandb: Validation Loss 1.29405 +wandb: +wandb: 🚀 View run strange-wraith-11 at: https://wandb.ai/maa_64/food-app/runs/gr6ablo3 +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20231031_221918-gr6ablo3/logs +[I 2023-10-31 22:43:49,791] Trial 4 finished with value: 0.5406512022018433 and parameters: {'lr': 3.1757764065521707e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 3 with value: 0.5425158739089966. +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: Tracking run with wandb version 0.15.12 +wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run occult-banshee-12 +wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app +wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/6h4rftgh +wandb: Waiting for W&B process to finish... (success). +wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: | 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: / 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: - 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: Train Accuracy ▁ +wandb: Train Loss ▁ +wandb: Validation Accuracy ▁ +wandb: Validation Loss ▁ +wandb: +wandb: Run summary: +wandb: Train Accuracy 0.48493 +wandb: Train Loss 1.48231 +wandb: Validation Accuracy 0.54034 +wandb: Validation Loss 1.29421 +wandb: +wandb: 🚀 View run occult-banshee-12 at: https://wandb.ai/maa_64/food-app/runs/6h4rftgh +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20231031_224349-6h4rftgh/logs +[I 2023-10-31 23:44:49,930] Trial 5 finished with value: 0.5403411984443665 and parameters: {'lr': 4.079324797271018e-08, 'n_estimators': 32, 'optimizer': 'RMSprop'}. Best is trial 3 with value: 0.5425158739089966. +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: Tracking run with wandb version 0.15.12 +wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run occult-gargoyle-13 +wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app +wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/7nbh63pb +wandb: Waiting for W&B process to finish... (success). +wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: | 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: / 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: - 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: Train Accuracy ▁ +wandb: Train Loss ▁ +wandb: Validation Accuracy ▁ +wandb: Validation Loss ▁ +wandb: +wandb: Run summary: +wandb: Train Accuracy 0.49463 +wandb: Train Loss 1.45459 +wandb: Validation Accuracy 0.54955 +wandb: Validation Loss 1.27562 +wandb: +wandb: 🚀 View run occult-gargoyle-13 at: https://wandb.ai/maa_64/food-app/runs/7nbh63pb +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20231031_234449-7nbh63pb/logs +[I 2023-11-01 00:13:51,934] Trial 6 finished with value: 0.5495542883872986 and parameters: {'lr': 4.678309863383098e-07, 'n_estimators': 35, 'optimizer': 'RMSprop'}. Best is trial 6 with value: 0.5495542883872986. +wandb: Tracking run with wandb version 0.15.12 +wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run dreadful-whisper-14 +wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app +wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/jyel0gll +wandb: Waiting for W&B process to finish... (success). +wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: | 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: / 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: - 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: \ 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: Train Accuracy ▁ +wandb: Train Loss ▁ +wandb: Validation Accuracy ▁ +wandb: Validation Loss ▁ +wandb: +wandb: Run summary: +wandb: Train Accuracy 0.50997 +wandb: Train Loss 1.39204 +wandb: Validation Accuracy 0.55506 +wandb: Validation Loss 1.23738 +wandb: +wandb: 🚀 View run dreadful-whisper-14 at: https://wandb.ai/maa_64/food-app/runs/jyel0gll +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20231101_001351-jyel0gll/logs +[I 2023-11-01 00:38:23,845] Trial 7 finished with value: 0.5550567507743835 and parameters: {'lr': 1.7101161575656505e-06, 'n_estimators': 29, 'optimizer': 'Adam'}. Best is trial 7 with value: 0.5550567507743835. +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: | Waiting for wandb.init()... wandb: Tracking run with wandb version 0.15.12 +wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run mischievous-specter-15 +wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app +wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/t9dkdynt +wandb: Waiting for W&B process to finish... (success). +wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: | 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: / 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: - 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: \ 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: | 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: Train Accuracy ▁ +wandb: Train Loss ▁ +wandb: Validation Accuracy ▁ +wandb: Validation Loss ▁ +wandb: +wandb: Run summary: +wandb: Train Accuracy 0.50968 +wandb: Train Loss 1.38178 +wandb: Validation Accuracy 0.55892 +wandb: Validation Loss 1.22877 +wandb: +wandb: 🚀 View run mischievous-specter-15 at: https://wandb.ai/maa_64/food-app/runs/t9dkdynt +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20231101_003823-t9dkdynt/logs +[I 2023-11-01 01:18:49,091] Trial 8 finished with value: 0.5589190125465393 and parameters: {'lr': 2.807500616871119e-07, 'n_estimators': 35, 'optimizer': 'SGD'}. Best is trial 8 with value: 0.5589190125465393. +wandb: Tracking run with wandb version 0.15.12 +wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run spooky-specter-16 +wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app +wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/ota77xrr +wandb: Waiting for W&B process to finish... (success). +wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: | 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: / 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: - 0.006 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: \ 0.006 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: Train Accuracy ▁ +wandb: Train Loss ▁ +wandb: Validation Accuracy ▁ +wandb: Validation Loss ▁ +wandb: +wandb: Run summary: +wandb: Train Accuracy 0.52209 +wandb: Train Loss 1.34973 +wandb: Validation Accuracy 0.56162 +wandb: Validation Loss 1.2041 +wandb: +wandb: 🚀 View run spooky-specter-16 at: https://wandb.ai/maa_64/food-app/runs/ota77xrr +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20231101_011849-ota77xrr/logs +[I 2023-11-01 01:46:58,059] Trial 9 finished with value: 0.5616196393966675 and parameters: {'lr': 1.0143024563093913e-06, 'n_estimators': 33, 'optimizer': 'Adam'}. Best is trial 9 with value: 0.5616196393966675. +wandb: - Waiting for wandb.init()... wandb: \ Waiting for wandb.init()... wandb: Tracking run with wandb version 0.15.12 +wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run fearful-candle-17 +wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app +wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/mj672tzu +wandb: Waiting for W&B process to finish... (success). +wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: \ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped) wandb: | 0.005 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: / 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: - 0.014 MB of 0.014 MB uploaded (0.000 MB deduped) wandb: +wandb: Run history: +wandb: Train Accuracy ▁ +wandb: Train Loss ▁ +wandb: Validation Accuracy ▁ +wandb: Validation Loss ▁ +wandb: +wandb: Run summary: +wandb: Train Accuracy 0.52938 +wandb: Train Loss 1.31677 +wandb: Validation Accuracy 0.56583 +wandb: Validation Loss 1.20021 +wandb: +wandb: 🚀 View run fearful-candle-17 at: https://wandb.ai/maa_64/food-app/runs/mj672tzu +wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: ./wandb/run-20231101_014658-mj672tzu/logs +[I 2023-11-01 02:11:05,590] Trial 10 finished with value: 0.5658254027366638 and parameters: {'lr': 9.441552500921141e-07, 'n_estimators': 25, 'optimizer': 'Adam'}. Best is trial 10 with value: 0.5658254027366638. +[LOG]: Login Succesfull. + +Model Loaded. + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Saving Model At: save_model/best_model.pth + +Study statistics: + Number of finished trials: 11 + Number of pruned trials: 0 + Number of complete trials: 11 +Best trial: + Value: 0.5658254027366638 + Params: + lr: 9.441552500921141e-07 + n_estimators: 25 + optimizer: Adam diff --git a/results/inference.txt b/results/inference.txt new file mode 100644 index 0000000000000000000000000000000000000000..5edc883026e77a5f146919fbd102d83b50afaebc --- /dev/null +++ b/results/inference.txt @@ -0,0 +1,7 @@ + +Model Loaded. +============= Diagnostic Run torch.onnx.export version 2.0.1+cu117 ============= +verbose: False, log level: Level.ERROR +======================= 0 NONE 0 NOTE 0 WARNING 0 ERROR ======================== + +Predicted: "foie_gras", Actual: "churros" diff --git a/results/norm_result.txt b/results/norm_result.txt new file mode 100644 index 0000000000000000000000000000000000000000..319b8cc3877903ffd5df23fc634372b69282c0b1 --- /dev/null +++ b/results/norm_result.txt @@ -0,0 +1,153 @@ + 0%| | 0/50 [00:00 + study.optimize(objective, n_trials=11) + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/study.py", line 451, in optimize + _optimize( + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 66, in _optimize + _optimize_sequential( + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 163, in _optimize_sequential + frozen_trial = _run_trial(study, func, catch) + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 251, in _run_trial + raise func_err + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 200, in _run_trial + value_or_values = func(trial) + File "hyperpara.py", line 68, in objective + val_loss, val_acc = engine.test_loop(model = model, dataloader = data.valid_dataloader, loss_fn = loss_fn, +TypeError: test_loop() got an unexpected keyword argument 'log_images' \ No newline at end of file diff --git a/wandb/run-20231031_141809-v2q0e45b/files/requirements.txt b/wandb/run-20231031_141809-v2q0e45b/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f84e89ededbacd967e61bd8559b50c3efecc0021 --- /dev/null +++ b/wandb/run-20231031_141809-v2q0e45b/files/requirements.txt @@ -0,0 +1,133 @@ +accelerate==0.22.0 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +executing==1.2.0 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +greenlet==3.0.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +regex==2023.8.8 +requests==2.28.1 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +six==1.16.0 +smmap==5.0.1 +sqlalchemy==2.0.22 +stack-data==0.6.2 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typing-extensions==4.4.0 +tzdata==2023.3 +urllib3==1.26.13 +wandb==0.15.12 +wcwidth==0.2.8 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_141809-v2q0e45b/files/wandb-metadata.json b/wandb/run-20231031_141809-v2q0e45b/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6baf7386deeedfef2c83d248cde1dd6fcc4c6e59 --- /dev/null +++ b/wandb/run-20231031_141809-v2q0e45b/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T10:18:11.054433", + "startedAt": "2023-10-31T10:18:09.552938", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.4950249999999998, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.191, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.601, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.541, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.52, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.195, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.204, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.7, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.205, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.203, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.207, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.203, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.597, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.595, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.548, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.195, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.207, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.7576103210449 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_141809-v2q0e45b/files/wandb-summary.json b/wandb/run-20231031_141809-v2q0e45b/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7eefd14f34818f4487c29e2d62e20ef87c177fd9 --- /dev/null +++ b/wandb/run-20231031_141809-v2q0e45b/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 149}} \ No newline at end of file diff --git a/wandb/run-20231031_141809-v2q0e45b/logs/debug-internal.log b/wandb/run-20231031_141809-v2q0e45b/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d03cb688803460adeac9971d24f9795b32668315 --- /dev/null +++ b/wandb/run-20231031_141809-v2q0e45b/logs/debug-internal.log @@ -0,0 +1,248 @@ +2023-10-31 14:18:09,563 INFO StreamThr :32614 [internal.py:wandb_internal():86] W&B internal server running at pid: 32614, started at: 2023-10-31 14:18:09.561968 +2023-10-31 14:18:09,565 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status +2023-10-31 14:18:09,568 INFO WriterThread:32614 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/run-v2q0e45b.wandb +2023-10-31 14:18:09,572 DEBUG SenderThread:32614 [sender.py:send():380] send: header +2023-10-31 14:18:09,572 DEBUG SenderThread:32614 [sender.py:send():380] send: run +2023-10-31 14:18:10,510 INFO SenderThread:32614 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files +2023-10-31 14:18:10,510 INFO SenderThread:32614 [sender.py:_start_run_threads():1122] run started: v2q0e45b with start time 1698747489.560302 +2023-10-31 14:18:10,527 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 14:18:10,527 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: check_version +2023-10-31 14:18:10,974 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 14:18:10,990 DEBUG HandlerThread:32614 [system_info.py:__init__():32] System info init +2023-10-31 14:18:10,990 DEBUG HandlerThread:32614 [system_info.py:__init__():47] System info init done +2023-10-31 14:18:10,990 INFO HandlerThread:32614 [system_monitor.py:start():194] Starting system monitor +2023-10-31 14:18:10,990 INFO SystemMonitor:32614 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 14:18:10,990 INFO HandlerThread:32614 [system_monitor.py:probe():214] Collecting system info +2023-10-31 14:18:10,991 INFO SystemMonitor:32614 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 14:18:10,992 INFO SystemMonitor:32614 [interfaces.py:start():190] Started disk monitoring +2023-10-31 14:18:10,994 INFO SystemMonitor:32614 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 14:18:10,995 INFO SystemMonitor:32614 [interfaces.py:start():190] Started memory monitoring +2023-10-31 14:18:10,997 INFO SystemMonitor:32614 [interfaces.py:start():190] Started network monitoring +2023-10-31 14:18:11,054 DEBUG HandlerThread:32614 [system_info.py:probe():196] Probing system +2023-10-31 14:18:11,061 DEBUG HandlerThread:32614 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 14:18:11,061 DEBUG HandlerThread:32614 [system_info.py:probe():244] Probing system done +2023-10-31 14:18:11,061 DEBUG HandlerThread:32614 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T10:18:11.054433', 'startedAt': '2023-10-31T10:18:09.552938', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.4950249999999998, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.191, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.601, 'min': 1200.0, 'max': 3600.0}, {'current': 1.541, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.52, 'min': 1200.0, 'max': 3600.0}, {'current': 2.195, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.204, 'min': 1200.0, 'max': 3600.0}, {'current': 2.7, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.203, 'min': 1200.0, 'max': 3600.0}, {'current': 1.207, 'min': 1200.0, 'max': 3600.0}, {'current': 1.203, 'min': 1200.0, 'max': 3600.0}, {'current': 1.597, 'min': 1200.0, 'max': 3600.0}, {'current': 1.595, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.548, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 2.195, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.207, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.7576103210449}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 14:18:11,062 INFO HandlerThread:32614 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 14:18:11,062 INFO HandlerThread:32614 [system_monitor.py:probe():227] Publishing system info +2023-10-31 14:18:11,062 DEBUG HandlerThread:32614 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 14:18:11,062 DEBUG HandlerThread:32614 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 14:18:11,062 DEBUG HandlerThread:32614 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 14:18:11,515 INFO Thread-12 :32614 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/requirements.txt +2023-10-31 14:18:11,516 INFO Thread-12 :32614 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/conda-environment.yaml +2023-10-31 14:18:17,500 DEBUG HandlerThread:32614 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 14:18:17,502 INFO HandlerThread:32614 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 14:18:17,507 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:18:17,507 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 14:18:17,508 DEBUG SenderThread:32614 [sender.py:send():380] send: files +2023-10-31 14:18:17,508 INFO SenderThread:32614 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 14:18:17,517 INFO Thread-12 :32614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/conda-environment.yaml +2023-10-31 14:18:17,517 INFO Thread-12 :32614 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/wandb-metadata.json +2023-10-31 14:18:17,523 DEBUG SenderThread:32614 [sender.py:send():380] send: telemetry +2023-10-31 14:18:17,569 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:18:17,569 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:18:17,570 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:18:18,740 INFO wandb-upload_0:32614 [upload_job.py:push():131] Uploaded file /tmp/tmp10vit22bwandb/9tnxfuxr-wandb-metadata.json +2023-10-31 14:18:20,076 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:18:25,077 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:18:30,078 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:18:32,520 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:18:32,521 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:18:32,561 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:18:35,879 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:18:40,890 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:18:41,528 INFO Thread-12 :32614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/config.yaml +2023-10-31 14:18:46,248 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:18:47,520 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:18:47,521 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:18:47,561 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:18:51,871 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:18:56,871 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:19:01,872 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:19:02,521 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:19:02,521 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:19:02,561 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:19:06,907 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:19:10,997 DEBUG SystemMonitor:32614 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 14:19:11,000 DEBUG SenderThread:32614 [sender.py:send():380] send: stats +2023-10-31 14:19:12,002 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:19:17,013 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:19:17,521 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:19:17,521 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:19:17,561 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:19:22,964 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:19:27,965 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:19:32,521 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:19:32,521 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:19:32,569 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:19:33,943 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:19:38,943 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:19:41,003 DEBUG SenderThread:32614 [sender.py:send():380] send: stats +2023-10-31 14:19:44,005 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:19:47,521 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:19:47,521 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:19:47,565 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:19:49,961 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:19:54,962 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:19:59,962 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:20:02,521 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:20:02,521 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:20:02,565 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:20:05,915 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:20:10,916 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:20:11,004 DEBUG SenderThread:32614 [sender.py:send():380] send: stats +2023-10-31 14:20:16,006 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:20:17,521 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:20:17,522 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:20:17,565 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:20:21,880 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:20:26,881 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:20:31,882 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:20:32,521 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:20:32,522 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:20:32,565 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:20:36,906 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:20:40,565 DEBUG SenderThread:32614 [sender.py:send():380] send: exit +2023-10-31 14:20:40,565 INFO SenderThread:32614 [sender.py:send_exit():585] handling exit code: 1 +2023-10-31 14:20:40,565 INFO SenderThread:32614 [sender.py:send_exit():587] handling runtime: 149 +2023-10-31 14:20:40,567 INFO SenderThread:32614 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 14:20:40,568 INFO SenderThread:32614 [sender.py:send_exit():593] send defer +2023-10-31 14:20:40,568 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:40,568 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 0 +2023-10-31 14:20:40,569 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:40,569 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-10-31 14:20:40,569 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 1 +2023-10-31 14:20:40,569 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:40,569 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 1 +2023-10-31 14:20:40,569 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:40,570 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-10-31 14:20:40,570 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 2 +2023-10-31 14:20:40,570 INFO Thread-12 :32614 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/output.log +2023-10-31 14:20:40,570 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:40,570 INFO Thread-12 :32614 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/wandb-summary.json +2023-10-31 14:20:40,571 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 2 +2023-10-31 14:20:40,571 INFO HandlerThread:32614 [system_monitor.py:finish():203] Stopping system monitor +2023-10-31 14:20:40,571 DEBUG SystemMonitor:32614 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-10-31 14:20:40,572 INFO HandlerThread:32614 [interfaces.py:finish():202] Joined cpu monitor +2023-10-31 14:20:40,572 DEBUG SystemMonitor:32614 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-10-31 14:20:40,572 INFO HandlerThread:32614 [interfaces.py:finish():202] Joined disk monitor +2023-10-31 14:20:42,289 INFO HandlerThread:32614 [interfaces.py:finish():202] Joined gpu monitor +2023-10-31 14:20:42,289 INFO HandlerThread:32614 [interfaces.py:finish():202] Joined memory monitor +2023-10-31 14:20:42,290 INFO HandlerThread:32614 [interfaces.py:finish():202] Joined network monitor +2023-10-31 14:20:42,290 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:20:42,291 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:42,291 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-10-31 14:20:42,291 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 3 +2023-10-31 14:20:42,292 DEBUG SenderThread:32614 [sender.py:send():380] send: stats +2023-10-31 14:20:42,292 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:42,293 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:20:42,293 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 3 +2023-10-31 14:20:42,294 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:20:42,294 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:42,295 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-10-31 14:20:42,295 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 4 +2023-10-31 14:20:42,295 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:42,295 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 4 +2023-10-31 14:20:42,296 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:42,296 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-10-31 14:20:42,296 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 5 +2023-10-31 14:20:42,296 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:42,296 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 5 +2023-10-31 14:20:42,297 DEBUG SenderThread:32614 [sender.py:send():380] send: summary +2023-10-31 14:20:42,297 INFO SenderThread:32614 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 14:20:42,298 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:42,298 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-10-31 14:20:42,298 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 6 +2023-10-31 14:20:42,298 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:42,298 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 6 +2023-10-31 14:20:42,299 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:42,299 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-10-31 14:20:42,299 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 7 +2023-10-31 14:20:42,299 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:20:42,299 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:42,300 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 7 +2023-10-31 14:20:42,300 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:42,300 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-10-31 14:20:42,550 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 8 +2023-10-31 14:20:42,550 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:42,551 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 8 +2023-10-31 14:20:42,551 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:42,551 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-10-31 14:20:42,551 INFO SenderThread:32614 [job_builder.py:build():281] Attempting to build job artifact +2023-10-31 14:20:42,552 INFO SenderThread:32614 [job_builder.py:_get_source_type():404] no source found +2023-10-31 14:20:42,552 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 9 +2023-10-31 14:20:42,552 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:42,552 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 9 +2023-10-31 14:20:42,553 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:42,553 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-10-31 14:20:42,553 INFO SenderThread:32614 [dir_watcher.py:finish():358] shutting down directory watcher +2023-10-31 14:20:42,566 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:20:42,571 INFO Thread-12 :32614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/output.log +2023-10-31 14:20:42,572 INFO SenderThread:32614 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/wandb-summary.json +2023-10-31 14:20:42,572 INFO SenderThread:32614 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files +2023-10-31 14:20:42,573 INFO SenderThread:32614 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/output.log output.log +2023-10-31 14:20:42,573 INFO SenderThread:32614 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/requirements.txt requirements.txt +2023-10-31 14:20:42,577 INFO SenderThread:32614 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/conda-environment.yaml conda-environment.yaml +2023-10-31 14:20:42,578 INFO SenderThread:32614 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/wandb-summary.json wandb-summary.json +2023-10-31 14:20:42,585 INFO SenderThread:32614 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/config.yaml config.yaml +2023-10-31 14:20:42,599 INFO SenderThread:32614 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/wandb-metadata.json wandb-metadata.json +2023-10-31 14:20:42,600 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 10 +2023-10-31 14:20:42,600 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:20:42,601 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:42,606 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 10 +2023-10-31 14:20:42,608 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:42,608 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-10-31 14:20:42,608 INFO SenderThread:32614 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 14:20:43,298 INFO wandb-upload_0:32614 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/output.log +2023-10-31 14:20:43,448 INFO wandb-upload_1:32614 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/requirements.txt +2023-10-31 14:20:43,567 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:20:43,568 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:20:43,680 INFO wandb-upload_2:32614 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/conda-environment.yaml +2023-10-31 14:20:43,683 INFO wandb-upload_4:32614 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/config.yaml +2023-10-31 14:20:43,862 INFO wandb-upload_3:32614 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/files/wandb-summary.json +2023-10-31 14:20:44,063 INFO Thread-11 :32614 [sender.py:transition_state():613] send defer: 11 +2023-10-31 14:20:44,063 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:44,063 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 11 +2023-10-31 14:20:44,064 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:44,064 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-10-31 14:20:44,064 INFO SenderThread:32614 [file_pusher.py:join():181] waiting for file pusher +2023-10-31 14:20:44,064 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 12 +2023-10-31 14:20:44,065 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:44,065 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 12 +2023-10-31 14:20:44,065 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:44,065 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-10-31 14:20:44,065 INFO SenderThread:32614 [file_stream.py:finish():595] file stream finish called +2023-10-31 14:20:44,569 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:20:45,153 INFO SenderThread:32614 [file_stream.py:finish():599] file stream finish is done +2023-10-31 14:20:45,154 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 13 +2023-10-31 14:20:45,154 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:20:45,154 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:45,155 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 13 +2023-10-31 14:20:45,155 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:45,155 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-10-31 14:20:45,156 INFO SenderThread:32614 [sender.py:transition_state():613] send defer: 14 +2023-10-31 14:20:45,156 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:20:45,156 DEBUG SenderThread:32614 [sender.py:send():380] send: final +2023-10-31 14:20:45,157 INFO HandlerThread:32614 [handler.py:handle_request_defer():172] handle defer: 14 +2023-10-31 14:20:45,157 DEBUG SenderThread:32614 [sender.py:send():380] send: footer +2023-10-31 14:20:45,157 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: defer +2023-10-31 14:20:45,157 INFO SenderThread:32614 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-10-31 14:20:45,158 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:20:45,159 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:20:45,159 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:20:45,161 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: server_info +2023-10-31 14:20:45,161 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:20:45,162 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: get_summary +2023-10-31 14:20:45,162 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: server_info +2023-10-31 14:20:45,162 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: sampled_history +2023-10-31 14:20:45,167 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:20:45,169 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: job_info +2023-10-31 14:20:45,494 DEBUG SenderThread:32614 [sender.py:send_request():407] send_request: job_info +2023-10-31 14:20:45,495 INFO MainThread:32614 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-10-31 14:20:45,495 INFO MainThread:32614 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-10-31 14:20:45,495 INFO MainThread:32614 [wandb_run.py:_footer_sync_info():3558] logging synced files +2023-10-31 14:20:45,496 DEBUG HandlerThread:32614 [handler.py:handle_request():146] handle_request: shutdown +2023-10-31 14:20:45,496 INFO HandlerThread:32614 [handler.py:finish():866] shutting down handler +2023-10-31 14:20:46,169 INFO WriterThread:32614 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/run-v2q0e45b.wandb +2023-10-31 14:20:46,494 INFO SenderThread:32614 [sender.py:finish():1534] shutting down sender +2023-10-31 14:20:46,495 INFO SenderThread:32614 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 14:20:46,495 INFO SenderThread:32614 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231031_141809-v2q0e45b/logs/debug.log b/wandb/run-20231031_141809-v2q0e45b/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e08df96394702adadf4e83c03dd8b30a460f60b6 --- /dev/null +++ b/wandb/run-20231031_141809-v2q0e45b/logs/debug.log @@ -0,0 +1,28 @@ +2023-10-31 14:18:09,555 INFO MainThread:32463 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 14:18:09,555 INFO MainThread:32463 [wandb_setup.py:_flush():76] Configure stats pid to 32463 +2023-10-31 14:18:09,555 INFO MainThread:32463 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 14:18:09,555 INFO MainThread:32463 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 14:18:09,555 INFO MainThread:32463 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 14:18:09,556 INFO MainThread:32463 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 14:18:09,556 INFO MainThread:32463 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 14:18:09,556 INFO MainThread:32463 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 14:18:09,556 INFO MainThread:32463 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/logs/debug.log +2023-10-31 14:18:09,556 INFO MainThread:32463 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_141809-v2q0e45b/logs/debug-internal.log +2023-10-31 14:18:09,556 INFO MainThread:32463 [wandb_init.py:init():568] calling init triggers +2023-10-31 14:18:09,556 INFO MainThread:32463 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'Adam', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 49} +2023-10-31 14:18:09,556 INFO MainThread:32463 [wandb_init.py:init():618] starting backend +2023-10-31 14:18:09,556 INFO MainThread:32463 [wandb_init.py:init():622] setting up manager +2023-10-31 14:18:09,558 INFO MainThread:32463 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 14:18:09,560 INFO MainThread:32463 [wandb_init.py:init():628] backend started and connected +2023-10-31 14:18:09,564 INFO MainThread:32463 [wandb_init.py:init():720] updated telemetry +2023-10-31 14:18:09,565 INFO MainThread:32463 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 14:18:10,526 INFO MainThread:32463 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 14:18:10,962 INFO MainThread:32463 [wandb_run.py:_on_init():2229] got version response +2023-10-31 14:18:10,962 INFO MainThread:32463 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 14:18:17,521 INFO MainThread:32463 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 14:18:17,521 INFO MainThread:32463 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 14:18:17,521 INFO MainThread:32463 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 14:18:17,522 INFO MainThread:32463 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 14:18:17,522 INFO MainThread:32463 [wandb_init.py:init():845] run started, returning control to user process +2023-10-31 14:20:47,097 WARNING MsgRouterThr:32463 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20231031_141809-v2q0e45b/run-v2q0e45b.wandb b/wandb/run-20231031_141809-v2q0e45b/run-v2q0e45b.wandb new file mode 100644 index 0000000000000000000000000000000000000000..54e32757ab042303864fa8e5b0589af6d8082f3d Binary files /dev/null and b/wandb/run-20231031_141809-v2q0e45b/run-v2q0e45b.wandb differ diff --git a/wandb/run-20231031_143426-uvt3fyc0/files/conda-environment.yaml b/wandb/run-20231031_143426-uvt3fyc0/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7f7cff6ba3dbbba8f29abfd550e642beb5aa829 --- /dev/null +++ b/wandb/run-20231031_143426-uvt3fyc0/files/conda-environment.yaml @@ -0,0 +1,162 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - greenlet==3.0.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pyparsing==3.1.1 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - regex==2023.8.8 + - requests==2.28.1 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - smmap==5.0.1 + - sqlalchemy==2.0.22 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typing-extensions==4.4.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - wandb==0.15.12 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231031_143426-uvt3fyc0/files/config.yaml b/wandb/run-20231031_143426-uvt3fyc0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b08b5cfdb49f6aa7df21d66e5ff78e143794fc1 --- /dev/null +++ b/wandb/run-20231031_143426-uvt3fyc0/files/config.yaml @@ -0,0 +1,50 @@ +wandb_version: 1 + +optimizer: + desc: null + value: Adam +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 32 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698748466.112382 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + 3: + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231031_143426-uvt3fyc0/files/output.log b/wandb/run-20231031_143426-uvt3fyc0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..eab4c6cdd3dbd700b417da90c4b1615d792b6e74 --- /dev/null +++ b/wandb/run-20231031_143426-uvt3fyc0/files/output.log @@ -0,0 +1,32 @@ +[W 2023-10-31 14:36:53,769] Trial 0 failed with parameters: {'lr': 9.827034231503784e-07, 'n_estimators': 32, 'optimizer': 'Adam'} because of the following error: AttributeError("'EfficientNet' object has no attribute 'inference_mode'"). +Traceback (most recent call last): + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 200, in _run_trial + value_or_values = func(trial) + File "hyperpara.py", line 68, in objective + val_loss, val_acc = engine.validation(model = model, dataloader = data.valid_dataloader, loss_fn = loss_fn, + File "/jash/Workspace/Food/engine.py", line 139, in validation + with model.inference_mode(): + File "/opt/conda/envs/env/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1614, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'EfficientNet' object has no attribute 'inference_mode' +[W 2023-10-31 14:36:53,770] Trial 0 failed with value None. +Traceback (most recent call last): + File "hyperpara.py", line 110, in + study.optimize(objective, n_trials=11) + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/study.py", line 451, in optimize + _optimize( + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 66, in _optimize + _optimize_sequential( + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 163, in _optimize_sequential + frozen_trial = _run_trial(study, func, catch) + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 251, in _run_trial + raise func_err + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 200, in _run_trial + value_or_values = func(trial) + File "hyperpara.py", line 68, in objective + val_loss, val_acc = engine.validation(model = model, dataloader = data.valid_dataloader, loss_fn = loss_fn, + File "/jash/Workspace/Food/engine.py", line 139, in validation + with model.inference_mode(): + File "/opt/conda/envs/env/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1614, in __getattr__ + raise AttributeError("'{}' object has no attribute '{}'".format( +AttributeError: 'EfficientNet' object has no attribute 'inference_mode' \ No newline at end of file diff --git a/wandb/run-20231031_143426-uvt3fyc0/files/requirements.txt b/wandb/run-20231031_143426-uvt3fyc0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f84e89ededbacd967e61bd8559b50c3efecc0021 --- /dev/null +++ b/wandb/run-20231031_143426-uvt3fyc0/files/requirements.txt @@ -0,0 +1,133 @@ +accelerate==0.22.0 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +executing==1.2.0 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +greenlet==3.0.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +regex==2023.8.8 +requests==2.28.1 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +six==1.16.0 +smmap==5.0.1 +sqlalchemy==2.0.22 +stack-data==0.6.2 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typing-extensions==4.4.0 +tzdata==2023.3 +urllib3==1.26.13 +wandb==0.15.12 +wcwidth==0.2.8 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_143426-uvt3fyc0/files/wandb-metadata.json b/wandb/run-20231031_143426-uvt3fyc0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8132cc23ad56da71472c6bef4c1430246af1b9e6 --- /dev/null +++ b/wandb/run-20231031_143426-uvt3fyc0/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T10:34:27.519739", + "startedAt": "2023-10-31T10:34:26.106611", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.5869499999999999, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.731, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.492, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.582, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.128, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.201, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.204, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.509, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.503, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.209, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.205, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.499, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.998, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.499, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.696, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.593, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.7576217651367 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_143426-uvt3fyc0/files/wandb-summary.json b/wandb/run-20231031_143426-uvt3fyc0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a8a308862849c5e833b652288b90aa954791dd4d --- /dev/null +++ b/wandb/run-20231031_143426-uvt3fyc0/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 146}} \ No newline at end of file diff --git a/wandb/run-20231031_143426-uvt3fyc0/logs/debug-internal.log b/wandb/run-20231031_143426-uvt3fyc0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..00639bab91cb7b4d88575c86458ae2a8da9c90b2 --- /dev/null +++ b/wandb/run-20231031_143426-uvt3fyc0/logs/debug-internal.log @@ -0,0 +1,247 @@ +2023-10-31 14:34:26,114 INFO StreamThr :32948 [internal.py:wandb_internal():86] W&B internal server running at pid: 32948, started at: 2023-10-31 14:34:26.113864 +2023-10-31 14:34:26,117 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status +2023-10-31 14:34:26,120 INFO WriterThread:32948 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/run-uvt3fyc0.wandb +2023-10-31 14:34:26,124 DEBUG SenderThread:32948 [sender.py:send():380] send: header +2023-10-31 14:34:26,124 DEBUG SenderThread:32948 [sender.py:send():380] send: run +2023-10-31 14:34:26,969 INFO SenderThread:32948 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files +2023-10-31 14:34:26,969 INFO SenderThread:32948 [sender.py:_start_run_threads():1122] run started: uvt3fyc0 with start time 1698748466.112382 +2023-10-31 14:34:26,987 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 14:34:26,988 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: check_version +2023-10-31 14:34:27,442 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 14:34:27,458 DEBUG HandlerThread:32948 [system_info.py:__init__():32] System info init +2023-10-31 14:34:27,458 DEBUG HandlerThread:32948 [system_info.py:__init__():47] System info init done +2023-10-31 14:34:27,458 INFO HandlerThread:32948 [system_monitor.py:start():194] Starting system monitor +2023-10-31 14:34:27,458 INFO SystemMonitor:32948 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 14:34:27,458 INFO HandlerThread:32948 [system_monitor.py:probe():214] Collecting system info +2023-10-31 14:34:27,459 INFO SystemMonitor:32948 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 14:34:27,460 INFO SystemMonitor:32948 [interfaces.py:start():190] Started disk monitoring +2023-10-31 14:34:27,461 INFO SystemMonitor:32948 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 14:34:27,462 INFO SystemMonitor:32948 [interfaces.py:start():190] Started memory monitoring +2023-10-31 14:34:27,463 INFO SystemMonitor:32948 [interfaces.py:start():190] Started network monitoring +2023-10-31 14:34:27,519 DEBUG HandlerThread:32948 [system_info.py:probe():196] Probing system +2023-10-31 14:34:27,526 DEBUG HandlerThread:32948 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 14:34:27,527 DEBUG HandlerThread:32948 [system_info.py:probe():244] Probing system done +2023-10-31 14:34:27,527 DEBUG HandlerThread:32948 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T10:34:27.519739', 'startedAt': '2023-10-31T10:34:26.106611', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.5869499999999999, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.731, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.492, 'min': 1200.0, 'max': 3600.0}, {'current': 1.582, 'min': 1200.0, 'max': 3600.0}, {'current': 2.128, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.201, 'min': 1200.0, 'max': 3600.0}, {'current': 1.204, 'min': 1200.0, 'max': 3600.0}, {'current': 1.509, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.599, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.503, 'min': 1200.0, 'max': 3600.0}, {'current': 2.209, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 1.499, 'min': 1200.0, 'max': 3600.0}, {'current': 1.599, 'min': 1200.0, 'max': 3600.0}, {'current': 1.998, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.499, 'min': 1200.0, 'max': 3600.0}, {'current': 2.696, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.593, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.7576217651367}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 14:34:27,527 INFO HandlerThread:32948 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 14:34:27,527 INFO HandlerThread:32948 [system_monitor.py:probe():227] Publishing system info +2023-10-31 14:34:27,527 DEBUG HandlerThread:32948 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 14:34:27,528 DEBUG HandlerThread:32948 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 14:34:27,528 DEBUG HandlerThread:32948 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 14:34:27,974 INFO Thread-12 :32948 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/conda-environment.yaml +2023-10-31 14:34:27,975 INFO Thread-12 :32948 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/requirements.txt +2023-10-31 14:34:33,758 DEBUG HandlerThread:32948 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 14:34:33,760 INFO HandlerThread:32948 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 14:34:33,765 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:34:33,765 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 14:34:33,766 DEBUG SenderThread:32948 [sender.py:send():380] send: files +2023-10-31 14:34:33,767 INFO SenderThread:32948 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 14:34:33,779 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:34:33,779 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:34:33,783 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:34:33,972 INFO Thread-12 :32948 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/conda-environment.yaml +2023-10-31 14:34:33,972 INFO Thread-12 :32948 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/wandb-metadata.json +2023-10-31 14:34:34,321 DEBUG SenderThread:32948 [sender.py:send():380] send: telemetry +2023-10-31 14:34:34,951 INFO wandb-upload_0:32948 [upload_job.py:push():131] Uploaded file /tmp/tmpx_8l59v2wandb/hzdocjh6-wandb-metadata.json +2023-10-31 14:34:37,322 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:34:42,322 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:34:47,323 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:34:48,778 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:34:48,779 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:34:48,821 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:34:53,128 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:34:58,134 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:34:58,980 INFO Thread-12 :32948 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/config.yaml +2023-10-31 14:35:03,488 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:35:03,779 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:35:03,779 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:35:03,821 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:35:09,149 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:35:14,150 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:35:18,779 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:35:18,779 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:35:18,821 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:35:20,134 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:35:25,134 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:35:27,463 DEBUG SystemMonitor:32948 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 14:35:27,467 DEBUG SenderThread:32948 [sender.py:send():380] send: stats +2023-10-31 14:35:30,469 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:35:33,779 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:35:33,779 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:35:33,821 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:35:36,115 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:35:41,116 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:35:46,116 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:35:48,779 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:35:48,779 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:35:48,821 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:35:51,147 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:35:56,148 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:35:57,470 DEBUG SenderThread:32948 [sender.py:send():380] send: stats +2023-10-31 14:36:01,471 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:36:03,781 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:36:03,781 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:36:03,781 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:36:07,155 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:36:12,155 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:36:17,156 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:36:18,779 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:36:18,779 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:36:18,821 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:36:22,192 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:36:27,193 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:36:27,473 DEBUG SenderThread:32948 [sender.py:send():380] send: stats +2023-10-31 14:36:32,475 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:36:33,779 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:36:33,780 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:36:33,821 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:36:38,208 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:36:43,209 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:36:48,210 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:36:48,779 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:36:48,780 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:36:48,821 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:36:53,215 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:36:53,818 DEBUG SenderThread:32948 [sender.py:send():380] send: exit +2023-10-31 14:36:53,818 INFO SenderThread:32948 [sender.py:send_exit():585] handling exit code: 1 +2023-10-31 14:36:53,818 INFO SenderThread:32948 [sender.py:send_exit():587] handling runtime: 146 +2023-10-31 14:36:53,822 INFO SenderThread:32948 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 14:36:53,823 INFO SenderThread:32948 [sender.py:send_exit():593] send defer +2023-10-31 14:36:53,823 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:53,823 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 0 +2023-10-31 14:36:53,824 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:53,824 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-10-31 14:36:53,824 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 1 +2023-10-31 14:36:53,824 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:53,824 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 1 +2023-10-31 14:36:53,825 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:53,825 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-10-31 14:36:53,825 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 2 +2023-10-31 14:36:53,825 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:53,825 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 2 +2023-10-31 14:36:53,825 INFO HandlerThread:32948 [system_monitor.py:finish():203] Stopping system monitor +2023-10-31 14:36:53,825 DEBUG SystemMonitor:32948 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-10-31 14:36:53,826 INFO HandlerThread:32948 [interfaces.py:finish():202] Joined cpu monitor +2023-10-31 14:36:53,826 DEBUG SystemMonitor:32948 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-10-31 14:36:53,827 INFO HandlerThread:32948 [interfaces.py:finish():202] Joined disk monitor +2023-10-31 14:36:54,019 INFO Thread-12 :32948 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/output.log +2023-10-31 14:36:54,019 INFO Thread-12 :32948 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/wandb-summary.json +2023-10-31 14:36:55,526 INFO HandlerThread:32948 [interfaces.py:finish():202] Joined gpu monitor +2023-10-31 14:36:55,527 INFO HandlerThread:32948 [interfaces.py:finish():202] Joined memory monitor +2023-10-31 14:36:55,527 INFO HandlerThread:32948 [interfaces.py:finish():202] Joined network monitor +2023-10-31 14:36:55,528 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:36:55,528 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:55,529 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-10-31 14:36:55,529 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 3 +2023-10-31 14:36:55,529 DEBUG SenderThread:32948 [sender.py:send():380] send: stats +2023-10-31 14:36:55,529 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:55,531 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:36:55,531 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 3 +2023-10-31 14:36:55,532 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:55,532 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-10-31 14:36:55,532 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 4 +2023-10-31 14:36:55,532 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:55,532 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 4 +2023-10-31 14:36:55,533 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:55,533 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-10-31 14:36:55,533 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 5 +2023-10-31 14:36:55,533 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:55,533 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 5 +2023-10-31 14:36:55,534 DEBUG SenderThread:32948 [sender.py:send():380] send: summary +2023-10-31 14:36:55,534 INFO SenderThread:32948 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 14:36:55,535 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:55,535 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-10-31 14:36:55,535 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 6 +2023-10-31 14:36:55,535 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:55,535 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 6 +2023-10-31 14:36:55,536 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:55,536 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-10-31 14:36:55,536 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 7 +2023-10-31 14:36:55,536 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:36:55,536 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:55,537 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 7 +2023-10-31 14:36:55,537 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:55,537 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-10-31 14:36:55,808 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 8 +2023-10-31 14:36:55,808 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:55,808 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 8 +2023-10-31 14:36:55,808 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:55,808 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-10-31 14:36:55,808 INFO SenderThread:32948 [job_builder.py:build():281] Attempting to build job artifact +2023-10-31 14:36:55,809 INFO SenderThread:32948 [job_builder.py:_get_source_type():404] no source found +2023-10-31 14:36:55,809 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 9 +2023-10-31 14:36:55,809 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:55,809 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 9 +2023-10-31 14:36:55,810 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:55,810 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-10-31 14:36:55,810 INFO SenderThread:32948 [dir_watcher.py:finish():358] shutting down directory watcher +2023-10-31 14:36:55,819 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:36:56,020 INFO Thread-12 :32948 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/output.log +2023-10-31 14:36:56,021 INFO SenderThread:32948 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/wandb-summary.json +2023-10-31 14:36:56,021 INFO SenderThread:32948 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files +2023-10-31 14:36:56,021 INFO SenderThread:32948 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/output.log output.log +2023-10-31 14:36:56,022 INFO SenderThread:32948 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/requirements.txt requirements.txt +2023-10-31 14:36:56,022 INFO SenderThread:32948 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/conda-environment.yaml conda-environment.yaml +2023-10-31 14:36:56,023 INFO SenderThread:32948 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/wandb-summary.json wandb-summary.json +2023-10-31 14:36:56,043 INFO SenderThread:32948 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/config.yaml config.yaml +2023-10-31 14:36:56,045 INFO SenderThread:32948 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/wandb-metadata.json wandb-metadata.json +2023-10-31 14:36:56,045 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 10 +2023-10-31 14:36:56,049 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:36:56,055 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:56,055 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 10 +2023-10-31 14:36:56,057 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:56,057 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-10-31 14:36:56,058 INFO SenderThread:32948 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 14:36:56,747 INFO wandb-upload_0:32948 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/output.log +2023-10-31 14:36:56,821 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:36:56,821 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:36:56,924 INFO wandb-upload_1:32948 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/requirements.txt +2023-10-31 14:36:57,104 INFO wandb-upload_4:32948 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/config.yaml +2023-10-31 14:36:57,126 INFO wandb-upload_3:32948 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/wandb-summary.json +2023-10-31 14:36:57,288 INFO wandb-upload_2:32948 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/files/conda-environment.yaml +2023-10-31 14:36:57,488 INFO Thread-11 :32948 [sender.py:transition_state():613] send defer: 11 +2023-10-31 14:36:57,489 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:57,489 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 11 +2023-10-31 14:36:57,490 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:57,490 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-10-31 14:36:57,490 INFO SenderThread:32948 [file_pusher.py:join():181] waiting for file pusher +2023-10-31 14:36:57,490 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 12 +2023-10-31 14:36:57,491 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:57,491 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 12 +2023-10-31 14:36:57,491 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:57,491 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-10-31 14:36:57,491 INFO SenderThread:32948 [file_stream.py:finish():595] file stream finish called +2023-10-31 14:36:57,821 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:36:58,556 INFO SenderThread:32948 [file_stream.py:finish():599] file stream finish is done +2023-10-31 14:36:58,556 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 13 +2023-10-31 14:36:58,557 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:36:58,557 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:58,557 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 13 +2023-10-31 14:36:58,558 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:58,558 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-10-31 14:36:58,558 INFO SenderThread:32948 [sender.py:transition_state():613] send defer: 14 +2023-10-31 14:36:58,559 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:36:58,559 DEBUG SenderThread:32948 [sender.py:send():380] send: final +2023-10-31 14:36:58,559 INFO HandlerThread:32948 [handler.py:handle_request_defer():172] handle defer: 14 +2023-10-31 14:36:58,559 DEBUG SenderThread:32948 [sender.py:send():380] send: footer +2023-10-31 14:36:58,560 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: defer +2023-10-31 14:36:58,560 INFO SenderThread:32948 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-10-31 14:36:58,561 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:36:58,563 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:36:58,563 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:36:58,563 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: server_info +2023-10-31 14:36:58,564 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:36:58,564 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: get_summary +2023-10-31 14:36:58,564 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: server_info +2023-10-31 14:36:58,565 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: sampled_history +2023-10-31 14:36:58,570 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:36:58,571 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: job_info +2023-10-31 14:36:58,893 DEBUG SenderThread:32948 [sender.py:send_request():407] send_request: job_info +2023-10-31 14:36:58,894 INFO MainThread:32948 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-10-31 14:36:58,894 INFO MainThread:32948 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-10-31 14:36:58,894 INFO MainThread:32948 [wandb_run.py:_footer_sync_info():3558] logging synced files +2023-10-31 14:36:58,895 DEBUG HandlerThread:32948 [handler.py:handle_request():146] handle_request: shutdown +2023-10-31 14:36:58,895 INFO HandlerThread:32948 [handler.py:finish():866] shutting down handler +2023-10-31 14:36:59,572 INFO WriterThread:32948 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/run-uvt3fyc0.wandb +2023-10-31 14:36:59,894 INFO SenderThread:32948 [sender.py:finish():1534] shutting down sender +2023-10-31 14:36:59,894 INFO SenderThread:32948 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 14:36:59,894 INFO SenderThread:32948 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231031_143426-uvt3fyc0/logs/debug.log b/wandb/run-20231031_143426-uvt3fyc0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d3139a81c98b1b261ba9ec8254f8e2a2e800fe8a --- /dev/null +++ b/wandb/run-20231031_143426-uvt3fyc0/logs/debug.log @@ -0,0 +1,28 @@ +2023-10-31 14:34:26,108 INFO MainThread:32760 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 14:34:26,108 INFO MainThread:32760 [wandb_setup.py:_flush():76] Configure stats pid to 32760 +2023-10-31 14:34:26,108 INFO MainThread:32760 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 14:34:26,108 INFO MainThread:32760 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 14:34:26,108 INFO MainThread:32760 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 14:34:26,108 INFO MainThread:32760 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 14:34:26,108 INFO MainThread:32760 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 14:34:26,108 INFO MainThread:32760 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 14:34:26,109 INFO MainThread:32760 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/logs/debug.log +2023-10-31 14:34:26,109 INFO MainThread:32760 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_143426-uvt3fyc0/logs/debug-internal.log +2023-10-31 14:34:26,109 INFO MainThread:32760 [wandb_init.py:init():568] calling init triggers +2023-10-31 14:34:26,109 INFO MainThread:32760 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'Adam', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 32} +2023-10-31 14:34:26,109 INFO MainThread:32760 [wandb_init.py:init():618] starting backend +2023-10-31 14:34:26,109 INFO MainThread:32760 [wandb_init.py:init():622] setting up manager +2023-10-31 14:34:26,110 INFO MainThread:32760 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 14:34:26,112 INFO MainThread:32760 [wandb_init.py:init():628] backend started and connected +2023-10-31 14:34:26,116 INFO MainThread:32760 [wandb_init.py:init():720] updated telemetry +2023-10-31 14:34:26,116 INFO MainThread:32760 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 14:34:26,987 INFO MainThread:32760 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 14:34:27,429 INFO MainThread:32760 [wandb_run.py:_on_init():2229] got version response +2023-10-31 14:34:27,429 INFO MainThread:32760 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 14:34:33,780 INFO MainThread:32760 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 14:34:33,780 INFO MainThread:32760 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 14:34:33,780 INFO MainThread:32760 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 14:34:33,780 INFO MainThread:32760 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 14:34:33,781 INFO MainThread:32760 [wandb_init.py:init():845] run started, returning control to user process +2023-10-31 14:37:00,198 WARNING MsgRouterThr:32760 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20231031_143426-uvt3fyc0/run-uvt3fyc0.wandb b/wandb/run-20231031_143426-uvt3fyc0/run-uvt3fyc0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c9f8ddea70c2578df8ed35a6dd1500e4e1380571 Binary files /dev/null and b/wandb/run-20231031_143426-uvt3fyc0/run-uvt3fyc0.wandb differ diff --git a/wandb/run-20231031_144427-e8zkn3bl/files/conda-environment.yaml b/wandb/run-20231031_144427-e8zkn3bl/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7f7cff6ba3dbbba8f29abfd550e642beb5aa829 --- /dev/null +++ b/wandb/run-20231031_144427-e8zkn3bl/files/conda-environment.yaml @@ -0,0 +1,162 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - greenlet==3.0.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pyparsing==3.1.1 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - regex==2023.8.8 + - requests==2.28.1 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - smmap==5.0.1 + - sqlalchemy==2.0.22 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typing-extensions==4.4.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - wandb==0.15.12 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231031_144427-e8zkn3bl/files/config.yaml b/wandb/run-20231031_144427-e8zkn3bl/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d6f6b2ea9e3454474d540d751c85c6a6c3a414b --- /dev/null +++ b/wandb/run-20231031_144427-e8zkn3bl/files/config.yaml @@ -0,0 +1,50 @@ +wandb_version: 1 + +optimizer: + desc: null + value: RMSprop +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 49 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698749067.025155 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + 3: + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231031_144427-e8zkn3bl/files/output.log b/wandb/run-20231031_144427-e8zkn3bl/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5278777dbda3f4a3f748d38ba4e6120869cc4ac5 --- /dev/null +++ b/wandb/run-20231031_144427-e8zkn3bl/files/output.log @@ -0,0 +1,28 @@ +[W 2023-10-31 14:46:56,707] Trial 0 failed with parameters: {'lr': 2.9497025173220695e-07, 'n_estimators': 49, 'optimizer': 'RMSprop'} because of the following error: NameError("name 'i' is not defined"). +Traceback (most recent call last): + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 200, in _run_trial + value_or_values = func(trial) + File "hyperpara.py", line 68, in objective + val_loss, val_acc = engine.validation(model = model, dataloader = data.valid_dataloader, loss_fn = loss_fn, + File "/jash/Workspace/Food/engine.py", line 150, in validation + if i==batch_idx and log_images: +NameError: name 'i' is not defined +[W 2023-10-31 14:46:56,707] Trial 0 failed with value None. +Traceback (most recent call last): + File "hyperpara.py", line 110, in + study.optimize(objective, n_trials=11) + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/study.py", line 451, in optimize + _optimize( + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 66, in _optimize + _optimize_sequential( + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 163, in _optimize_sequential + frozen_trial = _run_trial(study, func, catch) + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 251, in _run_trial + raise func_err + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 200, in _run_trial + value_or_values = func(trial) + File "hyperpara.py", line 68, in objective + val_loss, val_acc = engine.validation(model = model, dataloader = data.valid_dataloader, loss_fn = loss_fn, + File "/jash/Workspace/Food/engine.py", line 150, in validation + if i==batch_idx and log_images: +NameError: name 'i' is not defined \ No newline at end of file diff --git a/wandb/run-20231031_144427-e8zkn3bl/files/requirements.txt b/wandb/run-20231031_144427-e8zkn3bl/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f84e89ededbacd967e61bd8559b50c3efecc0021 --- /dev/null +++ b/wandb/run-20231031_144427-e8zkn3bl/files/requirements.txt @@ -0,0 +1,133 @@ +accelerate==0.22.0 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +executing==1.2.0 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +greenlet==3.0.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +regex==2023.8.8 +requests==2.28.1 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +six==1.16.0 +smmap==5.0.1 +sqlalchemy==2.0.22 +stack-data==0.6.2 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typing-extensions==4.4.0 +tzdata==2023.3 +urllib3==1.26.13 +wandb==0.15.12 +wcwidth==0.2.8 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_144427-e8zkn3bl/files/wandb-metadata.json b/wandb/run-20231031_144427-e8zkn3bl/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..099654ce9f816e59b5d345a3253d4675153df60a --- /dev/null +++ b/wandb/run-20231031_144427-e8zkn3bl/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T10:44:28.452875", + "startedAt": "2023-10-31T10:44:27.019163", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.524775, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.19, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.704, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.692, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.196, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.929, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.5, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.398, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.291, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.205, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.699, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.751, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.949, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.498, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.399, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.203, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.205, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.706, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.7576293945312 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_144427-e8zkn3bl/files/wandb-summary.json b/wandb/run-20231031_144427-e8zkn3bl/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..77966178d83756b456afa0bbc0e27623231978f6 --- /dev/null +++ b/wandb/run-20231031_144427-e8zkn3bl/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 148}} \ No newline at end of file diff --git a/wandb/run-20231031_144427-e8zkn3bl/logs/debug-internal.log b/wandb/run-20231031_144427-e8zkn3bl/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b26b6061a5e02e506b01295a03680e99fb2c1215 --- /dev/null +++ b/wandb/run-20231031_144427-e8zkn3bl/logs/debug-internal.log @@ -0,0 +1,247 @@ +2023-10-31 14:44:27,027 INFO StreamThr :33232 [internal.py:wandb_internal():86] W&B internal server running at pid: 33232, started at: 2023-10-31 14:44:27.026574 +2023-10-31 14:44:27,030 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status +2023-10-31 14:44:27,033 INFO WriterThread:33232 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/run-e8zkn3bl.wandb +2023-10-31 14:44:27,036 DEBUG SenderThread:33232 [sender.py:send():380] send: header +2023-10-31 14:44:27,037 DEBUG SenderThread:33232 [sender.py:send():380] send: run +2023-10-31 14:44:27,881 INFO SenderThread:33232 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files +2023-10-31 14:44:27,881 INFO SenderThread:33232 [sender.py:_start_run_threads():1122] run started: e8zkn3bl with start time 1698749067.025155 +2023-10-31 14:44:27,898 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 14:44:27,899 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: check_version +2023-10-31 14:44:28,373 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 14:44:28,389 DEBUG HandlerThread:33232 [system_info.py:__init__():32] System info init +2023-10-31 14:44:28,389 DEBUG HandlerThread:33232 [system_info.py:__init__():47] System info init done +2023-10-31 14:44:28,389 INFO HandlerThread:33232 [system_monitor.py:start():194] Starting system monitor +2023-10-31 14:44:28,389 INFO SystemMonitor:33232 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 14:44:28,389 INFO HandlerThread:33232 [system_monitor.py:probe():214] Collecting system info +2023-10-31 14:44:28,390 INFO SystemMonitor:33232 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 14:44:28,391 INFO SystemMonitor:33232 [interfaces.py:start():190] Started disk monitoring +2023-10-31 14:44:28,393 INFO SystemMonitor:33232 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 14:44:28,394 INFO SystemMonitor:33232 [interfaces.py:start():190] Started memory monitoring +2023-10-31 14:44:28,395 INFO SystemMonitor:33232 [interfaces.py:start():190] Started network monitoring +2023-10-31 14:44:28,452 DEBUG HandlerThread:33232 [system_info.py:probe():196] Probing system +2023-10-31 14:44:28,460 DEBUG HandlerThread:33232 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 14:44:28,460 DEBUG HandlerThread:33232 [system_info.py:probe():244] Probing system done +2023-10-31 14:44:28,460 DEBUG HandlerThread:33232 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T10:44:28.452875', 'startedAt': '2023-10-31T10:44:27.019163', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.524775, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.19, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.704, 'min': 1200.0, 'max': 3600.0}, {'current': 2.692, 'min': 1200.0, 'max': 3600.0}, {'current': 1.196, 'min': 1200.0, 'max': 3600.0}, {'current': 1.929, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.5, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.398, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.291, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.699, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.751, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.949, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.498, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.399, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.203, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.706, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.7576293945312}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 14:44:28,460 INFO HandlerThread:33232 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 14:44:28,460 INFO HandlerThread:33232 [system_monitor.py:probe():227] Publishing system info +2023-10-31 14:44:28,460 DEBUG HandlerThread:33232 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 14:44:28,461 DEBUG HandlerThread:33232 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 14:44:28,461 DEBUG HandlerThread:33232 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 14:44:28,885 INFO Thread-12 :33232 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/requirements.txt +2023-10-31 14:44:28,886 INFO Thread-12 :33232 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/conda-environment.yaml +2023-10-31 14:44:34,592 DEBUG HandlerThread:33232 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 14:44:34,594 INFO HandlerThread:33232 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 14:44:34,599 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:44:34,599 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 14:44:34,600 DEBUG SenderThread:33232 [sender.py:send():380] send: files +2023-10-31 14:44:34,600 INFO SenderThread:33232 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 14:44:34,613 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:44:34,613 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:44:34,619 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:44:34,884 INFO Thread-12 :33232 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/conda-environment.yaml +2023-10-31 14:44:34,884 INFO Thread-12 :33232 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/wandb-metadata.json +2023-10-31 14:44:35,117 DEBUG SenderThread:33232 [sender.py:send():380] send: telemetry +2023-10-31 14:44:36,059 INFO wandb-upload_0:33232 [upload_job.py:push():131] Uploaded file /tmp/tmpat8p1bi3wandb/s0k5kns1-wandb-metadata.json +2023-10-31 14:44:38,118 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:44:43,119 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:44:48,120 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:44:49,612 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:44:49,613 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:44:49,653 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:44:53,968 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:44:58,977 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:44:59,892 INFO Thread-12 :33232 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/config.yaml +2023-10-31 14:45:04,335 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:45:04,612 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:45:04,613 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:45:04,653 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:45:09,953 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:45:14,954 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:45:19,613 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:45:19,613 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:45:19,653 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:45:20,024 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:45:25,025 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:45:28,396 DEBUG SystemMonitor:33232 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 14:45:28,399 DEBUG SenderThread:33232 [sender.py:send():380] send: stats +2023-10-31 14:45:30,401 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:45:34,612 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:45:34,613 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:45:34,657 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:45:36,019 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:45:41,019 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:45:46,020 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:45:49,613 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:45:49,613 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:45:49,653 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:45:51,038 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:45:56,038 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:45:58,402 DEBUG SenderThread:33232 [sender.py:send():380] send: stats +2023-10-31 14:46:01,404 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:46:04,613 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:46:04,613 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:46:04,653 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:46:06,968 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:46:11,969 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:46:16,970 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:46:19,613 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:46:19,613 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:46:19,657 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:46:21,985 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:46:26,985 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:46:28,404 DEBUG SenderThread:33232 [sender.py:send():380] send: stats +2023-10-31 14:46:32,406 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:46:34,613 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:46:34,613 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:46:34,657 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:46:38,022 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:46:43,023 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:46:48,024 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:46:49,613 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 14:46:49,614 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: stop_status +2023-10-31 14:46:49,657 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:46:53,951 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:46:56,754 DEBUG SenderThread:33232 [sender.py:send():380] send: exit +2023-10-31 14:46:56,754 INFO SenderThread:33232 [sender.py:send_exit():585] handling exit code: 1 +2023-10-31 14:46:56,754 INFO SenderThread:33232 [sender.py:send_exit():587] handling runtime: 148 +2023-10-31 14:46:56,757 INFO SenderThread:33232 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 14:46:56,758 INFO SenderThread:33232 [sender.py:send_exit():593] send defer +2023-10-31 14:46:56,759 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:46:56,759 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 0 +2023-10-31 14:46:56,759 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:46:56,759 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-10-31 14:46:56,759 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 1 +2023-10-31 14:46:56,760 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:46:56,760 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 1 +2023-10-31 14:46:56,760 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:46:56,760 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-10-31 14:46:56,760 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 2 +2023-10-31 14:46:56,760 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:46:56,760 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 2 +2023-10-31 14:46:56,760 INFO HandlerThread:33232 [system_monitor.py:finish():203] Stopping system monitor +2023-10-31 14:46:56,761 DEBUG SystemMonitor:33232 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-10-31 14:46:56,761 DEBUG SystemMonitor:33232 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-10-31 14:46:56,762 INFO HandlerThread:33232 [interfaces.py:finish():202] Joined cpu monitor +2023-10-31 14:46:56,764 INFO HandlerThread:33232 [interfaces.py:finish():202] Joined disk monitor +2023-10-31 14:46:56,932 INFO Thread-12 :33232 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/wandb-summary.json +2023-10-31 14:46:56,932 INFO Thread-12 :33232 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/output.log +2023-10-31 14:46:58,642 INFO HandlerThread:33232 [interfaces.py:finish():202] Joined gpu monitor +2023-10-31 14:46:58,642 INFO HandlerThread:33232 [interfaces.py:finish():202] Joined memory monitor +2023-10-31 14:46:58,643 INFO HandlerThread:33232 [interfaces.py:finish():202] Joined network monitor +2023-10-31 14:46:58,643 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:46:58,643 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:46:58,644 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-10-31 14:46:58,644 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 3 +2023-10-31 14:46:58,644 DEBUG SenderThread:33232 [sender.py:send():380] send: stats +2023-10-31 14:46:58,644 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:46:58,646 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:46:58,646 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 3 +2023-10-31 14:46:58,647 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:46:58,647 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-10-31 14:46:58,647 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 4 +2023-10-31 14:46:58,647 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:46:58,647 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 4 +2023-10-31 14:46:58,647 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:46:58,648 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-10-31 14:46:58,648 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 5 +2023-10-31 14:46:58,648 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:46:58,648 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 5 +2023-10-31 14:46:58,648 DEBUG SenderThread:33232 [sender.py:send():380] send: summary +2023-10-31 14:46:58,649 INFO SenderThread:33232 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 14:46:58,649 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:46:58,650 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-10-31 14:46:58,650 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 6 +2023-10-31 14:46:58,650 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:46:58,650 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 6 +2023-10-31 14:46:58,650 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:46:58,650 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-10-31 14:46:58,651 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 7 +2023-10-31 14:46:58,651 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 14:46:58,651 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:46:58,651 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 7 +2023-10-31 14:46:58,651 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:46:58,651 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-10-31 14:46:58,744 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 8 +2023-10-31 14:46:58,745 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:46:58,745 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 8 +2023-10-31 14:46:58,745 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:46:58,745 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-10-31 14:46:58,745 INFO SenderThread:33232 [job_builder.py:build():281] Attempting to build job artifact +2023-10-31 14:46:58,746 INFO SenderThread:33232 [job_builder.py:_get_source_type():404] no source found +2023-10-31 14:46:58,746 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 9 +2023-10-31 14:46:58,746 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:46:58,746 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 9 +2023-10-31 14:46:58,746 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:46:58,746 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-10-31 14:46:58,746 INFO SenderThread:33232 [dir_watcher.py:finish():358] shutting down directory watcher +2023-10-31 14:46:58,755 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:46:58,933 INFO Thread-12 :33232 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/wandb-summary.json +2023-10-31 14:46:58,934 INFO SenderThread:33232 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/output.log +2023-10-31 14:46:58,934 INFO SenderThread:33232 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files +2023-10-31 14:46:58,934 INFO SenderThread:33232 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/output.log output.log +2023-10-31 14:46:58,934 INFO SenderThread:33232 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/requirements.txt requirements.txt +2023-10-31 14:46:58,934 INFO SenderThread:33232 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/conda-environment.yaml conda-environment.yaml +2023-10-31 14:46:58,940 INFO SenderThread:33232 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/wandb-summary.json wandb-summary.json +2023-10-31 14:46:58,954 INFO SenderThread:33232 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/config.yaml config.yaml +2023-10-31 14:46:58,955 INFO SenderThread:33232 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/wandb-metadata.json wandb-metadata.json +2023-10-31 14:46:58,956 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 10 +2023-10-31 14:46:58,956 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:46:58,962 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:46:58,968 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 10 +2023-10-31 14:46:58,969 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:46:58,969 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-10-31 14:46:58,970 INFO SenderThread:33232 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 14:46:59,691 INFO wandb-upload_0:33232 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/output.log +2023-10-31 14:46:59,756 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:46:59,757 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:46:59,859 INFO wandb-upload_2:33232 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/conda-environment.yaml +2023-10-31 14:47:00,066 INFO wandb-upload_1:33232 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/requirements.txt +2023-10-31 14:47:00,073 INFO wandb-upload_4:33232 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/config.yaml +2023-10-31 14:47:00,101 INFO wandb-upload_3:33232 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/files/wandb-summary.json +2023-10-31 14:47:00,301 INFO Thread-11 :33232 [sender.py:transition_state():613] send defer: 11 +2023-10-31 14:47:00,302 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:47:00,302 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 11 +2023-10-31 14:47:00,303 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:47:00,303 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-10-31 14:47:00,303 INFO SenderThread:33232 [file_pusher.py:join():181] waiting for file pusher +2023-10-31 14:47:00,303 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 12 +2023-10-31 14:47:00,303 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:47:00,304 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 12 +2023-10-31 14:47:00,304 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:47:00,304 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-10-31 14:47:00,304 INFO SenderThread:33232 [file_stream.py:finish():595] file stream finish called +2023-10-31 14:47:00,757 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:47:01,402 INFO SenderThread:33232 [file_stream.py:finish():599] file stream finish is done +2023-10-31 14:47:01,402 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 13 +2023-10-31 14:47:01,403 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:47:01,403 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:47:01,403 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 13 +2023-10-31 14:47:01,404 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:47:01,404 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-10-31 14:47:01,404 INFO SenderThread:33232 [sender.py:transition_state():613] send defer: 14 +2023-10-31 14:47:01,405 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: defer +2023-10-31 14:47:01,405 DEBUG SenderThread:33232 [sender.py:send():380] send: final +2023-10-31 14:47:01,405 INFO HandlerThread:33232 [handler.py:handle_request_defer():172] handle defer: 14 +2023-10-31 14:47:01,405 DEBUG SenderThread:33232 [sender.py:send():380] send: footer +2023-10-31 14:47:01,406 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: defer +2023-10-31 14:47:01,406 INFO SenderThread:33232 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-10-31 14:47:01,407 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:47:01,407 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:47:01,408 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 14:47:01,409 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 14:47:01,409 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: server_info +2023-10-31 14:47:01,410 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: server_info +2023-10-31 14:47:01,416 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: get_summary +2023-10-31 14:47:01,417 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: sampled_history +2023-10-31 14:47:01,417 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 14:47:01,418 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: job_info +2023-10-31 14:47:01,729 DEBUG SenderThread:33232 [sender.py:send_request():407] send_request: job_info +2023-10-31 14:47:01,729 INFO MainThread:33232 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-10-31 14:47:01,730 INFO MainThread:33232 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-10-31 14:47:01,730 INFO MainThread:33232 [wandb_run.py:_footer_sync_info():3558] logging synced files +2023-10-31 14:47:01,730 DEBUG HandlerThread:33232 [handler.py:handle_request():146] handle_request: shutdown +2023-10-31 14:47:01,731 INFO HandlerThread:33232 [handler.py:finish():866] shutting down handler +2023-10-31 14:47:02,418 INFO WriterThread:33232 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/run-e8zkn3bl.wandb +2023-10-31 14:47:02,729 INFO SenderThread:33232 [sender.py:finish():1534] shutting down sender +2023-10-31 14:47:02,730 INFO SenderThread:33232 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 14:47:02,730 INFO SenderThread:33232 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231031_144427-e8zkn3bl/logs/debug.log b/wandb/run-20231031_144427-e8zkn3bl/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a9bca883c60e9927b70b97cae7e064b8017144cb --- /dev/null +++ b/wandb/run-20231031_144427-e8zkn3bl/logs/debug.log @@ -0,0 +1,28 @@ +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_setup.py:_flush():76] Configure stats pid to 33101 +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/logs/debug.log +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_144427-e8zkn3bl/logs/debug-internal.log +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_init.py:init():568] calling init triggers +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'RMSprop', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 49} +2023-10-31 14:44:27,021 INFO MainThread:33101 [wandb_init.py:init():618] starting backend +2023-10-31 14:44:27,022 INFO MainThread:33101 [wandb_init.py:init():622] setting up manager +2023-10-31 14:44:27,023 INFO MainThread:33101 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 14:44:27,024 INFO MainThread:33101 [wandb_init.py:init():628] backend started and connected +2023-10-31 14:44:27,029 INFO MainThread:33101 [wandb_init.py:init():720] updated telemetry +2023-10-31 14:44:27,029 INFO MainThread:33101 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 14:44:27,898 INFO MainThread:33101 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 14:44:28,362 INFO MainThread:33101 [wandb_run.py:_on_init():2229] got version response +2023-10-31 14:44:28,362 INFO MainThread:33101 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 14:44:34,612 INFO MainThread:33101 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 14:44:34,613 INFO MainThread:33101 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 14:44:34,613 INFO MainThread:33101 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 14:44:34,613 INFO MainThread:33101 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 14:44:34,614 INFO MainThread:33101 [wandb_init.py:init():845] run started, returning control to user process +2023-10-31 14:47:03,065 WARNING MsgRouterThr:33101 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20231031_144427-e8zkn3bl/run-e8zkn3bl.wandb b/wandb/run-20231031_144427-e8zkn3bl/run-e8zkn3bl.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c449768ca7111dc0d6e24c713179372beda3bac1 Binary files /dev/null and b/wandb/run-20231031_144427-e8zkn3bl/run-e8zkn3bl.wandb differ diff --git a/wandb/run-20231031_150242-stds1vjn/files/conda-environment.yaml b/wandb/run-20231031_150242-stds1vjn/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7f7cff6ba3dbbba8f29abfd550e642beb5aa829 --- /dev/null +++ b/wandb/run-20231031_150242-stds1vjn/files/conda-environment.yaml @@ -0,0 +1,162 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - greenlet==3.0.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pyparsing==3.1.1 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - regex==2023.8.8 + - requests==2.28.1 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - smmap==5.0.1 + - sqlalchemy==2.0.22 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typing-extensions==4.4.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - wandb==0.15.12 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231031_150242-stds1vjn/files/config.yaml b/wandb/run-20231031_150242-stds1vjn/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebdcf9611021ca43357ba4488750e686f40ab755 --- /dev/null +++ b/wandb/run-20231031_150242-stds1vjn/files/config.yaml @@ -0,0 +1,50 @@ +wandb_version: 1 + +optimizer: + desc: null + value: Adam +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 38 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698750162.540392 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + 3: + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231031_150242-stds1vjn/files/output.log b/wandb/run-20231031_150242-stds1vjn/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e3bd489d326401aba46e43db1c31fef2ae181211 --- /dev/null +++ b/wandb/run-20231031_150242-stds1vjn/files/output.log @@ -0,0 +1,54 @@ + +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +[LOG]: Login Succesfull. +wandb: WARNING Calling wandb.login() after wandb.init() has no effect. +[W 2023-10-31 17:35:16,857] Trial 0 failed with parameters: {'lr': 3.4274903008423467e-07, 'n_estimators': 38, 'optimizer': 'Adam'} because of the following error: AttributeError("'torch.return_types.max' object has no attribute 'to'"). +Traceback (most recent call last): + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 200, in _run_trial + value_or_values = func(trial) + File "hyperpara.py", line 68, in objective + val_loss, val_acc = engine.validation(model = model, dataloader = data.valid_dataloader, loss_fn = loss_fn, + File "/jash/Workspace/Food/engine.py", line 151, in validation + log_image_table(x_val, torch.max(logits.data, 1), y_val, logits.softmax(dim=1)) + File "/jash/Workspace/Food/engine.py", line 110, in log_image_table + for img, pred, targ, prob in zip(images.to("cpu"), predicted.to("cpu"), labels.to("cpu"), probs.to("cpu")): +AttributeError: 'torch.return_types.max' object has no attribute 'to' +[W 2023-10-31 17:35:16,858] Trial 0 failed with value None. +Traceback (most recent call last): + File "hyperpara.py", line 110, in + study.optimize(objective, n_trials=11) + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/study.py", line 451, in optimize + _optimize( + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 66, in _optimize + _optimize_sequential( + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 163, in _optimize_sequential + frozen_trial = _run_trial(study, func, catch) + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 251, in _run_trial + raise func_err + File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 200, in _run_trial + value_or_values = func(trial) + File "hyperpara.py", line 68, in objective + val_loss, val_acc = engine.validation(model = model, dataloader = data.valid_dataloader, loss_fn = loss_fn, + File "/jash/Workspace/Food/engine.py", line 151, in validation + log_image_table(x_val, torch.max(logits.data, 1), y_val, logits.softmax(dim=1)) + File "/jash/Workspace/Food/engine.py", line 110, in log_image_table + for img, pred, targ, prob in zip(images.to("cpu"), predicted.to("cpu"), labels.to("cpu"), probs.to("cpu")): +AttributeError: 'torch.return_types.max' object has no attribute 'to' \ No newline at end of file diff --git a/wandb/run-20231031_150242-stds1vjn/files/requirements.txt b/wandb/run-20231031_150242-stds1vjn/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f84e89ededbacd967e61bd8559b50c3efecc0021 --- /dev/null +++ b/wandb/run-20231031_150242-stds1vjn/files/requirements.txt @@ -0,0 +1,133 @@ +accelerate==0.22.0 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +executing==1.2.0 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +greenlet==3.0.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +regex==2023.8.8 +requests==2.28.1 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +six==1.16.0 +smmap==5.0.1 +sqlalchemy==2.0.22 +stack-data==0.6.2 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typing-extensions==4.4.0 +tzdata==2023.3 +urllib3==1.26.13 +wandb==0.15.12 +wcwidth==0.2.8 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_150242-stds1vjn/files/wandb-metadata.json b/wandb/run-20231031_150242-stds1vjn/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..399c53beea0680a3bf69a3ec8a40419682c9b95a --- /dev/null +++ b/wandb/run-20231031_150242-stds1vjn/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T11:02:43.977627", + "startedAt": "2023-10-31T11:02:42.534507", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.5019999999999996, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.289, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.496, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.192, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.201, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.192, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.013, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.747, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.7, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.807, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.305, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.298, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.396, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.204, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.201, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.803, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.701, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.802, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.7576484680176 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_150242-stds1vjn/files/wandb-summary.json b/wandb/run-20231031_150242-stds1vjn/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..993ba0b3bfdecf7e2456bc12d284deebcf3da03d --- /dev/null +++ b/wandb/run-20231031_150242-stds1vjn/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 9152}} \ No newline at end of file diff --git a/wandb/run-20231031_150242-stds1vjn/logs/debug-internal.log b/wandb/run-20231031_150242-stds1vjn/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b051b88f640b2779ddfd6c98fc70e48b8e6ee2ab --- /dev/null +++ b/wandb/run-20231031_150242-stds1vjn/logs/debug-internal.log @@ -0,0 +1,4077 @@ +2023-10-31 15:02:42,543 INFO StreamThr :33534 [internal.py:wandb_internal():86] W&B internal server running at pid: 33534, started at: 2023-10-31 15:02:42.542294 +2023-10-31 15:02:42,545 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status +2023-10-31 15:02:42,548 INFO WriterThread:33534 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/run-stds1vjn.wandb +2023-10-31 15:02:42,552 DEBUG SenderThread:33534 [sender.py:send():380] send: header +2023-10-31 15:02:42,552 DEBUG SenderThread:33534 [sender.py:send():380] send: run +2023-10-31 15:02:43,401 INFO SenderThread:33534 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files +2023-10-31 15:02:43,401 INFO SenderThread:33534 [sender.py:_start_run_threads():1122] run started: stds1vjn with start time 1698750162.540392 +2023-10-31 15:02:43,418 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 15:02:43,419 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: check_version +2023-10-31 15:02:43,889 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 15:02:43,905 DEBUG HandlerThread:33534 [system_info.py:__init__():32] System info init +2023-10-31 15:02:43,905 DEBUG HandlerThread:33534 [system_info.py:__init__():47] System info init done +2023-10-31 15:02:43,905 INFO HandlerThread:33534 [system_monitor.py:start():194] Starting system monitor +2023-10-31 15:02:43,905 INFO SystemMonitor:33534 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 15:02:43,905 INFO HandlerThread:33534 [system_monitor.py:probe():214] Collecting system info +2023-10-31 15:02:43,906 INFO SystemMonitor:33534 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 15:02:43,907 INFO SystemMonitor:33534 [interfaces.py:start():190] Started disk monitoring +2023-10-31 15:02:43,908 INFO SystemMonitor:33534 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 15:02:43,909 INFO SystemMonitor:33534 [interfaces.py:start():190] Started memory monitoring +2023-10-31 15:02:43,909 INFO SystemMonitor:33534 [interfaces.py:start():190] Started network monitoring +2023-10-31 15:02:43,977 DEBUG HandlerThread:33534 [system_info.py:probe():196] Probing system +2023-10-31 15:02:43,985 DEBUG HandlerThread:33534 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 15:02:43,985 DEBUG HandlerThread:33534 [system_info.py:probe():244] Probing system done +2023-10-31 15:02:43,985 DEBUG HandlerThread:33534 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T11:02:43.977627', 'startedAt': '2023-10-31T11:02:42.534507', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.5019999999999996, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.289, 'min': 1200.0, 'max': 3600.0}, {'current': 1.496, 'min': 1200.0, 'max': 3600.0}, {'current': 1.192, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.201, 'min': 1200.0, 'max': 3600.0}, {'current': 1.192, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 2.013, 'min': 1200.0, 'max': 3600.0}, {'current': 1.747, 'min': 1200.0, 'max': 3600.0}, {'current': 2.7, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.807, 'min': 1200.0, 'max': 3600.0}, {'current': 1.305, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.298, 'min': 1200.0, 'max': 3600.0}, {'current': 1.396, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.204, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.201, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.803, 'min': 1200.0, 'max': 3600.0}, {'current': 1.701, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.802, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.7576484680176}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 15:02:43,986 INFO HandlerThread:33534 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 15:02:43,986 INFO HandlerThread:33534 [system_monitor.py:probe():227] Publishing system info +2023-10-31 15:02:43,986 DEBUG HandlerThread:33534 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 15:02:43,986 DEBUG HandlerThread:33534 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 15:02:43,987 DEBUG HandlerThread:33534 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 15:02:44,406 INFO Thread-12 :33534 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/conda-environment.yaml +2023-10-31 15:02:44,407 INFO Thread-12 :33534 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/requirements.txt +2023-10-31 15:02:50,170 DEBUG HandlerThread:33534 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 15:02:50,172 INFO HandlerThread:33534 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 15:02:50,177 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:02:50,177 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 15:02:50,178 DEBUG SenderThread:33534 [sender.py:send():380] send: files +2023-10-31 15:02:50,178 INFO SenderThread:33534 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 15:02:50,189 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:02:50,190 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:02:50,195 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:02:50,404 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/conda-environment.yaml +2023-10-31 15:02:50,404 INFO Thread-12 :33534 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/wandb-metadata.json +2023-10-31 15:02:50,715 DEBUG SenderThread:33534 [sender.py:send():380] send: telemetry +2023-10-31 15:02:51,465 INFO wandb-upload_0:33534 [upload_job.py:push():131] Uploaded file /tmp/tmpc9o_44c7wandb/mo1yjlxf-wandb-metadata.json +2023-10-31 15:02:53,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:02:58,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:03:03,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:03:05,190 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:03:05,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:03:05,191 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:03:09,549 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:03:14,562 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:03:15,413 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/config.yaml +2023-10-31 15:03:19,962 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:03:20,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:03:20,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:03:20,191 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:03:25,627 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:03:30,628 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:03:35,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:03:35,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:03:35,192 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:03:36,598 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:03:41,598 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:03:43,910 DEBUG SystemMonitor:33534 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 15:03:43,914 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:03:46,915 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:03:50,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:03:50,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:03:50,192 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:03:52,624 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:03:57,624 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:04:02,625 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:04:05,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:04:05,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:04:05,191 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:04:08,551 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:04:13,552 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:04:13,915 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:04:18,917 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:04:20,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:04:20,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:04:20,191 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:04:24,563 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:04:29,564 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:04:34,565 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:04:35,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:04:35,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:04:35,191 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:04:39,584 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:04:43,918 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:04:44,920 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:04:49,920 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:04:50,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:04:50,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:04:50,192 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:04:55,565 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:05:00,566 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:05:05,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:05:05,191 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:05:05,233 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:05:05,604 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:05:10,605 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:05:13,921 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:05:15,922 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:05:20,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:05:20,191 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:05:20,233 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:05:21,549 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:05:26,550 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:05:31,551 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:05:35,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:05:35,192 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:05:35,192 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:05:36,554 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:05:41,555 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:05:43,924 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:05:46,926 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:05:50,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:05:50,192 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:05:50,192 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:05:52,566 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:05:57,567 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:06:02,568 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:06:05,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:06:05,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:06:05,194 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:06:08,539 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:06:13,540 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:06:13,926 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:06:18,928 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:06:20,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:06:20,192 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:06:20,233 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:06:24,612 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:06:29,613 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:06:34,613 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:06:35,192 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:06:35,192 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:06:35,233 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:06:40,547 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:06:43,928 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:06:45,930 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:06:50,192 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:06:50,192 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:06:50,233 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:06:51,566 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:06:56,567 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:06:57,491 INFO Thread-12 :33534 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 15:06:59,492 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 15:07:01,974 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:07:05,192 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:07:05,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:07:05,193 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:07:07,555 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:07:12,556 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:07:13,931 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:07:17,933 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:07:20,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:07:20,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:07:20,193 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:07:23,630 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:07:28,631 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:07:33,632 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:07:35,192 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:07:35,193 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:07:35,233 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:07:39,598 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:07:43,934 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:07:44,936 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:07:49,937 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:07:50,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:07:50,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:07:50,194 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:07:55,565 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:08:00,566 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:08:05,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:08:05,193 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:08:05,233 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:08:05,600 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:08:10,601 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:08:13,936 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:08:15,938 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:08:20,192 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:08:20,193 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:08:20,233 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:08:21,627 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:08:26,628 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:08:31,629 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:08:35,192 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:08:35,193 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:08:35,233 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:08:37,586 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:08:42,586 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:08:43,939 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:08:47,941 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:08:50,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:08:50,193 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:08:50,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:08:53,521 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:08:58,522 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:09:03,523 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:09:05,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:09:05,193 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:09:05,233 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:09:08,551 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:09:13,552 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:09:13,941 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:09:18,942 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:09:20,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:09:20,193 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:09:20,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:09:24,571 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:09:29,572 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:09:34,572 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:09:35,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:09:35,193 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:09:35,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:09:39,580 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:09:43,943 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:09:44,944 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:09:49,945 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:09:50,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:09:50,194 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:09:50,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:09:55,613 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:10:00,613 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:10:05,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:10:05,194 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:10:05,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:10:06,531 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:10:11,532 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:10:13,945 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:10:16,947 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:10:20,194 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:10:20,194 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:10:20,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:10:22,604 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:10:27,605 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:10:32,606 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:10:35,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:10:35,193 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:10:35,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:10:38,542 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:10:43,543 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:10:43,948 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:10:48,950 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:10:50,193 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:10:50,194 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:10:50,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:10:54,632 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:10:59,632 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:11:04,633 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:11:05,194 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:11:05,194 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:11:05,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:11:10,608 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:11:13,950 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:11:15,951 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:11:20,194 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:11:20,194 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:11:20,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:11:21,595 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:11:26,596 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:11:31,597 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:11:35,194 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:11:35,194 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:11:35,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:11:37,588 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:11:42,589 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:11:43,952 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:11:47,954 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:11:50,194 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:11:50,195 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:11:50,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:11:53,581 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:11:58,581 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:12:03,582 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:12:05,194 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:12:05,195 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:12:05,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:12:09,570 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:12:13,956 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:12:14,957 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:12:19,958 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:12:20,194 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:12:20,194 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:12:20,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:12:25,613 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:12:30,613 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:12:35,194 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:12:35,195 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:12:35,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:12:36,531 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:12:41,532 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:12:43,958 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:12:46,960 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:12:50,194 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:12:50,195 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:12:50,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:12:52,539 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:12:57,540 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:13:02,541 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:13:05,196 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:13:05,196 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:13:05,196 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:13:07,542 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:13:12,543 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:13:13,961 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:13:17,963 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:13:20,196 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:13:20,196 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:13:20,196 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:13:23,551 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:13:28,551 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:13:33,552 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:13:35,195 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:13:35,196 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:13:35,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:13:38,608 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:13:43,608 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:13:43,963 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:13:48,964 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:13:50,196 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:13:50,196 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:13:50,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:13:54,596 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:13:59,597 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:14:04,598 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:14:05,196 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:14:05,197 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:14:05,197 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:14:10,595 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:14:13,965 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:14:15,966 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:14:20,197 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:14:20,197 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:14:20,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:14:21,610 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:14:26,611 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:14:31,612 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:14:35,196 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:14:35,196 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:14:35,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:14:37,600 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:14:42,600 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:14:43,966 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:14:47,968 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:14:50,196 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:14:50,196 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:14:50,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:14:53,541 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:14:58,542 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:15:03,543 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:15:05,196 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:15:05,196 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:15:05,200 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:15:05,672 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 15:15:08,566 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:15:13,567 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:15:13,969 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:15:18,971 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:15:20,196 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:15:20,197 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:15:20,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:15:24,622 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:15:29,623 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:15:34,623 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:15:35,196 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:15:35,197 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:15:35,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:15:40,593 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:15:43,971 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:15:45,973 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:15:50,196 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:15:50,197 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:15:50,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:15:51,531 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:15:56,531 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:16:01,532 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:16:05,197 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:16:05,197 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:16:05,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:16:06,642 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:16:11,643 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:16:13,973 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:16:16,975 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:16:20,197 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:16:20,197 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:16:20,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:16:22,621 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:16:27,621 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:16:32,622 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:16:35,197 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:16:35,197 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:16:35,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:16:38,593 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:16:43,593 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:16:43,975 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:16:48,977 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:16:50,197 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:16:50,198 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:16:50,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:16:54,555 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:16:59,555 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:17:04,556 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:17:05,197 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:17:05,198 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:17:05,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:17:09,601 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:17:13,977 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:17:14,979 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:17:19,980 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:17:20,197 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:17:20,197 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:17:20,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:17:25,604 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:17:30,604 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:17:35,197 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:17:35,198 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:17:35,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:17:36,565 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:17:41,566 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:17:43,981 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:17:46,982 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:17:50,198 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:17:50,198 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:17:50,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:17:52,616 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:17:57,617 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:18:02,617 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:18:05,198 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:18:05,198 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:18:05,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:18:07,622 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:18:12,623 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:18:13,983 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:18:17,985 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:18:20,197 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:18:20,198 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:18:20,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:18:23,593 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:18:28,593 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:18:33,594 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:18:35,198 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:18:35,198 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:18:35,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:18:38,614 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:18:43,615 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:18:43,985 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:18:48,987 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:18:50,198 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:18:50,198 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:18:50,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:18:54,540 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:18:59,541 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:19:04,541 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:19:05,198 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:19:05,198 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:19:05,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:19:09,628 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:19:13,768 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 15:19:13,988 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:19:14,990 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:19:19,991 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:19:20,198 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:19:20,198 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:19:20,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:19:25,593 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:19:30,594 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:19:35,198 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:19:35,198 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:19:35,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:19:35,616 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:19:40,617 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:19:43,990 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:19:45,991 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:19:50,198 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:19:50,199 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:19:50,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:19:51,613 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:19:56,613 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:20:01,614 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:20:05,198 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:20:05,199 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:20:05,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:20:06,614 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:20:11,615 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:20:13,992 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:20:16,994 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:20:20,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:20:20,199 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:20:20,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:20:22,606 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:20:27,607 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:20:32,608 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:20:35,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:20:35,199 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:20:35,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:20:38,578 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:20:43,579 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:20:43,994 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:20:48,995 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:20:50,198 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:20:50,199 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:20:50,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:20:54,539 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:20:59,540 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:21:04,541 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:21:05,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:21:05,199 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:21:05,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:21:09,586 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:21:13,996 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:21:14,998 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:21:19,999 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:21:20,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:21:20,199 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:21:20,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:21:25,604 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:21:30,605 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:21:35,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:21:35,199 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:21:35,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:21:36,555 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:21:41,556 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:21:43,998 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:21:47,000 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:21:50,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:21:50,199 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:21:50,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:21:52,655 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:21:57,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:22:02,657 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:22:05,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:22:05,200 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:22:05,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:22:08,612 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:22:13,613 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:22:14,001 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:22:19,003 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:22:20,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:22:20,199 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:22:20,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:22:24,624 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:22:29,625 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:22:34,626 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:22:35,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:22:35,200 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:22:35,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:22:40,576 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:22:44,005 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:22:46,007 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:22:50,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:22:50,200 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:22:50,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:22:51,654 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:22:56,655 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:23:01,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:23:05,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:23:05,200 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:23:05,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:23:07,611 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:23:12,611 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:23:14,007 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:23:18,197 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:23:18,858 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 15:23:20,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:23:20,200 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:23:20,200 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:23:23,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:23:28,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:23:33,657 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:23:35,200 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:23:35,201 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:23:35,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:23:39,626 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:23:44,009 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:23:45,011 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:23:50,012 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:23:50,200 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:23:50,201 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:23:50,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:23:55,652 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:24:00,652 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:24:05,201 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:24:05,201 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:24:05,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:24:06,569 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:24:11,570 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:24:14,012 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:24:17,014 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:24:20,201 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:24:20,201 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:24:20,201 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:24:22,617 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:24:27,618 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:24:32,619 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:24:35,201 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:24:35,201 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:24:35,202 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:24:38,567 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:24:43,567 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:24:44,015 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:24:49,017 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:24:50,201 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:24:50,202 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:24:50,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:24:54,617 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:24:59,617 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:25:04,618 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:25:05,201 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:25:05,202 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:25:05,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:25:09,629 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:25:14,017 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:25:15,019 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:25:20,020 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:25:20,201 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:25:20,201 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:25:20,202 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:25:25,523 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:25:30,524 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:25:35,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:25:35,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:25:35,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:25:35,602 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:25:40,603 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:25:44,019 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:25:46,020 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:25:50,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:25:50,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:25:50,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:25:51,561 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:25:56,562 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:26:01,563 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:26:05,201 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:26:05,202 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:26:05,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:26:06,618 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:26:11,619 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:26:14,022 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:26:17,024 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:26:20,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:26:20,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:26:20,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:26:22,608 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:26:27,609 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:26:32,609 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:26:35,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:26:35,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:26:35,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:26:38,573 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:26:43,574 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:26:44,023 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:26:49,025 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:26:50,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:26:50,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:26:50,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:26:54,546 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:26:59,546 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:27:04,547 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:27:05,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:27:05,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:27:05,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:27:09,628 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:27:14,026 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:27:15,027 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:27:20,028 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:27:20,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:27:20,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:27:20,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:27:22,947 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 15:27:25,351 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:27:30,352 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:27:35,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:27:35,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:27:35,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:27:35,619 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:27:40,620 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:27:44,027 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:27:46,029 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:27:50,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:27:50,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:27:50,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:27:51,643 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:27:56,644 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:28:01,644 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:28:05,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:28:05,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:28:05,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:28:07,602 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:28:12,603 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:28:14,030 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:28:18,031 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:28:20,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:28:20,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:28:20,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:28:23,552 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:28:28,553 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:28:33,554 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:28:35,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:28:35,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:28:35,204 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:28:38,558 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:28:43,559 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:28:44,033 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:28:49,035 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:28:50,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:28:50,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:28:50,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:28:54,527 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:28:59,528 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:29:04,529 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:29:05,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:29:05,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:29:05,204 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:29:09,569 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:29:14,035 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:29:15,036 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:29:20,037 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:29:20,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:29:20,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:29:20,204 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:29:25,575 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:29:30,576 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:29:35,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:29:35,203 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:29:35,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:29:36,563 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:29:41,564 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:29:44,037 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:29:47,039 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:29:50,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:29:50,204 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:29:50,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:29:52,551 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:29:57,552 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:30:02,552 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:30:05,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:30:05,204 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:30:05,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:30:07,587 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:30:12,588 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:30:14,040 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:30:18,041 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:30:20,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:30:20,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:30:20,204 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:30:23,552 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:30:28,553 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:30:33,554 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:30:35,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:30:35,204 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:30:35,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:30:38,610 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:30:43,611 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:30:44,043 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:30:49,045 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:30:50,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:30:50,204 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:30:50,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:30:54,565 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:30:59,565 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:31:04,566 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:31:05,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:31:05,205 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:31:05,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:31:10,516 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:31:14,045 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:31:16,047 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:31:20,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:31:20,204 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:31:20,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:31:22,001 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:31:23,034 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 15:31:27,002 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:31:32,003 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:31:35,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:31:35,204 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:31:35,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:31:37,639 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:31:42,640 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:31:44,047 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:31:48,049 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:31:50,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:31:50,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:31:50,205 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:31:53,645 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:31:58,646 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:32:03,647 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:32:05,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:32:05,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:32:05,205 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:32:09,603 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:32:14,051 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:32:15,052 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:32:20,053 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:32:20,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:32:20,204 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:32:20,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:32:25,586 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:32:30,587 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:32:35,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:32:35,205 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:32:35,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:32:35,616 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:32:40,617 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:32:44,052 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:32:46,053 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:32:50,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:32:50,205 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:32:50,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:32:51,626 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:32:56,627 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:33:01,628 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:33:05,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:33:05,205 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:33:05,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:33:07,571 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:33:12,572 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:33:14,054 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:33:18,056 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:33:20,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:33:20,205 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:33:20,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:33:23,539 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:33:28,540 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:33:33,540 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:33:35,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:33:35,205 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:33:35,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:33:38,551 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:33:43,552 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:33:44,056 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:33:49,058 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:33:50,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:33:50,205 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:33:50,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:33:54,633 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:33:59,633 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:34:04,634 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:34:05,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:34:05,206 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:34:05,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:34:10,585 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:34:14,058 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:34:16,060 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:34:20,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:34:20,206 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:34:20,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:34:21,596 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:34:26,596 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:34:31,597 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:34:35,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:34:35,205 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:34:35,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:34:36,599 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:34:41,600 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:34:44,060 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:34:47,062 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:34:50,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:34:50,206 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:34:50,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:34:52,573 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:34:57,574 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:35:02,574 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:35:05,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:35:05,206 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:35:05,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:35:07,632 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:35:12,633 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:35:14,062 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:35:18,064 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:35:20,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:35:20,206 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:35:20,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:35:23,119 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 15:35:23,901 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:35:28,902 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:35:33,903 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:35:35,205 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:35:35,206 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:35:35,209 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:35:39,540 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:35:44,064 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:35:45,067 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:35:50,068 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:35:50,206 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:35:50,206 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:35:50,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:35:55,629 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:36:00,630 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:36:05,206 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:36:05,206 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:36:05,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:36:06,544 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:36:11,544 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:36:14,066 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:36:17,068 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:36:20,206 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:36:20,206 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:36:20,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:36:22,590 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:36:27,591 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:36:32,592 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:36:35,206 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:36:35,206 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:36:35,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:36:38,544 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:36:43,545 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:36:44,068 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:36:49,070 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:36:50,207 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:36:50,207 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:36:50,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:36:54,597 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:36:59,598 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:37:04,598 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:37:05,206 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:37:05,206 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:37:05,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:37:10,575 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:37:14,070 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:37:16,072 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:37:20,206 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:37:20,207 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:37:20,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:37:21,554 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:37:26,555 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:37:31,555 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:37:35,207 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:37:35,207 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:37:35,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:37:37,548 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:37:42,549 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:37:44,072 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:37:48,074 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:37:50,207 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:37:50,207 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:37:50,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:37:53,622 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:37:58,623 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:38:03,624 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:38:05,207 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:38:05,207 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:38:05,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:38:09,559 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:38:14,075 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:38:15,076 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:38:20,077 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:38:20,207 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:38:20,207 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:38:20,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:38:25,593 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:38:30,594 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:38:35,207 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:38:35,208 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:38:35,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:38:36,526 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:38:41,527 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:38:44,077 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:38:47,079 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:38:50,207 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:38:50,208 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:38:50,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:38:52,615 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:38:57,616 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:39:02,617 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:39:05,207 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:39:05,207 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:39:05,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:39:08,554 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:39:13,555 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:39:14,080 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:39:19,081 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:39:20,207 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:39:20,208 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:39:20,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:39:24,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:39:25,214 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 15:39:29,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:39:34,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:39:35,207 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:39:35,208 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:39:35,211 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:39:39,587 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:39:44,082 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:39:45,083 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:39:50,084 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:39:50,207 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:39:50,208 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:39:50,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:39:55,579 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:40:00,580 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:40:05,208 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:40:05,208 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:40:05,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:40:05,632 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:40:10,633 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:40:14,084 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:40:16,085 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:40:20,208 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:40:20,208 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:40:20,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:40:21,606 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:40:26,606 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:40:31,607 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:40:35,210 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:40:35,210 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:40:35,210 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:40:36,633 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:40:41,634 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:40:44,087 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:40:47,089 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:40:50,209 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:40:50,209 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:40:50,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:40:52,569 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:40:57,569 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:41:02,570 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:41:05,209 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:41:05,209 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:41:05,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:41:07,629 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:41:12,630 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:41:14,089 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:41:18,091 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:41:20,208 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:41:20,209 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:41:20,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:41:23,655 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:41:28,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:41:33,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:41:35,209 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:41:35,209 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:41:35,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:41:39,632 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:41:44,092 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:41:45,093 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:41:50,094 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:41:50,208 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:41:50,209 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:41:50,249 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:41:55,582 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:42:00,583 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:42:05,209 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:42:05,209 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:42:05,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:42:05,664 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:42:10,665 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:42:14,094 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:42:16,096 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:42:20,209 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:42:20,210 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:42:20,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:42:21,620 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:42:26,621 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:42:31,621 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:42:35,209 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:42:35,209 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:42:35,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:42:37,526 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:42:42,527 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:42:44,097 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:42:48,099 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:42:50,210 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:42:50,210 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:42:50,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:42:53,534 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:42:58,535 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:43:03,536 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:43:05,209 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:43:05,210 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:43:05,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:43:08,580 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:43:13,581 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:43:14,099 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:43:19,101 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:43:20,210 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:43:20,210 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:43:20,211 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:43:24,600 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:43:29,600 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:43:32,304 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 15:43:34,946 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:43:35,209 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:43:35,210 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:43:35,214 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:43:40,531 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:43:44,102 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:43:46,103 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:43:50,209 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:43:50,210 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:43:50,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:43:51,588 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:43:56,588 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:44:01,589 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:44:05,209 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:44:05,210 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:44:05,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:44:06,618 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:44:11,619 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:44:14,104 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:44:17,106 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:44:20,210 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:44:20,210 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:44:20,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:44:22,546 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:44:27,547 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:44:32,548 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:44:35,210 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:44:35,210 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:44:35,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:44:37,643 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:44:42,643 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:44:44,106 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:44:48,107 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:44:50,210 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:44:50,210 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:44:50,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:44:53,562 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:44:58,562 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:45:03,563 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:45:05,210 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:45:05,210 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:45:05,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:45:08,600 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:45:13,601 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:45:14,108 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:45:19,110 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:45:20,210 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:45:20,211 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:45:20,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:45:24,552 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:45:29,553 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:45:34,554 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:45:35,210 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:45:35,211 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:45:35,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:45:39,584 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:45:44,110 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:45:45,112 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:45:50,112 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:45:50,210 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:45:50,211 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:45:50,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:45:55,592 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:46:00,592 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:46:05,212 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:46:05,212 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:46:05,212 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:46:06,548 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:46:11,549 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:46:14,112 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:46:17,114 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:46:20,211 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:46:20,211 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:46:20,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:46:22,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:46:27,657 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:46:32,657 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:46:35,211 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:46:35,211 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:46:35,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:46:38,638 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:46:43,639 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:46:44,115 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:46:49,116 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:46:50,211 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:46:50,211 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:46:50,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:46:54,630 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:46:59,631 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:47:04,631 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:47:05,211 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:47:05,211 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:47:05,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:47:09,640 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:47:14,117 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:47:15,118 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:47:20,119 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:47:20,211 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:47:20,211 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:47:20,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:47:25,632 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:47:30,633 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:47:35,211 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:47:35,211 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:47:35,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:47:36,537 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:47:41,538 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:47:44,120 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:47:47,122 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:47:50,211 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:47:50,212 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:47:50,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:47:52,612 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:47:57,613 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:48:02,614 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:48:05,211 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:48:05,212 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:48:05,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:48:08,589 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:48:13,590 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:48:14,122 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:48:19,123 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:48:20,211 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:48:20,212 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:48:20,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:48:24,593 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:48:29,593 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:48:34,594 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:48:35,211 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:48:35,212 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:48:35,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:48:40,590 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:48:44,125 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:48:46,126 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:48:50,212 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:48:50,212 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:48:50,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:48:51,600 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:48:56,601 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:49:01,601 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:49:05,212 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:49:05,212 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:49:05,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:49:06,629 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:49:11,630 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:49:14,127 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:49:17,128 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:49:20,212 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:49:20,212 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:49:20,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:49:22,548 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:49:27,549 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:49:32,550 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:49:35,212 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:49:35,213 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:49:35,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:49:37,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:49:42,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:49:44,128 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:49:48,130 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:49:50,212 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:49:50,213 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:49:50,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:49:53,631 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:49:58,632 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:50:03,633 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:50:05,213 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:50:05,213 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:50:05,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:50:09,586 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:50:14,130 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:50:15,131 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:50:20,132 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:50:20,213 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:50:20,213 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:50:20,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:50:25,555 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:50:30,556 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:50:35,213 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:50:35,213 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:50:35,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:50:36,551 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:50:41,551 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:50:44,132 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:50:47,134 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:50:50,213 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:50:50,213 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:50:50,257 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:50:52,602 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:50:57,603 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:51:02,603 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:51:05,213 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:51:05,213 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:51:05,253 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:51:07,629 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:51:12,629 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:51:14,134 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:51:18,135 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:51:20,213 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:51:20,213 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:51:20,257 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:51:23,588 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:51:28,589 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:51:33,590 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:51:35,213 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:51:35,213 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:51:35,257 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:51:38,629 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:51:43,630 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:51:44,136 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:51:49,138 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:51:50,213 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:51:50,214 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:51:50,257 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:51:54,638 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:51:59,639 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:52:04,639 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:52:05,213 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:52:05,213 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:52:05,257 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:52:10,635 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:52:14,138 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:52:16,140 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:52:20,213 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:52:20,214 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:52:20,257 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:52:21,650 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:52:26,651 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:52:31,651 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:52:35,213 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:52:35,213 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:52:35,257 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:52:37,641 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:52:42,642 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:52:44,142 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:52:48,143 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:52:50,213 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:52:50,214 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:52:50,257 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:52:53,618 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:52:58,619 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:53:03,619 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:53:05,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:53:05,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:53:05,291 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:53:08,680 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:53:13,681 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:53:14,143 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:53:19,145 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:53:20,289 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:53:20,289 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:53:20,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:53:24,633 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:53:29,634 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:53:34,634 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:53:35,289 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:53:35,289 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:53:35,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:53:39,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:53:44,147 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:53:45,148 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:53:50,149 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:53:50,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:53:50,290 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:53:50,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:53:55,623 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:54:00,624 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:54:05,289 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:54:05,290 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:54:05,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:54:05,678 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:54:10,679 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:54:14,149 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:54:16,151 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:54:20,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:54:20,290 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:54:20,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:54:21,705 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:54:26,706 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:54:31,706 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:54:35,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:54:35,290 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:54:35,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:54:37,662 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:54:42,663 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:54:44,151 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:54:48,153 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:54:50,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:54:50,290 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:54:50,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:54:53,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:54:58,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:55:03,691 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:55:05,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:55:05,290 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:55:05,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:55:09,684 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:55:14,153 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:55:15,154 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:55:20,155 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:55:20,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:55:20,290 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:55:20,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:55:25,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:55:30,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:55:35,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:55:35,290 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:55:35,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:55:36,687 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:55:42,177 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:55:42,571 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 15:55:44,156 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:55:48,158 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:55:50,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:55:50,290 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:55:50,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:55:53,728 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:55:58,729 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:56:03,730 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:56:05,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:56:05,291 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:56:05,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:56:09,655 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:56:14,159 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:56:15,160 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:56:20,161 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:56:20,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:56:20,291 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:56:20,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:56:25,625 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:56:30,626 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:56:35,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:56:35,291 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:56:35,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:56:35,706 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:56:40,706 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:56:44,162 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:56:46,164 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:56:50,291 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:56:50,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:56:50,292 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:56:51,647 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:56:56,648 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:57:01,648 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:57:05,291 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:57:05,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:57:05,293 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:57:06,659 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:57:11,659 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:57:14,165 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:57:17,166 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:57:20,291 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:57:20,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:57:20,292 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:57:22,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:57:27,725 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:57:32,726 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:57:35,291 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:57:35,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:57:35,293 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:57:38,685 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:57:43,686 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:57:44,168 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:57:49,170 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:57:50,291 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:57:50,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:57:50,293 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:57:54,713 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:57:59,713 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:58:04,714 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:58:05,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:58:05,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:58:05,293 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:58:10,697 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:58:14,170 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:58:16,171 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:58:20,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:58:20,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:58:20,293 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:58:21,723 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:58:26,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:58:31,725 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:58:35,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:58:35,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:58:35,293 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:58:37,681 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:58:42,682 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:58:44,173 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:58:48,175 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:58:50,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:58:50,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:58:50,293 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:58:53,688 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:58:58,688 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:59:03,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:59:05,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:59:05,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:59:05,293 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:59:09,655 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:59:14,175 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:59:15,176 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:59:20,177 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:59:20,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:59:20,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:59:20,293 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:59:25,725 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:59:30,726 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:59:35,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:59:35,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:59:35,294 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:59:36,723 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:59:41,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:59:44,177 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 15:59:47,145 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:59:48,658 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 15:59:50,292 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 15:59:50,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 15:59:50,294 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 15:59:52,709 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 15:59:57,710 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:00:02,711 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:00:05,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:00:05,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:00:05,294 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:00:08,661 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:00:13,662 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:00:14,180 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:00:19,182 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:00:20,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:00:20,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:00:20,294 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:00:24,654 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:00:29,654 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:00:34,655 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:00:35,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:00:35,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:00:35,294 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:00:39,743 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:00:44,182 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:00:45,184 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:00:50,184 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:00:50,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:00:50,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:00:50,295 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:00:55,676 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:01:00,677 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:01:05,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:01:05,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:01:05,294 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:01:06,667 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:01:11,668 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:01:14,184 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:01:17,186 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:01:20,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:01:20,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:01:20,295 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:01:22,629 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:01:27,630 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:01:32,630 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:01:35,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:01:35,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:01:35,295 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:01:37,734 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:01:42,735 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:01:44,186 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:01:48,188 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:01:50,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:01:50,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:01:50,295 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:01:53,721 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:01:58,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:02:03,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:02:05,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:02:05,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:02:05,295 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:02:09,638 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:02:14,189 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:02:15,190 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:02:20,191 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:02:20,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:02:20,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:02:20,295 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:02:25,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:02:30,691 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:02:35,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:02:35,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:02:35,295 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:02:36,633 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:02:41,634 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:02:44,191 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:02:47,192 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:02:50,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:02:50,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:02:50,295 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:02:52,713 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:02:57,713 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:03:02,714 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:03:05,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:03:05,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:03:05,295 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:03:07,788 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:03:12,789 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:03:14,193 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:03:18,194 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:03:20,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:03:20,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:03:20,295 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:03:23,646 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:03:28,646 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:03:33,647 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:03:35,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:03:35,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:03:35,295 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:03:38,732 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:03:43,733 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:03:44,195 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:03:49,197 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:03:50,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:03:50,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:03:50,296 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:03:52,744 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 16:03:54,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:03:59,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:04:04,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:04:05,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:04:05,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:04:05,296 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:04:09,736 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:04:14,198 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:04:15,199 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:04:20,200 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:04:20,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:04:20,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:04:20,296 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:04:25,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:04:30,723 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:04:35,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:04:35,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:04:35,296 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:04:36,703 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:04:41,703 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:04:44,200 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:04:47,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:04:50,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:04:50,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:04:50,296 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:04:52,738 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:04:57,738 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:05:02,739 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:05:05,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:05:05,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:05:05,296 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:05:08,721 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:05:13,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:05:14,202 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:05:19,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:05:20,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:05:20,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:05:20,296 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:05:24,718 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:05:29,719 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:05:34,720 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:05:35,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:05:35,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:05:35,297 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:05:40,683 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:05:44,204 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:05:46,206 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:05:50,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:05:50,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:05:50,297 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:05:51,630 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:05:56,631 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:06:01,631 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:06:05,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:06:05,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:06:05,297 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:06:06,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:06:11,725 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:06:14,207 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:06:17,208 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:06:20,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:06:20,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:06:20,297 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:06:22,624 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:06:27,625 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:06:32,626 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:06:35,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:06:35,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:06:35,297 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:06:37,679 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:06:42,679 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:06:44,224 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:06:48,226 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:06:50,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:06:50,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:06:50,298 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:06:53,676 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:06:58,677 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:07:03,677 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:07:05,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:07:05,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:07:05,297 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:07:09,665 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:07:14,223 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:07:15,224 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:07:20,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:07:20,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:07:20,297 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:07:20,669 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:07:25,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:07:30,671 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:07:35,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:07:35,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:07:35,297 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:07:35,705 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:07:40,705 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:07:44,229 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:07:46,230 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:07:50,295 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:07:50,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:07:50,297 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:07:51,669 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:07:56,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:08:01,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:08:05,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:08:05,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:08:05,297 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:08:06,728 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:08:11,729 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:08:14,228 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:08:17,230 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:08:20,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:08:20,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:08:20,298 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:08:22,666 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:08:27,667 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:08:32,668 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:08:35,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:08:35,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:08:35,298 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:08:38,627 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:08:43,627 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:08:44,230 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:08:49,232 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:08:50,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:08:50,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:08:50,298 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:08:54,691 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:08:59,692 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:09:04,692 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:09:05,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:09:05,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:09:05,298 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:09:10,658 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:09:14,233 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:09:16,236 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:09:20,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:09:20,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:09:20,298 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:09:21,633 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:09:26,634 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:09:31,634 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:09:35,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:09:35,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:09:35,298 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:09:36,643 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:09:41,643 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:09:44,235 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:09:47,237 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:09:50,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:09:50,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:09:50,298 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:09:52,625 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:09:57,626 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:10:02,627 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:10:05,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:10:05,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:10:05,299 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:10:07,712 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:10:12,713 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:10:14,239 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:10:18,241 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:10:20,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:10:20,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:10:20,299 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:10:23,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:10:28,718 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:10:33,719 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:10:35,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:10:35,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:10:35,299 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:10:39,713 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:10:44,241 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:10:45,243 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:10:50,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:10:50,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:10:50,299 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:10:50,649 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:10:55,649 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:11:00,650 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:11:05,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:11:05,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:11:05,299 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:11:05,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:11:10,691 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:11:14,243 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:11:16,245 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:11:20,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:11:20,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:11:20,299 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:11:21,658 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:11:26,659 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:11:31,660 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:11:35,297 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:11:35,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:11:35,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:11:36,684 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:11:41,684 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:11:44,245 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:11:47,247 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:11:50,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:11:50,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:11:50,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:11:52,635 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:11:57,636 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:12:02,637 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:12:05,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:12:05,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:12:05,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:12:08,627 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:12:13,628 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:12:14,248 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:12:19,250 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:12:20,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:12:20,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:12:20,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:12:24,655 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:12:29,655 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:12:34,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:12:35,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:12:35,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:12:35,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:12:39,677 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:12:44,250 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:12:45,252 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:12:50,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:12:50,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:12:50,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:12:50,715 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:12:55,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:13:00,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:13:05,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:13:05,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:13:05,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:13:05,971 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:13:10,971 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:13:14,252 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:13:16,254 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:13:20,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:13:20,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:13:20,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:13:21,634 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:13:26,635 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:13:31,636 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:13:35,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:13:35,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:13:35,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:13:36,721 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:13:41,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:13:44,254 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:13:47,256 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:13:50,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:13:50,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:13:50,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:13:52,639 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:13:57,639 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:14:02,640 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:14:05,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:14:05,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:14:05,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:14:07,659 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:14:12,660 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:14:14,256 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:14:18,257 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:14:20,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:14:20,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:14:20,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:14:23,649 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:14:28,649 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:14:33,650 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:14:35,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:14:35,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:14:35,300 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:14:38,651 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:14:43,652 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:14:44,259 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:14:49,261 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:14:50,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:14:50,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:14:50,301 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:14:54,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:14:59,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:15:04,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:15:05,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:15:05,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:15:05,301 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:15:10,668 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:15:14,261 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:15:16,263 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:15:20,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:15:20,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:15:20,301 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:15:21,725 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:15:26,726 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:15:31,727 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:15:35,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:15:35,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:15:35,301 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:15:37,622 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:15:42,623 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:15:44,263 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:15:48,264 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:15:50,008 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 16:15:50,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:15:50,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:15:50,302 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:15:53,650 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:15:58,651 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:16:03,652 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:16:05,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:16:05,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:16:05,301 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:16:08,686 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:16:13,687 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:16:14,265 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:16:19,267 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:16:20,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:16:20,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:16:20,302 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:16:24,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:16:29,723 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:16:34,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:16:35,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:16:35,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:16:35,302 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:16:40,674 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:16:44,268 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:16:46,270 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:16:50,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:16:50,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:16:50,301 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:16:51,679 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:16:56,680 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:17:01,680 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:17:05,299 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:17:05,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:17:05,302 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:17:07,668 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:17:12,669 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:17:14,270 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:17:18,272 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:17:20,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:17:20,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:17:20,302 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:17:23,708 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:17:28,708 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:17:33,709 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:17:35,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:17:35,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:17:35,302 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:17:38,721 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:17:43,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:17:44,273 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:17:49,274 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:17:50,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:17:50,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:17:50,302 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:17:54,683 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:17:59,684 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:18:04,684 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:18:05,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:18:05,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:18:05,302 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:18:09,735 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:18:14,276 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:18:15,278 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:18:20,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:18:20,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:18:20,302 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:18:20,728 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:18:25,729 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:18:30,730 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:18:35,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:18:35,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:18:35,302 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:18:36,692 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:18:41,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:18:44,279 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:18:47,281 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:18:50,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:18:50,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:18:50,303 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:18:52,663 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:18:57,664 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:19:02,665 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:19:05,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:19:05,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:19:05,303 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:19:07,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:19:12,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:19:14,281 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:19:18,283 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:19:20,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:19:20,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:19:20,303 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:19:23,709 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:19:28,710 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:19:33,711 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:19:35,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:19:35,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:19:35,302 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:19:39,709 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:19:44,283 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:19:45,285 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:19:50,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:19:50,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:19:50,303 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:19:50,648 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:19:55,649 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:20:00,650 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:20:05,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:20:05,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:20:05,303 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:20:05,707 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:20:10,707 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:20:14,286 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:20:16,288 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:20:20,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:20:20,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:20:20,303 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:20:21,715 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:20:26,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:20:31,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:20:35,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:20:35,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:20:35,303 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:20:36,746 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:20:41,746 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:20:44,288 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:20:47,290 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:20:50,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:20:50,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:20:50,303 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:20:52,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:20:57,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:21:02,671 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:21:05,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:21:05,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:21:05,303 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:21:07,739 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:21:12,740 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:21:14,291 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:21:18,293 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:21:20,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:21:20,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:21:20,303 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:21:23,645 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:21:28,646 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:21:33,647 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:21:35,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:21:35,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:21:35,303 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:21:38,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:21:43,671 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:21:44,292 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:21:49,294 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:21:50,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:21:50,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:21:50,303 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:21:54,684 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:21:59,685 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:22:04,685 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:22:05,301 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:22:05,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:22:05,304 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:22:10,684 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:22:14,294 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:22:16,296 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:22:20,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:22:20,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:22:20,304 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:22:21,683 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:22:26,683 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:22:31,684 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:22:35,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:22:35,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:22:35,304 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:22:37,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:22:42,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:22:44,297 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:22:48,298 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:22:50,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:22:50,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:22:50,304 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:22:53,707 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:22:58,708 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:23:03,709 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:23:05,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:23:05,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:23:05,304 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:23:09,695 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:23:14,298 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:23:15,300 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:23:20,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:23:20,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:23:20,304 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:23:20,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:23:25,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:23:30,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:23:35,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:23:35,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:23:35,304 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:23:35,719 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:23:40,720 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:23:44,301 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:23:46,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:23:50,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:23:50,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:23:50,304 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:23:51,749 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:23:56,749 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:24:01,750 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:24:05,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:24:05,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:24:05,304 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:24:07,676 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:24:12,676 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:24:14,303 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:24:18,305 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:24:20,302 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:24:20,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:24:20,304 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:24:23,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:24:28,723 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:24:33,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:24:35,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:24:35,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:24:35,305 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:24:39,698 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:24:44,305 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:24:45,306 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:24:50,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:24:50,306 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:24:50,307 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:24:50,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:24:55,723 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:25:00,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:25:05,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:25:05,306 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:25:05,307 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:25:06,712 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:25:11,713 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:25:14,307 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:25:17,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:25:20,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:25:20,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:25:20,307 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:25:22,669 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:25:27,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:25:32,671 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:25:35,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:25:35,306 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:25:35,307 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:25:38,664 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:25:43,665 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:25:44,309 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:25:49,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:25:50,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:25:50,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:25:50,307 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:25:54,725 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:25:59,726 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:26:04,727 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:26:05,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:26:05,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:26:05,307 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:26:10,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:26:14,311 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:26:16,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:26:20,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:26:20,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:26:20,307 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:26:21,721 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:26:26,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:26:31,723 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:26:35,303 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:26:35,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:26:35,308 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:26:37,654 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:26:42,655 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:26:44,314 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:26:48,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:26:50,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:26:50,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:26:50,308 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:26:53,762 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:26:58,763 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:27:03,764 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:27:05,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:27:05,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:27:05,308 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:27:09,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:27:14,317 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:27:15,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:27:20,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:27:20,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:27:20,308 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:27:20,734 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:27:25,735 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:27:30,736 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:27:35,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:27:35,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:27:35,308 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:27:36,687 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:27:41,687 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:27:44,319 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:27:47,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:27:50,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:27:50,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:27:50,308 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:27:52,732 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:27:57,732 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:28:02,733 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:28:05,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:28:05,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:28:05,308 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:28:08,639 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:28:13,640 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:28:14,319 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:28:19,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:28:20,305 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:28:20,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:28:20,308 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:28:24,666 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:28:29,666 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:28:34,667 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:28:35,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:28:35,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:28:35,308 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:28:39,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:28:44,323 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:28:45,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:28:50,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:28:50,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:28:50,309 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:28:50,766 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:28:55,766 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:29:00,767 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:29:05,304 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:29:05,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:29:05,308 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:29:06,676 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:29:11,677 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:29:14,324 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:29:17,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:29:20,305 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:29:20,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:29:20,309 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:29:22,733 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:29:27,734 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:29:32,734 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:29:35,305 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:29:35,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:29:35,308 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:29:38,647 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:29:43,648 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:29:44,331 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:29:49,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:29:50,305 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:29:50,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:29:50,309 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:29:54,672 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:29:59,673 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:30:04,674 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:30:05,305 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:30:05,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:30:05,309 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:30:09,708 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:30:14,330 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:30:15,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:30:20,305 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:30:20,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:30:20,309 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:30:20,699 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:30:25,700 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:30:30,701 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:30:35,305 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:30:35,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:30:35,309 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:30:36,688 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:30:41,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:30:44,331 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:30:47,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:30:50,305 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:30:50,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:30:50,309 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:30:52,653 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:30:57,654 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:31:02,655 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:31:05,305 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:31:05,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:31:05,309 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:31:07,672 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:31:12,673 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:31:14,334 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:31:18,336 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:31:20,306 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:31:20,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:31:20,309 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:31:23,704 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:31:28,705 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:31:33,705 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:31:35,305 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:31:35,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:31:35,310 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:31:39,674 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:31:44,336 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:31:45,337 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:31:50,306 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:31:50,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:31:50,310 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:31:50,685 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:31:55,686 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:32:00,687 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:32:05,306 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:32:05,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:32:05,310 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:32:06,685 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:32:11,685 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:32:14,338 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:32:17,340 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:32:20,306 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:32:20,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:32:20,310 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:32:22,680 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:32:27,681 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:32:32,681 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:32:35,306 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:32:35,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:32:35,310 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:32:37,730 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:32:42,731 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:32:44,340 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:32:48,342 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:32:50,306 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:32:50,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:32:50,310 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:32:53,748 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:32:58,748 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:33:03,749 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:33:05,306 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:33:05,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:33:05,310 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:33:09,657 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:33:14,342 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:33:15,344 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:33:20,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:33:20,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:33:20,310 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:33:20,741 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:33:25,742 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:33:30,743 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:33:35,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:33:35,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:33:35,310 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:33:36,639 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:33:41,640 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:33:44,345 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:33:47,346 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:33:50,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:33:50,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:33:50,310 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:33:52,666 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:33:57,667 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:34:02,668 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:34:05,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:34:05,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:34:05,310 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:34:07,752 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:34:12,753 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:34:14,346 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:34:18,348 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:34:20,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:34:20,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:34:20,310 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:34:23,657 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:34:28,658 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:34:33,659 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:34:35,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:34:35,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:34:35,311 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:34:38,682 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:34:43,683 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:34:44,349 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:34:49,351 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:34:50,307 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:34:50,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:34:50,311 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:34:54,739 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:34:59,740 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:35:04,741 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:35:05,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:35:05,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:35:05,311 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:35:10,708 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:35:14,351 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:35:16,353 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:35:20,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:35:20,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:35:20,311 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:35:21,678 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:35:26,679 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:35:31,679 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:35:35,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:35:35,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:35:35,311 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:35:36,747 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:35:41,748 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:35:44,354 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:35:47,355 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:35:50,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:35:50,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:35:50,311 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:35:52,658 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:35:57,659 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:36:01,454 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 16:36:03,271 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:36:05,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:36:05,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:36:05,311 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:36:08,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:36:13,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:36:14,356 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:36:19,358 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:36:20,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:36:20,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:36:20,311 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:36:24,648 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:36:29,649 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:36:34,650 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:36:35,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:36:35,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:36:35,311 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:36:39,661 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:36:44,358 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:36:45,360 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:36:50,308 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:36:50,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:36:50,311 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:36:50,706 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:36:55,707 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:37:00,707 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:37:05,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:37:05,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:37:05,311 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:37:05,727 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:37:10,728 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:37:14,361 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:37:16,363 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:37:20,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:37:20,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:37:20,312 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:37:21,681 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:37:26,682 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:37:31,683 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:37:35,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:37:35,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:37:35,312 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:37:37,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:37:42,671 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:37:44,365 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:37:48,367 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:37:50,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:37:50,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:37:50,312 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:37:53,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:37:58,671 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:38:03,671 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:38:05,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:38:05,312 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:38:05,313 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:38:09,653 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:38:14,367 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:38:15,368 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:38:20,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:38:20,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:38:20,313 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:38:20,672 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:38:25,673 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:38:30,673 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:38:35,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:38:35,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:38:35,313 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:38:35,683 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:38:40,683 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:38:44,369 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:38:46,371 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:38:50,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:38:50,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:38:50,313 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:38:51,764 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:38:56,765 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:39:01,766 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:39:05,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:39:05,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:39:05,313 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:39:07,739 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:39:12,740 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:39:14,371 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:39:18,372 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:39:20,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:39:20,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:39:20,313 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:39:23,747 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:39:28,748 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:39:33,749 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:39:35,309 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:39:35,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:39:35,314 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:39:39,673 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:39:44,373 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:39:45,374 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:39:50,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:39:50,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:39:50,314 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:39:50,665 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:39:55,665 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:40:00,666 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:40:05,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:40:05,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:40:05,314 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:40:05,750 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:40:10,750 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:40:14,374 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:40:16,375 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:40:20,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:40:20,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:40:20,314 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:40:21,728 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:40:26,729 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:40:31,729 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:40:35,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:40:35,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:40:35,314 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:40:37,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:40:42,657 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:40:44,376 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:40:48,378 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:40:50,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:40:50,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:40:50,314 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:40:53,728 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:40:58,729 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:41:03,730 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:41:05,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:41:05,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:41:05,314 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:41:09,706 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:41:14,378 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:41:15,380 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:41:20,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:41:20,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:41:20,314 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:41:20,697 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:41:25,698 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:41:30,699 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:41:35,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:41:35,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:41:35,314 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:41:35,736 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:41:40,737 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:41:44,381 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:41:46,383 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:41:50,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:41:50,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:41:50,314 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:41:51,704 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:41:56,704 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:42:01,705 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:42:05,310 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:42:05,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:42:05,315 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:42:07,657 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:42:12,658 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:42:14,383 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:42:18,385 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:42:20,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:42:20,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:42:20,314 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:42:23,742 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:42:28,742 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:42:33,743 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:42:35,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:42:35,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:42:35,314 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:42:39,637 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:42:44,385 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:42:45,386 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:42:50,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:42:50,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:42:50,315 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:42:50,641 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:42:55,642 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:43:00,643 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:43:05,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:43:05,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:43:05,315 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:43:05,714 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:43:10,714 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:43:14,387 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:43:16,388 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:43:20,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:43:20,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:43:20,315 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:43:21,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:43:26,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:43:31,718 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:43:35,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:43:35,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:43:35,315 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:43:37,718 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:43:42,718 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:43:44,389 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:43:48,391 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:43:50,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:43:50,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:43:50,315 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:43:53,671 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:43:58,672 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:44:03,673 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:44:05,311 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:44:05,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:44:05,315 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:44:08,732 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:44:13,733 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:44:14,392 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:44:19,394 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:44:20,312 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:44:20,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:44:20,315 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:44:24,735 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:44:29,736 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:44:34,737 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:44:35,312 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:44:35,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:44:35,315 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:44:40,657 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:44:44,394 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:44:46,396 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:44:50,312 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:44:50,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:44:50,315 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:44:51,712 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:44:56,712 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:45:01,713 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:45:05,312 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:45:05,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:45:05,316 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:45:06,739 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:45:11,740 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:45:14,396 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:45:17,397 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:45:20,312 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:45:20,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:45:20,316 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:45:22,692 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:45:27,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:45:32,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:45:35,312 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:45:35,345 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:45:35,345 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:45:37,750 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:45:42,751 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:45:44,398 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:45:48,400 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:45:50,312 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:45:50,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:45:50,316 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:45:53,708 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:45:58,709 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:46:03,710 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:46:05,312 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:46:05,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:46:05,316 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:46:08,720 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:46:13,721 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:46:14,400 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:46:19,402 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:46:20,312 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:46:20,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:46:20,316 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:46:24,729 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:46:29,730 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:46:34,731 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:46:35,312 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:46:35,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:46:35,316 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:46:39,733 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:46:44,402 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:46:45,404 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:46:50,312 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:46:50,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:46:50,316 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:46:50,651 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:46:55,651 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:47:00,652 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:47:05,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:47:05,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:47:05,317 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:47:05,663 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:47:10,664 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:47:14,405 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:47:16,406 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:47:20,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:47:20,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:47:20,317 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:47:21,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:47:26,694 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:47:31,695 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:47:35,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:47:35,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:47:35,317 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:47:36,736 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:47:41,737 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:47:44,407 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:47:47,408 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:47:50,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:47:50,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:47:50,317 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:47:52,669 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:47:57,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:48:01,722 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 16:48:03,027 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:48:05,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:48:05,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:48:05,317 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:48:08,686 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:48:13,687 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:48:14,410 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:48:19,412 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:48:20,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:48:20,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:48:20,317 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:48:24,695 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:48:29,696 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:48:34,697 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:48:35,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:48:35,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:48:35,317 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:48:40,676 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:48:44,414 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:48:46,416 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:48:50,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:48:50,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:48:50,317 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:48:51,643 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:48:56,644 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:49:01,644 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:49:05,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:49:05,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:49:05,317 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:49:06,758 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:49:11,759 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:49:14,416 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:49:17,417 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:49:20,313 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:49:20,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:49:20,317 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:49:22,714 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:49:27,714 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:49:32,715 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:49:35,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:49:35,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:49:35,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:49:37,761 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:49:42,761 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:49:44,419 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:49:48,420 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:49:50,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:49:50,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:49:50,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:49:53,738 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:49:58,738 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:50:03,739 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:50:05,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:50:05,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:50:05,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:50:09,712 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:50:14,420 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:50:15,422 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:50:20,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:50:20,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:50:20,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:50:20,697 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:50:25,697 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:50:30,698 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:50:35,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:50:35,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:50:35,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:50:36,691 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:50:41,691 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:50:44,424 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:50:47,425 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:50:50,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:50:50,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:50:50,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:50:52,737 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:50:57,738 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:51:02,739 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:51:05,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:51:05,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:51:05,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:51:08,691 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:51:13,692 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:51:14,427 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:51:19,428 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:51:20,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:51:20,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:51:20,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:51:24,735 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:51:29,736 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:51:34,737 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:51:35,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:51:35,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:51:35,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:51:40,648 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:51:44,428 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:51:46,430 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:51:50,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:51:50,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:51:50,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:51:51,712 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:51:56,713 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:52:01,808 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 16:52:02,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:52:05,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:52:05,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:52:05,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:52:07,740 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:52:12,741 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:52:14,431 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:52:18,433 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:52:20,314 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:52:20,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:52:20,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:52:23,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:52:28,694 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:52:33,695 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:52:35,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:52:35,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:52:35,318 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:52:38,703 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:52:43,704 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:52:44,432 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:52:49,433 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:52:50,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:52:50,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:52:50,319 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:52:54,645 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:52:59,646 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:53:04,647 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:53:05,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:53:05,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:53:05,319 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:53:09,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:53:14,435 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:53:15,436 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:53:20,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:53:20,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:53:20,319 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:53:20,692 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:53:25,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:53:30,694 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:53:35,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:53:35,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:53:35,320 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:53:36,682 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:53:41,683 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:53:44,436 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:53:47,438 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:53:50,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:53:50,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:53:50,319 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:53:52,676 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:53:57,676 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:54:02,677 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:54:05,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:54:05,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:54:05,319 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:54:07,738 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:54:12,739 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:54:14,439 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:54:18,441 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:54:20,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:54:20,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:54:20,319 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:54:23,675 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:54:28,675 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:54:33,676 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:54:35,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:54:35,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:54:35,320 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:54:38,733 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:54:43,734 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:54:44,441 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:54:49,443 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:54:50,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:54:50,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:54:50,319 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:54:54,732 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:54:59,732 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:55:04,733 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:55:05,315 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:55:05,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:55:05,320 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:55:10,665 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:55:14,443 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:55:16,444 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:55:20,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:55:20,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:55:20,319 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:55:21,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:55:26,672 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:55:31,672 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:55:35,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:55:35,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:55:35,320 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:55:36,702 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:55:41,702 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:55:44,445 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:55:47,447 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:55:50,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:55:50,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:55:50,320 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:55:52,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:55:57,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:56:02,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:56:05,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:56:05,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:56:05,320 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:56:08,660 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:56:13,660 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:56:14,447 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:56:19,449 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:56:20,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:56:20,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:56:20,320 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:56:24,755 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:56:29,756 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:56:34,756 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:56:35,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:56:35,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:56:35,320 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:56:40,753 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:56:44,449 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:56:46,451 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:56:50,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:56:50,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:56:50,320 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:56:51,673 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:56:56,674 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:57:01,675 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:57:05,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:57:05,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:57:05,320 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:57:06,728 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:57:11,729 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:57:14,452 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:57:17,454 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:57:20,316 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:57:20,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:57:20,320 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:57:22,687 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:57:27,688 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:57:32,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:57:35,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:57:35,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:57:35,321 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:57:37,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:57:42,725 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:57:44,454 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:57:48,456 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:57:50,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:57:50,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:57:50,321 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:57:53,742 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:57:58,743 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:58:03,744 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:58:05,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:58:05,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:58:05,322 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:58:09,695 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:58:14,457 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:58:15,458 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:58:20,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:58:20,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:58:20,321 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:58:20,649 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:58:25,650 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:58:30,651 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:58:35,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:58:35,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:58:35,321 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:58:35,669 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:58:40,670 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:58:44,459 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:58:46,460 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:58:50,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:58:50,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:58:50,321 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:58:51,945 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:58:56,945 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:59:01,946 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:59:05,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:59:05,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:59:05,321 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:59:07,661 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:59:12,662 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:59:14,460 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:59:18,462 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:59:20,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:59:20,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:59:20,321 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:59:23,679 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:59:28,680 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:59:33,681 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:59:35,317 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:59:35,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:59:35,321 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:59:38,738 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:59:43,739 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:59:44,462 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 16:59:49,464 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:59:50,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 16:59:50,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 16:59:50,321 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 16:59:54,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 16:59:59,657 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:00:04,657 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:00:05,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:00:05,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:00:05,322 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:00:09,667 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:00:14,465 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:00:15,466 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:00:16,987 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 17:00:20,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:00:20,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:00:20,322 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:00:20,715 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:00:25,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:00:30,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:00:35,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:00:35,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:00:35,322 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:00:35,748 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:00:40,749 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:00:44,466 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:00:46,468 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:00:50,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:00:50,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:00:50,322 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:00:51,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:00:56,723 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:01:01,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:01:05,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:01:05,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:01:05,322 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:01:07,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:01:12,723 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:01:14,469 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:01:18,471 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:01:20,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:01:20,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:01:20,322 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:01:23,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:01:28,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:01:33,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:01:35,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:01:35,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:01:35,322 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:01:38,729 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:01:43,730 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:01:44,471 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:01:49,473 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:01:50,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:01:50,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:01:50,322 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:01:54,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:01:59,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:02:04,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:02:05,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:02:05,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:02:05,322 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:02:10,656 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:02:14,473 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:02:16,475 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:02:20,318 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:02:20,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:02:20,322 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:02:21,740 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:02:26,741 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:02:31,742 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:02:35,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:02:35,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:02:35,322 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:02:37,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:02:42,723 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:02:44,477 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:02:44,529 WARNING FileStreamThread:33534 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer')). func: functools.partial(>, timeout=180), args: ('https://api.wandb.ai/files/maa_64/food-app/stds1vjn/file_stream',), kwargs: {'json': {'files': {'wandb-events.jsonl': {'offset': 238, 'content': ['{"system.network.sent": 1729723.93, "system.network.recv": 1081600.8, "system.proc.memory.availableMB": 188048.55, "system.memory": 27.1, "system.proc.memory.rssMB": 3417.7, "system.proc.memory.percent": 1.33, "system.cpu": 20.31, "system.cpu.0.cpu_percent": 23.55, "system.cpu.1.cpu_percent": 99.9, "system.cpu.2.cpu_percent": 10.07, "system.cpu.3.cpu_percent": 16.06, "system.cpu.4.cpu_percent": 11.47, "system.cpu.5.cpu_percent": 7.61, "system.cpu.6.cpu_percent": 5.9, "system.cpu.7.cpu_percent": 13.49, "system.cpu.8.cpu_percent": 21.53, "system.cpu.9.cpu_percent": 91.57, "system.cpu.10.cpu_percent": 25.37, "system.cpu.11.cpu_percent": 30.42, "system.cpu.12.cpu_percent": 36.27, "system.cpu.13.cpu_percent": 21.67, "system.cpu.14.cpu_percent": 30.12, "system.cpu.15.cpu_percent": 28.79, "system.cpu.16.cpu_percent": 36.55, "system.cpu.17.cpu_percent": 25.33, "system.cpu.18.cpu_percent": 10.73, "system.cpu.19.cpu_percent": 99.9, "system.cpu.20.cpu_percent": 22.87, "system.cpu.21.cpu_percent": 0.43, "system.cpu.22.cpu_percent": 28.71, "system.cpu.23.cpu_percent": 26.0, "system.cpu.24.cpu_percent": 34.57, "system.cpu.25.cpu_percent": 28.06, "system.cpu.26.cpu_percent": 36.35, "system.cpu.27.cpu_percent": 33.65, "system.cpu.28.cpu_percent": 34.05, "system.cpu.29.cpu_percent": 8.05, "system.cpu.30.cpu_percent": 17.24, "system.cpu.31.cpu_percent": 11.81, "system.cpu.32.cpu_percent": 25.49, "system.cpu.33.cpu_percent": 21.89, "system.cpu.34.cpu_percent": 29.57, "system.cpu.35.cpu_percent": 31.07, "system.cpu.36.cpu_percent": 8.44, "system.cpu.37.cpu_percent": 12.68, "system.cpu.38.cpu_percent": 34.85, "system.cpu.39.cpu_percent": 8.48, "system.proc.cpu.threads": 30, "system.disk.\\\\.usagePercent": 51.9, "system.disk.\\\\.usageGB": 866.77, "system.disk.in": 1074.79, "system.disk.out": 10132.56, "system.gpu.0.memoryAllocated": 93.91, "system.gpu.1.memoryAllocated": 90.12, "system.gpu.2.memoryAllocated": 0.82, "system.gpu.3.memoryAllocated": 88.37, "system.gpu.0.memoryAllocatedBytes": 32265601024.0, "system.gpu.1.memoryAllocatedBytes": 30964252672.0, "system.gpu.2.memoryAllocatedBytes": 281870336.0, "system.gpu.3.memoryAllocatedBytes": 30364467200.0, "system.gpu.0.memory": 49.33, "system.gpu.1.memory": 50.13, "system.gpu.2.memory": 0.0, "system.gpu.3.memory": 13.07, "system.gpu.0.gpu": 93.87, "system.gpu.1.gpu": 93.13, "system.gpu.2.gpu": 0.0, "system.gpu.3.gpu": 23.8, "system.gpu.0.temp": 57.13, "system.gpu.1.temp": 54.73, "system.gpu.2.temp": 48.0, "system.gpu.3.temp": 49.47, "system.gpu.0.powerWatts": 255.08, "system.gpu.1.powerWatts": 261.83, "system.gpu.2.powerWatts": 39.08, "system.gpu.3.powerWatts": 94.32, "system.gpu.0.powerPercent": 86.97, "system.gpu.1.powerPercent": 87.03, "system.gpu.2.powerPercent": 13.02, "system.gpu.3.powerPercent": 27.75, "_wandb": true, "_timestamp": 1698757364.473838, "_runtime": 7201.933446}']}}, 'dropped': 0}} +2023-10-31 17:02:48,478 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:02:50,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:02:50,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:02:50,323 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:02:54,003 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:02:59,004 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:03:04,005 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:03:05,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:03:05,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:03:05,323 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:03:09,651 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:03:14,479 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:03:15,481 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:03:20,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:03:20,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:03:20,323 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:03:20,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:03:25,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:03:30,694 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:03:35,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:03:35,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:03:35,323 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:03:37,022 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:03:42,023 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:03:44,482 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:03:44,504 WARNING FileStreamThread:33534 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer')). func: functools.partial(>, timeout=180), args: ('https://api.wandb.ai/files/maa_64/food-app/stds1vjn/file_stream',), kwargs: {'json': {'files': {'wandb-events.jsonl': {'offset': 240, 'content': ['{"system.network.sent": 1750571.33, "system.network.recv": 1100833.13, "system.proc.memory.availableMB": 188044.4, "system.memory": 27.1, "system.proc.memory.rssMB": 3417.94, "system.proc.memory.percent": 1.33, "system.cpu": 24.02, "system.cpu.0.cpu_percent": 24.4, "system.cpu.1.cpu_percent": 34.75, "system.cpu.2.cpu_percent": 38.42, "system.cpu.3.cpu_percent": 28.49, "system.cpu.4.cpu_percent": 18.42, "system.cpu.5.cpu_percent": 11.29, "system.cpu.6.cpu_percent": 35.23, "system.cpu.7.cpu_percent": 26.19, "system.cpu.8.cpu_percent": 10.51, "system.cpu.9.cpu_percent": 99.83, "system.cpu.10.cpu_percent": 24.48, "system.cpu.11.cpu_percent": 34.19, "system.cpu.12.cpu_percent": 22.7, "system.cpu.13.cpu_percent": 44.05, "system.cpu.14.cpu_percent": 35.09, "system.cpu.15.cpu_percent": 44.46, "system.cpu.16.cpu_percent": 41.69, "system.cpu.17.cpu_percent": 57.66, "system.cpu.18.cpu_percent": 9.49, "system.cpu.19.cpu_percent": 95.72, "system.cpu.20.cpu_percent": 28.8, "system.cpu.21.cpu_percent": 24.83, "system.cpu.22.cpu_percent": 20.16, "system.cpu.23.cpu_percent": 33.21, "system.cpu.24.cpu_percent": 38.77, "system.cpu.25.cpu_percent": 44.62, "system.cpu.26.cpu_percent": 30.97, "system.cpu.27.cpu_percent": 30.71, "system.cpu.28.cpu_percent": 38.05, "system.cpu.29.cpu_percent": 44.52, "system.cpu.30.cpu_percent": 29.51, "system.cpu.31.cpu_percent": 14.23, "system.cpu.32.cpu_percent": 25.89, "system.cpu.33.cpu_percent": 17.19, "system.cpu.34.cpu_percent": 12.55, "system.cpu.35.cpu_percent": 10.11, "system.cpu.36.cpu_percent": 13.36, "system.cpu.37.cpu_percent": 24.21, "system.cpu.38.cpu_percent": 34.59, "system.cpu.39.cpu_percent": 2.06, "system.proc.cpu.threads": 30, "system.disk.\\\\.usagePercent": 51.9, "system.disk.\\\\.usageGB": 866.77, "system.disk.in": 1074.79, "system.disk.out": 10137.39, "system.gpu.0.memoryAllocated": 93.9, "system.gpu.1.memoryAllocated": 90.12, "system.gpu.2.memoryAllocated": 0.82, "system.gpu.3.memoryAllocated": 88.37, "system.gpu.0.memoryAllocatedBytes": 32264814592.0, "system.gpu.1.memoryAllocatedBytes": 30964252672.0, "system.gpu.2.memoryAllocatedBytes": 281870336.0, "system.gpu.3.memoryAllocatedBytes": 30364467200.0, "system.gpu.0.memory": 51.73, "system.gpu.1.memory": 51.33, "system.gpu.2.memory": 0.0, "system.gpu.3.memory": 5.0, "system.gpu.0.gpu": 96.53, "system.gpu.1.gpu": 96.47, "system.gpu.2.gpu": 0.0, "system.gpu.3.gpu": 9.0, "system.gpu.0.temp": 57.27, "system.gpu.1.temp": 54.67, "system.gpu.2.temp": 48.0, "system.gpu.3.temp": 49.0, "system.gpu.0.powerWatts": 242.12, "system.gpu.1.powerWatts": 251.66, "system.gpu.2.powerWatts": 39.03, "system.gpu.3.powerWatts": 68.76, "system.gpu.0.powerPercent": 83.31, "system.gpu.1.powerPercent": 83.73, "system.gpu.2.powerPercent": 13.01, "system.gpu.3.powerPercent": 20.59, "_wandb": true, "_timestamp": 1698757424.479727, "_runtime": 7261.939335}']}}, 'dropped': 0}} +2023-10-31 17:03:47,483 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:03:50,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:03:50,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:03:50,323 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:03:52,723 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:03:57,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:04:02,725 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:04:05,319 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:04:05,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:04:05,323 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:04:07,732 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:04:12,732 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:04:14,484 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:04:18,485 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:04:20,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:04:20,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:04:20,323 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:04:23,762 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:04:28,762 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:04:33,763 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:04:35,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:04:35,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:04:35,323 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:04:39,741 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:04:44,487 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:04:45,489 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:04:50,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:04:50,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:04:50,323 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:04:50,763 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:04:55,764 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:05:00,764 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:05:05,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:05:05,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:05:05,323 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:05:06,680 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:05:11,681 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:05:14,489 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:05:17,491 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:05:20,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:05:20,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:05:20,324 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:05:22,673 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:05:27,674 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:05:32,675 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:05:35,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:05:35,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:05:35,324 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:05:37,683 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:05:42,684 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:05:44,492 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:05:48,494 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:05:50,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:05:50,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:05:50,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:05:53,660 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:05:58,660 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:06:03,661 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:06:05,320 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:06:05,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:06:05,324 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:06:08,751 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:06:13,752 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:06:14,494 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:06:19,496 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:06:20,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:06:20,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:06:20,324 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:06:24,759 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:06:29,760 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:06:34,761 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:06:35,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:06:35,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:06:35,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:06:40,709 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:06:44,498 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:06:46,499 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:06:50,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:06:50,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:06:50,324 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:06:51,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:06:56,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:07:01,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:07:05,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:07:05,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:07:05,324 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:07:06,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:07:11,725 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:07:14,499 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:07:17,501 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:07:20,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:07:20,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:07:20,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:07:22,706 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:07:27,707 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:07:32,708 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:07:35,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:07:35,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:07:35,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:07:38,668 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:07:43,669 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:07:44,502 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:07:49,504 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:07:50,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:07:50,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:07:50,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:07:54,673 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:07:59,674 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:08:04,675 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:08:05,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:08:05,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:08:05,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:08:09,742 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:08:14,504 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:08:15,506 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:08:20,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:08:20,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:08:20,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:08:20,657 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:08:25,171 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 17:08:25,738 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:08:30,739 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:08:35,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:08:35,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:08:35,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:08:36,707 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:08:41,708 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:08:44,506 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:08:47,508 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:08:50,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:08:50,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:08:50,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:08:52,763 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:08:57,764 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:09:02,765 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:09:05,321 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:09:05,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:09:05,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:09:08,695 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:09:13,696 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:09:14,509 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:09:19,511 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:09:20,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:09:20,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:09:20,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:09:24,769 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:09:29,770 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:09:34,770 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:09:35,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:09:35,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:09:35,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:09:40,739 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:09:44,512 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:09:46,513 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:09:50,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:09:50,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:09:50,326 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:09:51,750 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:09:56,750 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:10:01,751 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:10:05,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:10:05,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:10:05,326 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:10:07,696 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:10:12,697 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:10:14,513 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:10:18,515 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:10:20,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:10:20,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:10:20,325 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:10:23,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:10:28,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:10:33,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:10:35,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:10:35,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:10:35,326 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:10:39,681 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:10:44,515 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:10:45,516 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:10:50,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:10:50,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:10:50,326 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:10:50,679 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:10:55,679 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:11:00,680 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:11:05,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:11:05,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:11:05,326 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:11:05,747 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:11:10,747 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:11:14,518 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:11:16,519 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:11:20,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:11:20,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:11:20,326 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:11:21,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:11:26,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:11:31,718 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:11:35,322 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:11:35,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:11:35,326 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:11:37,711 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:11:42,711 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:11:44,520 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:11:48,521 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:11:50,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:11:50,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:11:50,326 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:11:53,701 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:11:58,702 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:12:03,703 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:12:05,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:12:05,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:12:05,327 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:12:08,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:12:13,718 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:12:14,522 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:12:19,523 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:12:20,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:12:20,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:12:20,326 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:12:24,756 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:12:29,756 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:12:34,757 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:12:35,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:12:35,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:12:35,327 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:12:40,747 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:12:44,523 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:12:46,525 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:12:50,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:12:50,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:12:50,327 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:12:51,723 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:12:56,724 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:13:01,725 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:13:05,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:13:05,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:13:05,327 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:13:06,760 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:13:11,761 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:13:14,526 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:13:17,527 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:13:20,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:13:20,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:13:20,327 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:13:22,725 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:13:27,726 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:13:32,727 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:13:35,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:13:35,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:13:35,327 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:13:37,734 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:13:42,735 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:13:44,529 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:13:48,531 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:13:50,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:13:50,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:13:50,327 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:13:53,762 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:13:58,763 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:14:03,764 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:14:05,323 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:14:05,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:14:05,327 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:14:09,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:14:14,204 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:14:14,531 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:14:14,613 WARNING FileStreamThread:33534 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer')). func: functools.partial(>, timeout=180), args: ('https://api.wandb.ai/files/maa_64/food-app/stds1vjn/file_stream',), kwargs: {'json': {'files': {'wandb-events.jsonl': {'offset': 261, 'content': ['{"system.network.sent": 1911573.93, "system.network.recv": 1207253.13, "system.proc.memory.availableMB": 187925.13, "system.memory": 27.1, "system.proc.memory.rssMB": 3453.26, "system.proc.memory.percent": 1.34, "system.cpu": 20.04, "system.cpu.0.cpu_percent": 28.75, "system.cpu.1.cpu_percent": 33.6, "system.cpu.2.cpu_percent": 99.9, "system.cpu.3.cpu_percent": 34.23, "system.cpu.4.cpu_percent": 15.3, "system.cpu.5.cpu_percent": 92.3, "system.cpu.6.cpu_percent": 16.29, "system.cpu.7.cpu_percent": 24.45, "system.cpu.8.cpu_percent": 37.1, "system.cpu.9.cpu_percent": 15.95, "system.cpu.10.cpu_percent": 13.49, "system.cpu.11.cpu_percent": 35.09, "system.cpu.12.cpu_percent": 99.9, "system.cpu.13.cpu_percent": 22.07, "system.cpu.14.cpu_percent": 23.54, "system.cpu.15.cpu_percent": 32.03, "system.cpu.16.cpu_percent": 30.25, "system.cpu.17.cpu_percent": 31.81, "system.cpu.18.cpu_percent": 8.75, "system.cpu.19.cpu_percent": 11.07, "system.cpu.20.cpu_percent": 15.89, "system.cpu.21.cpu_percent": 12.68, "system.cpu.22.cpu_percent": 0.77, "system.cpu.23.cpu_percent": 12.79, "system.cpu.24.cpu_percent": 26.83, "system.cpu.25.cpu_percent": 29.67, "system.cpu.26.cpu_percent": 30.8, "system.cpu.27.cpu_percent": 35.0, "system.cpu.28.cpu_percent": 23.14, "system.cpu.29.cpu_percent": 25.35, "system.cpu.30.cpu_percent": 33.69, "system.cpu.31.cpu_percent": 4.12, "system.cpu.32.cpu_percent": 0.77, "system.cpu.33.cpu_percent": 14.47, "system.cpu.34.cpu_percent": 31.41, "system.cpu.35.cpu_percent": 18.22, "system.cpu.36.cpu_percent": 6.33, "system.cpu.37.cpu_percent": 16.43, "system.cpu.38.cpu_percent": 27.47, "system.cpu.39.cpu_percent": 27.49, "system.proc.cpu.threads": 30, "system.disk.\\\\.usagePercent": 51.9, "system.disk.\\\\.usageGB": 866.77, "system.disk.in": 1074.79, "system.disk.out": 10252.77, "system.gpu.0.memoryAllocated": 93.9, "system.gpu.1.memoryAllocated": 90.12, "system.gpu.2.memoryAllocated": 0.82, "system.gpu.3.memoryAllocated": 88.37, "system.gpu.0.memoryAllocatedBytes": 32264814592.0, "system.gpu.1.memoryAllocatedBytes": 30964252672.0, "system.gpu.2.memoryAllocatedBytes": 281870336.0, "system.gpu.3.memoryAllocatedBytes": 30364467200.0, "system.gpu.0.memory": 49.93, "system.gpu.1.memory": 51.27, "system.gpu.2.memory": 0.0, "system.gpu.3.memory": 13.0, "system.gpu.0.gpu": 96.13, "system.gpu.1.gpu": 97.33, "system.gpu.2.gpu": 0.0, "system.gpu.3.gpu": 24.33, "system.gpu.0.temp": 57.13, "system.gpu.1.temp": 54.87, "system.gpu.2.temp": 47.87, "system.gpu.3.temp": 49.4, "system.gpu.0.powerWatts": 244.11, "system.gpu.1.powerWatts": 260.62, "system.gpu.2.powerWatts": 39.04, "system.gpu.3.powerWatts": 92.25, "system.gpu.0.powerPercent": 84.3, "system.gpu.1.powerPercent": 84.38, "system.gpu.2.powerPercent": 13.02, "system.gpu.3.powerPercent": 33.35, "_wandb": true, "_timestamp": 1698758054.528766, "_runtime": 7891.988374}']}}, 'dropped': 0}} +2023-10-31 17:14:19,533 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:14:20,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:14:20,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:14:20,328 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:14:24,715 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:14:29,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:14:34,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:14:35,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:14:35,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:14:35,328 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:14:40,707 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:14:44,533 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:14:46,535 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:14:50,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:14:50,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:14:50,327 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:14:51,745 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:14:56,746 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:15:01,746 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:15:05,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:15:05,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:15:05,328 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:15:07,704 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:15:12,704 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:15:14,536 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:15:14,609 WARNING FileStreamThread:33534 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer')). func: functools.partial(>, timeout=180), args: ('https://api.wandb.ai/files/maa_64/food-app/stds1vjn/file_stream',), kwargs: {'json': {'files': {'wandb-events.jsonl': {'offset': 263, 'content': ['{"system.network.sent": 1933208.07, "system.network.recv": 1228052.2, "system.proc.memory.availableMB": 187944.24, "system.memory": 27.1, "system.proc.memory.rssMB": 3453.26, "system.proc.memory.percent": 1.34, "system.cpu": 20.28, "system.cpu.0.cpu_percent": 26.69, "system.cpu.1.cpu_percent": 23.23, "system.cpu.2.cpu_percent": 11.54, "system.cpu.3.cpu_percent": 25.69, "system.cpu.4.cpu_percent": 99.8, "system.cpu.5.cpu_percent": 18.28, "system.cpu.6.cpu_percent": 20.15, "system.cpu.7.cpu_percent": 44.44, "system.cpu.8.cpu_percent": 23.73, "system.cpu.9.cpu_percent": 27.55, "system.cpu.10.cpu_percent": 15.97, "system.cpu.11.cpu_percent": 14.29, "system.cpu.12.cpu_percent": 99.77, "system.cpu.13.cpu_percent": 11.69, "system.cpu.14.cpu_percent": 4.54, "system.cpu.15.cpu_percent": 11.87, "system.cpu.16.cpu_percent": 71.53, "system.cpu.17.cpu_percent": 28.89, "system.cpu.18.cpu_percent": 11.93, "system.cpu.19.cpu_percent": 17.29, "system.cpu.20.cpu_percent": 14.89, "system.cpu.21.cpu_percent": 21.47, "system.cpu.22.cpu_percent": 33.26, "system.cpu.23.cpu_percent": 12.61, "system.cpu.24.cpu_percent": 31.61, "system.cpu.25.cpu_percent": 32.75, "system.cpu.26.cpu_percent": 30.39, "system.cpu.27.cpu_percent": 12.53, "system.cpu.28.cpu_percent": 24.9, "system.cpu.29.cpu_percent": 28.76, "system.cpu.30.cpu_percent": 31.04, "system.cpu.31.cpu_percent": 33.43, "system.cpu.32.cpu_percent": 1.53, "system.cpu.33.cpu_percent": 25.94, "system.cpu.34.cpu_percent": 35.54, "system.cpu.35.cpu_percent": 29.82, "system.cpu.36.cpu_percent": 18.13, "system.cpu.37.cpu_percent": 15.17, "system.cpu.38.cpu_percent": 32.81, "system.cpu.39.cpu_percent": 23.67, "system.proc.cpu.threads": 30, "system.disk.\\\\.usagePercent": 51.9, "system.disk.\\\\.usageGB": 866.77, "system.disk.in": 1074.79, "system.disk.out": 10256.82, "system.gpu.0.memoryAllocated": 93.9, "system.gpu.1.memoryAllocated": 90.12, "system.gpu.2.memoryAllocated": 0.82, "system.gpu.3.memoryAllocated": 88.37, "system.gpu.0.memoryAllocatedBytes": 32264971878.4, "system.gpu.1.memoryAllocatedBytes": 30964252672.0, "system.gpu.2.memoryAllocatedBytes": 281870336.0, "system.gpu.3.memoryAllocatedBytes": 30364467200.0, "system.gpu.0.memory": 51.27, "system.gpu.1.memory": 51.47, "system.gpu.2.memory": 0.0, "system.gpu.3.memory": 8.47, "system.gpu.0.gpu": 99.2, "system.gpu.1.gpu": 99.33, "system.gpu.2.gpu": 0.0, "system.gpu.3.gpu": 15.47, "system.gpu.0.temp": 57.4, "system.gpu.1.temp": 54.8, "system.gpu.2.temp": 48.0, "system.gpu.3.temp": 49.07, "system.gpu.0.powerWatts": 257.24, "system.gpu.1.powerWatts": 256.52, "system.gpu.2.powerWatts": 39.04, "system.gpu.3.powerWatts": 63.72, "system.gpu.0.powerPercent": 82.94, "system.gpu.1.powerPercent": 84.44, "system.gpu.2.powerPercent": 13.03, "system.gpu.3.powerPercent": 26.35, "_wandb": true, "_timestamp": 1698758114.533664, "_runtime": 7951.993272}']}}, 'dropped': 0}} +2023-10-31 17:15:18,538 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:15:20,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:15:20,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:15:20,328 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:15:24,202 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:15:29,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:15:34,203 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:15:35,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:15:35,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:15:35,328 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:15:39,743 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:15:44,539 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:15:45,540 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:15:50,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:15:50,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:15:50,328 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:15:50,756 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:15:55,757 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:16:00,757 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:16:05,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:16:05,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:16:05,328 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:16:06,747 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:16:11,748 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:16:14,541 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:16:17,543 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:16:20,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:16:20,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:16:20,328 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:16:22,662 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:16:27,663 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:16:32,664 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:16:35,324 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:16:35,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:16:35,328 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:16:37,710 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:16:42,711 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:16:44,542 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:16:48,543 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:16:50,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:16:50,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:16:50,328 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:16:53,756 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:16:58,757 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:17:03,757 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:17:05,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:17:05,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:17:05,328 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:17:09,659 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:17:14,544 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:17:15,545 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:17:20,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:17:20,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:17:20,328 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:17:20,774 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:17:25,775 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:17:30,775 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:17:35,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:17:35,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:17:35,329 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:17:36,767 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:17:41,768 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:17:44,546 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:17:47,548 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:17:50,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:17:50,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:17:50,329 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:17:52,677 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:17:57,678 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:18:02,679 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:18:05,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:18:05,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:18:05,329 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:18:07,705 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:18:12,706 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:18:14,549 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:18:18,551 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:18:20,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:18:20,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:18:20,329 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:18:23,715 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:18:28,715 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:18:33,716 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:18:35,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:18:35,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:18:35,329 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:18:39,704 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:18:44,551 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:18:45,553 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:18:50,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:18:50,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:18:50,329 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:18:50,742 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:18:55,742 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:19:00,743 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:19:05,325 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:19:05,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:19:05,329 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:19:06,740 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:19:11,741 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:19:14,553 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:19:17,555 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:19:20,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:19:20,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:19:20,329 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:19:22,748 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:19:27,748 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:19:32,749 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:19:35,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:19:35,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:19:35,330 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:19:38,692 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:19:43,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:19:44,556 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:19:49,558 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:19:50,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:19:50,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:19:50,330 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:19:54,736 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:19:59,736 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:20:04,737 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:20:05,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:20:05,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:20:05,330 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:20:10,729 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:20:14,559 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:20:16,561 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:20:20,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:20:20,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:20:20,330 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:20:21,681 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:20:26,682 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:20:31,682 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:20:35,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:20:35,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:20:35,330 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:20:36,697 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:20:41,698 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:20:44,560 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:20:47,561 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:20:50,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:20:50,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:20:50,330 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:20:52,718 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:20:57,719 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:21:02,719 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:21:05,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:21:05,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:21:05,330 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:21:08,692 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:21:13,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:21:14,562 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:21:19,564 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:21:20,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:21:20,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:21:20,330 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:21:24,697 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:21:29,698 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:21:34,699 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:21:35,326 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:21:35,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:21:35,330 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:21:40,654 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:21:44,564 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:21:46,565 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:21:50,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:21:50,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:21:50,330 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:21:51,703 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:21:56,703 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:22:01,704 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:22:05,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:22:05,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:22:05,330 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:22:06,728 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:22:11,729 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:22:14,567 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:22:17,569 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:22:20,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:22:20,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:22:20,330 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:22:22,754 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:22:27,754 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:22:32,755 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:22:35,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:22:35,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:22:35,331 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:22:38,694 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:22:43,695 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:22:44,568 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:22:49,570 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:22:50,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:22:50,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:22:50,331 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:22:54,683 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:22:59,683 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:23:04,684 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:23:05,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:23:05,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:23:05,331 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:23:09,770 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:23:14,571 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:23:15,572 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:23:20,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:23:20,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:23:20,331 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:23:20,686 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:23:25,686 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:23:30,687 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:23:35,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:23:35,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:23:35,331 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:23:35,721 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:23:40,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:23:44,574 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:23:46,575 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:23:50,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:23:50,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:23:50,331 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:23:51,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:23:56,691 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:24:01,692 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:24:05,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:24:05,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:24:05,331 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:24:07,679 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:24:12,680 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:24:14,576 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:24:18,578 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:24:20,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:24:20,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:24:20,331 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:24:23,710 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:24:28,711 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:24:33,712 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:24:35,327 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:24:35,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:24:35,332 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:24:38,745 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:24:43,746 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:24:44,579 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:24:48,533 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 17:24:49,369 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:24:50,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:24:50,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:24:50,332 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:24:54,684 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:24:59,685 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:25:04,686 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:25:05,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:25:05,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:25:05,332 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:25:09,706 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:25:14,582 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:25:15,584 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:25:20,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:25:20,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:25:20,332 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:25:20,769 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:25:25,769 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:25:30,770 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:25:35,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:25:35,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:25:35,332 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:25:36,743 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:25:41,744 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:25:44,585 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:25:47,586 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:25:50,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:25:50,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:25:50,332 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:25:52,750 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:25:57,751 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:26:02,752 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:26:05,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:26:05,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:26:05,332 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:26:08,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:26:13,694 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:26:14,586 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:26:19,588 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:26:20,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:26:20,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:26:20,332 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:26:24,688 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:26:29,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:26:34,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:26:35,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:26:35,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:26:35,332 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:26:39,691 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:26:44,589 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:26:45,590 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:26:50,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:26:50,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:26:50,332 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:26:50,689 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:26:55,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:27:00,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:27:05,328 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:27:05,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:27:05,333 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:27:05,706 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:27:10,707 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:27:14,591 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:27:16,592 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:27:20,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:27:20,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:27:20,333 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:27:21,720 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:27:26,721 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:27:31,722 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:27:35,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:27:35,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:27:35,333 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:27:36,734 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:27:41,734 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:27:44,594 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:27:47,596 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:27:50,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:27:50,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:27:50,333 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:27:52,736 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:27:57,737 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:28:02,738 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:28:05,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:28:05,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:28:05,333 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:28:08,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:28:13,718 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:28:14,597 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:28:19,598 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:28:20,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:28:20,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:28:20,333 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:28:24,745 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:28:29,746 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:28:34,746 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:28:35,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:28:35,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:28:35,333 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:28:39,771 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:28:44,598 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:28:45,599 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:28:50,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:28:50,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:28:50,333 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:28:50,685 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:28:55,686 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:29:00,687 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:29:05,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:29:05,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:29:05,333 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:29:05,808 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:29:10,808 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:29:14,601 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:29:16,602 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:29:20,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:29:20,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:29:20,334 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:29:21,710 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:29:26,710 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:29:31,711 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:29:35,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:29:35,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:29:35,333 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:29:36,726 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:29:41,726 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:29:44,604 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:29:47,606 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:29:50,329 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:29:50,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:29:50,334 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:29:52,759 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:29:57,760 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:30:02,761 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:30:05,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:30:05,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:30:05,334 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:30:08,731 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:30:13,731 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:30:14,606 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:30:19,607 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:30:20,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:30:20,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:30:20,334 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:30:24,694 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:30:29,695 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:30:34,695 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:30:35,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:30:35,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:30:35,334 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:30:39,746 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:30:44,608 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:30:45,609 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:30:50,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:30:50,333 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:30:50,334 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:30:50,714 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:30:55,714 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:31:00,715 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:31:05,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:31:05,334 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:31:05,334 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:31:05,775 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:31:10,775 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:31:14,609 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:31:16,611 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:31:20,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:31:20,334 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:31:20,334 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:31:21,734 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:31:26,735 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:31:31,735 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:31:35,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:31:35,334 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:31:35,334 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:31:37,703 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:31:42,704 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:31:44,611 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:31:48,613 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:31:50,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:31:50,334 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:31:50,334 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:31:53,690 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:31:58,691 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:32:03,692 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:32:05,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:32:05,334 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:32:05,335 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:32:09,685 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:32:14,614 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:32:15,615 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:32:20,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:32:20,334 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:32:20,335 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:32:20,778 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:32:25,779 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:32:30,780 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:32:35,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:32:35,335 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:32:35,335 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:32:36,666 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:32:41,667 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:32:44,616 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:32:47,617 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:32:50,330 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:32:50,334 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:32:50,335 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:32:52,694 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:32:57,694 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:33:02,695 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:33:05,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:33:05,335 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:33:05,335 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:33:07,720 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:33:12,721 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:33:14,617 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:33:18,619 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:33:20,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:33:20,335 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:33:20,335 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:33:23,795 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:33:28,795 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:33:33,796 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:33:35,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:33:35,335 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:33:35,335 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:33:39,713 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:33:44,620 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:33:45,621 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:33:50,331 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:33:50,335 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:33:50,335 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:33:50,692 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:33:55,692 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:34:00,693 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:34:05,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:34:05,336 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:34:05,336 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:34:05,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:34:10,717 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:34:14,622 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:34:16,624 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:34:20,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:34:20,336 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:34:20,336 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:34:21,671 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:34:26,672 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:34:31,673 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:34:35,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:34:35,336 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:34:35,336 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:34:36,774 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:34:41,775 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:34:44,624 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:34:47,625 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:34:50,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:34:50,336 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:34:50,336 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:34:52,703 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:34:57,704 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:35:02,705 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:35:05,332 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:35:05,336 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 17:35:05,336 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: stop_status +2023-10-31 17:35:07,737 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:35:12,738 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:35:14,625 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:35:16,865 DEBUG SenderThread:33534 [sender.py:send():380] send: exit +2023-10-31 17:35:16,865 INFO SenderThread:33534 [sender.py:send_exit():585] handling exit code: 1 +2023-10-31 17:35:16,865 INFO SenderThread:33534 [sender.py:send_exit():587] handling runtime: 9152 +2023-10-31 17:35:16,865 INFO SenderThread:33534 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 17:35:16,865 INFO SenderThread:33534 [sender.py:send_exit():593] send defer +2023-10-31 17:35:16,866 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:16,866 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 0 +2023-10-31 17:35:16,866 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:16,866 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-10-31 17:35:16,866 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 1 +2023-10-31 17:35:16,866 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:16,866 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 1 +2023-10-31 17:35:16,867 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:16,867 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-10-31 17:35:16,867 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 2 +2023-10-31 17:35:16,867 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:16,867 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 2 +2023-10-31 17:35:16,867 INFO HandlerThread:33534 [system_monitor.py:finish():203] Stopping system monitor +2023-10-31 17:35:16,868 DEBUG SystemMonitor:33534 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-10-31 17:35:16,868 INFO HandlerThread:33534 [interfaces.py:finish():202] Joined cpu monitor +2023-10-31 17:35:16,869 DEBUG SystemMonitor:33534 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-10-31 17:35:16,869 INFO HandlerThread:33534 [interfaces.py:finish():202] Joined disk monitor +2023-10-31 17:35:17,762 INFO Thread-12 :33534 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/wandb-summary.json +2023-10-31 17:35:18,635 INFO HandlerThread:33534 [interfaces.py:finish():202] Joined gpu monitor +2023-10-31 17:35:18,636 INFO HandlerThread:33534 [interfaces.py:finish():202] Joined memory monitor +2023-10-31 17:35:18,636 INFO HandlerThread:33534 [interfaces.py:finish():202] Joined network monitor +2023-10-31 17:35:18,637 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 17:35:18,637 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:35:18,637 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:18,638 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-10-31 17:35:18,638 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 3 +2023-10-31 17:35:18,639 DEBUG SenderThread:33534 [sender.py:send():380] send: stats +2023-10-31 17:35:18,639 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:18,640 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 17:35:18,640 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 3 +2023-10-31 17:35:18,641 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:18,641 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-10-31 17:35:18,641 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 4 +2023-10-31 17:35:18,641 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:18,641 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 4 +2023-10-31 17:35:18,642 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:18,642 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-10-31 17:35:18,642 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 5 +2023-10-31 17:35:18,642 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:18,642 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 5 +2023-10-31 17:35:18,643 DEBUG SenderThread:33534 [sender.py:send():380] send: summary +2023-10-31 17:35:18,643 INFO SenderThread:33534 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 17:35:18,644 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:18,644 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-10-31 17:35:18,644 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 6 +2023-10-31 17:35:18,644 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:18,644 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 6 +2023-10-31 17:35:18,645 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:18,645 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-10-31 17:35:18,645 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 7 +2023-10-31 17:35:18,645 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 17:35:18,646 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:18,646 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 7 +2023-10-31 17:35:18,646 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:18,646 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-10-31 17:35:18,762 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/wandb-summary.json +2023-10-31 17:35:18,763 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 17:35:18,873 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 17:35:19,763 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 17:35:20,883 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 8 +2023-10-31 17:35:20,883 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 17:35:20,883 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:20,884 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 8 +2023-10-31 17:35:20,884 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:20,884 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-10-31 17:35:20,884 INFO SenderThread:33534 [job_builder.py:build():281] Attempting to build job artifact +2023-10-31 17:35:20,885 INFO SenderThread:33534 [job_builder.py:_get_source_type():404] no source found +2023-10-31 17:35:20,885 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 9 +2023-10-31 17:35:20,885 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:20,886 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 9 +2023-10-31 17:35:20,886 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:20,886 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-10-31 17:35:20,886 INFO SenderThread:33534 [dir_watcher.py:finish():358] shutting down directory watcher +2023-10-31 17:35:21,764 INFO Thread-12 :33534 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 17:35:21,765 INFO SenderThread:33534 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files +2023-10-31 17:35:21,765 INFO SenderThread:33534 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log output.log +2023-10-31 17:35:21,765 INFO SenderThread:33534 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/requirements.txt requirements.txt +2023-10-31 17:35:21,766 INFO SenderThread:33534 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/conda-environment.yaml conda-environment.yaml +2023-10-31 17:35:21,773 INFO SenderThread:33534 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/wandb-summary.json wandb-summary.json +2023-10-31 17:35:21,773 INFO SenderThread:33534 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/config.yaml config.yaml +2023-10-31 17:35:21,779 INFO SenderThread:33534 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/wandb-metadata.json wandb-metadata.json +2023-10-31 17:35:21,786 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 10 +2023-10-31 17:35:21,795 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:21,801 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 10 +2023-10-31 17:35:21,803 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:21,803 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-10-31 17:35:21,804 INFO SenderThread:33534 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 17:35:22,706 INFO wandb-upload_0:33534 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/output.log +2023-10-31 17:35:22,975 INFO wandb-upload_1:33534 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/requirements.txt +2023-10-31 17:35:23,024 INFO wandb-upload_2:33534 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/conda-environment.yaml +2023-10-31 17:35:23,185 INFO wandb-upload_4:33534 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/config.yaml +2023-10-31 17:35:23,269 INFO wandb-upload_3:33534 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/files/wandb-summary.json +2023-10-31 17:35:23,470 INFO Thread-11 :33534 [sender.py:transition_state():613] send defer: 11 +2023-10-31 17:35:23,470 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:23,470 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 11 +2023-10-31 17:35:23,471 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:23,471 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-10-31 17:35:23,471 INFO SenderThread:33534 [file_pusher.py:join():181] waiting for file pusher +2023-10-31 17:35:23,471 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 12 +2023-10-31 17:35:23,472 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:23,472 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 12 +2023-10-31 17:35:23,472 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:23,472 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-10-31 17:35:23,472 INFO SenderThread:33534 [file_stream.py:finish():595] file stream finish called +2023-10-31 17:35:23,876 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 17:35:24,537 INFO SenderThread:33534 [file_stream.py:finish():599] file stream finish is done +2023-10-31 17:35:24,538 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 13 +2023-10-31 17:35:24,538 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:24,538 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 13 +2023-10-31 17:35:24,539 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:24,539 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-10-31 17:35:24,539 INFO SenderThread:33534 [sender.py:transition_state():613] send defer: 14 +2023-10-31 17:35:24,540 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: defer +2023-10-31 17:35:24,540 DEBUG SenderThread:33534 [sender.py:send():380] send: final +2023-10-31 17:35:24,540 INFO HandlerThread:33534 [handler.py:handle_request_defer():172] handle defer: 14 +2023-10-31 17:35:24,540 DEBUG SenderThread:33534 [sender.py:send():380] send: footer +2023-10-31 17:35:24,541 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: defer +2023-10-31 17:35:24,541 INFO SenderThread:33534 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-10-31 17:35:24,542 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 17:35:24,542 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 17:35:24,544 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: server_info +2023-10-31 17:35:24,544 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: get_summary +2023-10-31 17:35:24,545 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: sampled_history +2023-10-31 17:35:24,545 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: server_info +2023-10-31 17:35:24,545 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 17:35:24,551 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: job_info +2023-10-31 17:35:24,874 DEBUG SenderThread:33534 [sender.py:send_request():407] send_request: job_info +2023-10-31 17:35:24,875 INFO MainThread:33534 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-10-31 17:35:24,875 INFO MainThread:33534 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-10-31 17:35:24,875 INFO MainThread:33534 [wandb_run.py:_footer_sync_info():3558] logging synced files +2023-10-31 17:35:24,876 DEBUG HandlerThread:33534 [handler.py:handle_request():146] handle_request: shutdown +2023-10-31 17:35:24,876 INFO HandlerThread:33534 [handler.py:finish():866] shutting down handler +2023-10-31 17:35:25,552 INFO WriterThread:33534 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/run-stds1vjn.wandb +2023-10-31 17:35:25,875 INFO SenderThread:33534 [sender.py:finish():1534] shutting down sender +2023-10-31 17:35:25,875 INFO SenderThread:33534 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 17:35:25,875 INFO SenderThread:33534 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231031_150242-stds1vjn/logs/debug.log b/wandb/run-20231031_150242-stds1vjn/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1ed0d4eb9d76e684d67379f716b7e7734f723643 --- /dev/null +++ b/wandb/run-20231031_150242-stds1vjn/logs/debug.log @@ -0,0 +1,28 @@ +2023-10-31 15:02:42,536 INFO MainThread:33411 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 15:02:42,536 INFO MainThread:33411 [wandb_setup.py:_flush():76] Configure stats pid to 33411 +2023-10-31 15:02:42,536 INFO MainThread:33411 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 15:02:42,536 INFO MainThread:33411 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 15:02:42,536 INFO MainThread:33411 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 15:02:42,536 INFO MainThread:33411 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 15:02:42,536 INFO MainThread:33411 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 15:02:42,536 INFO MainThread:33411 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 15:02:42,536 INFO MainThread:33411 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/logs/debug.log +2023-10-31 15:02:42,537 INFO MainThread:33411 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_150242-stds1vjn/logs/debug-internal.log +2023-10-31 15:02:42,537 INFO MainThread:33411 [wandb_init.py:init():568] calling init triggers +2023-10-31 15:02:42,537 INFO MainThread:33411 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'Adam', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 38} +2023-10-31 15:02:42,537 INFO MainThread:33411 [wandb_init.py:init():618] starting backend +2023-10-31 15:02:42,537 INFO MainThread:33411 [wandb_init.py:init():622] setting up manager +2023-10-31 15:02:42,538 INFO MainThread:33411 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 15:02:42,540 INFO MainThread:33411 [wandb_init.py:init():628] backend started and connected +2023-10-31 15:02:42,544 INFO MainThread:33411 [wandb_init.py:init():720] updated telemetry +2023-10-31 15:02:42,544 INFO MainThread:33411 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 15:02:43,418 INFO MainThread:33411 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 15:02:43,877 INFO MainThread:33411 [wandb_run.py:_on_init():2229] got version response +2023-10-31 15:02:43,877 INFO MainThread:33411 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 15:02:50,189 INFO MainThread:33411 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 15:02:50,189 INFO MainThread:33411 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 15:02:50,189 INFO MainThread:33411 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 15:02:50,189 INFO MainThread:33411 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 15:02:50,190 INFO MainThread:33411 [wandb_init.py:init():845] run started, returning control to user process +2023-10-31 17:35:26,192 WARNING MsgRouterThr:33411 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20231031_150242-stds1vjn/run-stds1vjn.wandb b/wandb/run-20231031_150242-stds1vjn/run-stds1vjn.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a87836af6482775ac32d6884f1f66a88192799bc Binary files /dev/null and b/wandb/run-20231031_150242-stds1vjn/run-stds1vjn.wandb differ diff --git a/wandb/run-20231031_192920-ss558q6i/files/conda-environment.yaml b/wandb/run-20231031_192920-ss558q6i/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231031_192920-ss558q6i/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231031_192920-ss558q6i/files/config.yaml b/wandb/run-20231031_192920-ss558q6i/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6031abd7ffe0016d9af4b7fff537682f5ea5cf57 --- /dev/null +++ b/wandb/run-20231031_192920-ss558q6i/files/config.yaml @@ -0,0 +1,53 @@ +wandb_version: 1 + +optimizer: + desc: null + value: RMSprop +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 47 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698766160.934414 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 2 + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231031_192920-ss558q6i/files/output.log b/wandb/run-20231031_192920-ss558q6i/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ffe8419abb7fcad674b4fde316501b4ab7e97bc6 --- /dev/null +++ b/wandb/run-20231031_192920-ss558q6i/files/output.log @@ -0,0 +1,4 @@ + +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth \ No newline at end of file diff --git a/wandb/run-20231031_192920-ss558q6i/files/requirements.txt b/wandb/run-20231031_192920-ss558q6i/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231031_192920-ss558q6i/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_192920-ss558q6i/files/wandb-metadata.json b/wandb/run-20231031_192920-ss558q6i/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2db1e29a36938d21fe259d0b7fe1c187abf849bd --- /dev/null +++ b/wandb/run-20231031_192920-ss558q6i/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T15:29:22.427694", + "startedAt": "2023-10-31T15:29:20.928277", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.4367500000000004, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.286, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.782, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.593, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.533, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.3, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.398, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.399, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.196, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.195, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.697, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.19, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.818, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.493, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.419, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.297, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.398, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.401, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.9233894348145 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_192920-ss558q6i/files/wandb-summary.json b/wandb/run-20231031_192920-ss558q6i/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e42e258cb0711a9acc559239ce227ea97cc42600 --- /dev/null +++ b/wandb/run-20231031_192920-ss558q6i/files/wandb-summary.json @@ -0,0 +1 @@ +{"Train Loss": 1.5196858644485474, "Train Accuracy": 0.4781430959701538, "Validation Loss": 0.538533627986908, "Validation Accuracy": 1.3069567680358887, "_timestamp": 1698768119.855775, "_runtime": 1958.921361207962, "_step": 0, "_wandb": {"runtime": 1957}} \ No newline at end of file diff --git a/wandb/run-20231031_192920-ss558q6i/logs/debug-internal.log b/wandb/run-20231031_192920-ss558q6i/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..31720302c159f4ceae63f4f1e6b83a0e1dff450a --- /dev/null +++ b/wandb/run-20231031_192920-ss558q6i/logs/debug-internal.log @@ -0,0 +1,1024 @@ +2023-10-31 19:29:20,936 INFO StreamThr :33870 [internal.py:wandb_internal():86] W&B internal server running at pid: 33870, started at: 2023-10-31 19:29:20.935643 +2023-10-31 19:29:20,939 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status +2023-10-31 19:29:20,942 INFO WriterThread:33870 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/run-ss558q6i.wandb +2023-10-31 19:29:20,946 DEBUG SenderThread:33870 [sender.py:send():380] send: header +2023-10-31 19:29:20,946 DEBUG SenderThread:33870 [sender.py:send():380] send: run +2023-10-31 19:29:21,834 INFO SenderThread:33870 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files +2023-10-31 19:29:21,834 INFO SenderThread:33870 [sender.py:_start_run_threads():1122] run started: ss558q6i with start time 1698766160.934414 +2023-10-31 19:29:21,852 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 19:29:21,853 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: check_version +2023-10-31 19:29:22,309 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 19:29:22,325 DEBUG HandlerThread:33870 [system_info.py:__init__():32] System info init +2023-10-31 19:29:22,325 DEBUG HandlerThread:33870 [system_info.py:__init__():47] System info init done +2023-10-31 19:29:22,325 INFO HandlerThread:33870 [system_monitor.py:start():194] Starting system monitor +2023-10-31 19:29:22,325 INFO SystemMonitor:33870 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 19:29:22,325 INFO HandlerThread:33870 [system_monitor.py:probe():214] Collecting system info +2023-10-31 19:29:22,326 INFO SystemMonitor:33870 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 19:29:22,327 INFO SystemMonitor:33870 [interfaces.py:start():190] Started disk monitoring +2023-10-31 19:29:22,329 INFO SystemMonitor:33870 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 19:29:22,330 INFO SystemMonitor:33870 [interfaces.py:start():190] Started memory monitoring +2023-10-31 19:29:22,331 INFO SystemMonitor:33870 [interfaces.py:start():190] Started network monitoring +2023-10-31 19:29:22,427 DEBUG HandlerThread:33870 [system_info.py:probe():196] Probing system +2023-10-31 19:29:22,435 DEBUG HandlerThread:33870 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 19:29:22,435 DEBUG HandlerThread:33870 [system_info.py:probe():244] Probing system done +2023-10-31 19:29:22,436 DEBUG HandlerThread:33870 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T15:29:22.427694', 'startedAt': '2023-10-31T15:29:20.928277', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.4367500000000004, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.286, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.782, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.593, 'min': 1200.0, 'max': 3600.0}, {'current': 1.533, 'min': 1200.0, 'max': 3600.0}, {'current': 1.3, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.398, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.399, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.196, 'min': 1200.0, 'max': 3600.0}, {'current': 1.195, 'min': 1200.0, 'max': 3600.0}, {'current': 2.697, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.19, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.818, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.493, 'min': 1200.0, 'max': 3600.0}, {'current': 1.419, 'min': 1200.0, 'max': 3600.0}, {'current': 1.297, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.398, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.401, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9233894348145}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 19:29:22,436 INFO HandlerThread:33870 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 19:29:22,436 INFO HandlerThread:33870 [system_monitor.py:probe():227] Publishing system info +2023-10-31 19:29:22,436 DEBUG HandlerThread:33870 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 19:29:22,437 DEBUG HandlerThread:33870 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 19:29:22,437 DEBUG HandlerThread:33870 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 19:29:22,839 INFO Thread-12 :33870 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/conda-environment.yaml +2023-10-31 19:29:22,840 INFO Thread-12 :33870 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/requirements.txt +2023-10-31 19:29:28,975 DEBUG HandlerThread:33870 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 19:29:28,978 INFO HandlerThread:33870 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 19:29:28,983 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:29:28,983 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 19:29:28,984 DEBUG SenderThread:33870 [sender.py:send():380] send: files +2023-10-31 19:29:28,984 INFO SenderThread:33870 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 19:29:28,997 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:29:28,997 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:29:29,003 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:29:29,517 DEBUG SenderThread:33870 [sender.py:send():380] send: telemetry +2023-10-31 19:29:29,838 INFO Thread-12 :33870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/conda-environment.yaml +2023-10-31 19:29:29,838 INFO Thread-12 :33870 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/wandb-metadata.json +2023-10-31 19:29:30,266 INFO wandb-upload_0:33870 [upload_job.py:push():131] Uploaded file /tmp/tmpsorgkgqiwandb/tdj8fk64-wandb-metadata.json +2023-10-31 19:29:31,518 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:29:36,519 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:29:41,520 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:29:43,996 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:29:43,997 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:29:44,037 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:29:47,394 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:29:52,406 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:29:52,846 INFO Thread-12 :33870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/config.yaml +2023-10-31 19:29:57,776 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:29:58,996 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:29:58,997 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:29:59,037 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:30:03,396 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:30:08,397 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:30:13,398 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:30:13,996 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:30:13,997 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:30:14,037 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:30:19,386 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:30:22,332 DEBUG SystemMonitor:33870 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 19:30:22,335 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:30:25,336 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:30:28,996 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:30:28,997 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:30:29,037 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:30:30,413 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:30:35,413 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:30:40,414 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:30:43,996 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:30:43,997 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:30:44,037 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:30:46,387 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:30:51,388 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:30:52,338 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:30:57,340 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:30:58,996 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:30:58,997 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:30:59,037 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:31:02,390 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:31:07,391 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:31:12,392 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:31:13,996 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:31:13,997 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:31:14,037 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:31:17,408 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:31:22,341 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:31:23,343 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:31:28,344 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:31:28,997 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:31:28,997 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:31:29,037 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:31:33,409 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:31:38,410 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:31:43,411 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:31:43,997 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:31:43,997 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:31:44,037 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:31:48,423 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:31:52,344 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:31:54,346 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:31:58,997 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:31:58,998 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:31:59,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:31:59,426 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:32:04,427 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:32:09,427 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:32:13,997 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:32:13,997 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:32:14,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:32:14,437 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:32:19,438 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:32:22,346 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:32:25,348 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:32:28,997 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:32:28,998 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:32:29,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:32:30,420 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:32:35,421 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:32:40,421 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:32:43,997 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:32:43,998 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:32:44,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:32:45,428 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:32:50,429 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:32:52,349 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:32:56,351 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:32:58,997 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:32:58,998 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:32:59,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:33:01,426 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:33:06,427 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:33:11,428 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:33:13,997 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:33:13,998 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:33:14,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:33:17,416 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:33:22,352 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:33:23,353 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:33:28,354 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:33:28,997 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:33:28,998 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:33:29,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:33:31,921 INFO Thread-12 :33870 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/output.log +2023-10-31 19:33:33,922 INFO Thread-12 :33870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/output.log +2023-10-31 19:33:34,101 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:33:39,102 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:33:43,998 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:33:43,998 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:33:44,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:33:44,359 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:33:49,359 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:33:52,355 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:33:55,356 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:33:58,998 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:33:58,998 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:33:59,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:34:00,417 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:34:05,418 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:34:10,419 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:34:13,998 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:34:13,998 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:34:14,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:34:16,404 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:34:21,404 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:34:22,357 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:34:27,359 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:34:28,998 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:34:28,998 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:34:29,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:34:32,371 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:34:37,372 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:34:42,372 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:34:43,998 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:34:43,999 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:34:44,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:34:47,403 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:34:52,360 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:34:53,362 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:34:58,362 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:34:58,998 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:34:58,998 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:34:59,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:35:03,404 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:35:08,405 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:35:13,406 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:35:13,999 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:35:13,999 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:35:14,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:35:18,437 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:35:22,362 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:35:24,364 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:35:28,998 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:35:28,999 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:35:29,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:35:29,414 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:35:34,415 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:35:39,416 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:35:43,998 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:35:43,999 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:35:44,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:35:45,405 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:35:50,405 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:35:52,363 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:35:56,365 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:35:58,998 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:35:58,999 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:35:59,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:36:01,445 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:36:06,446 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:36:11,446 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:36:13,999 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:36:13,999 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:36:14,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:36:17,376 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:36:22,367 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:36:23,369 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:36:28,370 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:36:28,999 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:36:28,999 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:36:29,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:36:33,405 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:36:38,406 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:36:43,407 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:36:43,999 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:36:43,999 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:36:44,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:36:49,326 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:36:52,370 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:36:54,372 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:36:58,999 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:36:59,000 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:36:59,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:37:00,365 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:37:05,365 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:37:10,366 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:37:13,999 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:37:14,000 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:37:14,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:37:15,437 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:37:20,438 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:37:22,373 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:37:26,375 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:37:29,000 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:37:29,000 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:37:29,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:37:31,428 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:37:36,429 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:37:38,013 INFO Thread-12 :33870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/output.log +2023-10-31 19:37:42,050 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:37:43,999 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:37:44,000 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:37:44,004 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:37:47,383 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:37:52,375 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:37:53,376 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:37:58,377 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:37:58,999 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:37:59,000 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:37:59,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:38:04,357 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:38:09,358 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:38:14,000 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:38:14,000 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:38:14,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:38:14,422 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:38:19,423 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:38:22,378 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:38:25,380 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:38:29,000 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:38:29,000 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:38:29,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:38:30,425 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:38:35,425 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:38:40,426 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:38:44,000 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:38:44,001 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:38:44,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:38:45,437 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:38:50,438 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:38:52,380 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:38:56,382 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:38:59,000 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:38:59,000 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:38:59,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:39:01,440 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:39:06,440 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:39:11,441 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:39:14,000 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:39:14,001 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:39:14,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:39:17,368 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:39:22,369 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:39:22,382 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:39:27,384 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:39:29,000 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:39:29,001 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:39:29,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:39:33,355 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:39:38,355 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:39:43,356 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:39:44,000 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:39:44,001 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:39:44,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:39:49,342 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:39:52,385 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:39:54,386 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:39:59,000 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:39:59,001 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:39:59,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:40:00,357 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:40:05,358 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:40:10,359 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:40:14,000 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:40:14,001 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:40:14,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:40:15,454 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:40:20,455 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:40:22,387 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:40:26,389 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:40:29,000 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:40:29,002 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:40:29,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:40:32,345 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:40:37,346 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:40:42,347 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:40:44,001 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:40:44,001 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:40:44,041 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:40:47,380 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:40:52,381 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:40:52,390 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:40:57,391 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:40:59,001 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:40:59,001 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:40:59,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:41:03,362 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:41:08,363 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:41:13,363 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:41:14,001 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:41:14,002 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:41:14,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:41:18,400 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:41:22,392 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:41:24,393 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:41:29,001 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:41:29,002 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:41:29,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:41:29,434 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:41:34,435 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:41:39,435 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:41:42,102 INFO Thread-12 :33870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/output.log +2023-10-31 19:41:44,001 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:41:44,002 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:41:44,005 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:41:45,384 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:41:50,384 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:41:52,393 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:41:55,394 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:41:59,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:41:59,003 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:41:59,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:42:00,417 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:42:05,418 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:42:10,419 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:42:14,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:42:14,002 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:42:14,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:42:16,397 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:42:21,398 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:42:22,396 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:42:27,397 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:42:29,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:42:29,002 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:42:29,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:42:32,408 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:42:37,408 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:42:42,409 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:42:44,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:42:44,002 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:42:44,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:42:48,364 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:42:52,397 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:42:53,399 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:42:58,399 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:42:59,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:42:59,002 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:42:59,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:43:03,406 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:43:08,407 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:43:13,407 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:43:14,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:43:14,002 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:43:14,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:43:18,432 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:43:22,399 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:43:24,401 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:43:29,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:43:29,003 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:43:29,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:43:29,420 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:43:34,421 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:43:39,422 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:43:44,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:43:44,003 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:43:44,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:43:44,427 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:43:49,428 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:43:52,402 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:43:55,405 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:43:59,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:43:59,003 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:43:59,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:44:00,444 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:44:05,445 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:44:10,445 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:44:14,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:44:14,003 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:44:14,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:44:16,401 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:44:21,402 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:44:22,406 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:44:26,408 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:44:29,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:44:29,003 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:44:29,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:44:32,343 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:44:37,344 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:44:42,345 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:44:44,002 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:44:44,003 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:44:44,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:44:47,396 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:44:52,397 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:44:52,407 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:44:57,409 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:44:59,003 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:44:59,004 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:44:59,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:45:03,402 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:45:08,402 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:45:13,403 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:45:14,003 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:45:14,003 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:45:14,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:45:19,390 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:45:22,409 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:45:24,411 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:45:29,003 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:45:29,003 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:45:29,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:45:30,405 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:45:35,405 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:45:40,406 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:45:44,003 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:45:44,004 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:45:44,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:45:46,381 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:45:51,381 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:45:52,412 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:45:56,413 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:45:59,003 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:45:59,004 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:45:59,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:46:01,421 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:46:06,421 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:46:11,422 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:46:14,003 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:46:14,004 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:46:14,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:46:16,425 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:46:21,425 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:46:22,414 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:46:27,415 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:46:29,003 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:46:29,004 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:46:29,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:46:33,377 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:46:38,377 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:46:43,379 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:46:44,003 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:46:44,004 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:46:44,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:46:48,402 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:46:52,416 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:46:53,417 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:46:58,418 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:46:59,004 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:46:59,004 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:46:59,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:47:04,371 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:47:09,372 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:47:14,004 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:47:14,004 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:47:14,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:47:14,394 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:47:19,395 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:47:22,419 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:47:24,420 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:47:29,004 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:47:29,004 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:47:29,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:47:29,446 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:47:34,446 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:47:39,447 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:47:44,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:47:44,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:47:44,006 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:47:45,409 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:47:50,410 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:47:52,420 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:47:55,421 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:47:59,004 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:47:59,004 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:47:59,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:48:01,412 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:48:06,413 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:48:11,413 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:48:14,004 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:48:14,004 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:48:14,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:48:16,422 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:48:21,423 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:48:22,423 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:48:26,425 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:48:29,004 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:48:29,005 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:48:29,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:48:31,434 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:48:36,435 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:48:41,435 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:48:44,005 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:48:44,005 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:48:44,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:48:47,383 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:48:52,383 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:48:52,426 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:48:57,427 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:48:59,004 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:48:59,005 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:48:59,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:49:03,406 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:49:08,407 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:49:13,408 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:49:14,004 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:49:14,005 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:49:14,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:49:18,423 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:49:22,428 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:49:23,429 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:49:28,430 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:49:29,005 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:49:29,005 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:49:29,045 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:49:33,471 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:49:38,472 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:49:43,472 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:49:44,005 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:49:44,005 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:49:44,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:49:49,391 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:49:52,430 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:49:54,431 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:49:59,005 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:49:59,005 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:49:59,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:50:00,413 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:50:05,414 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:50:10,414 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:50:14,005 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:50:14,006 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:50:14,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:50:15,450 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:50:20,450 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:50:22,432 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:50:26,434 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:50:29,005 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:50:29,006 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:50:29,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:50:32,388 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:50:37,389 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:50:42,389 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:50:44,005 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:50:44,006 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:50:44,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:50:48,371 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:50:52,434 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:50:53,435 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:50:58,436 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:50:59,005 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:50:59,006 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:50:59,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:51:04,399 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:51:09,399 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:51:14,005 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:51:14,006 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:51:14,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:51:14,410 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:51:19,411 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:51:22,436 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:51:24,438 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:51:29,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:51:29,006 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:51:29,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:51:30,386 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:51:35,387 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:51:40,388 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:51:44,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:51:44,006 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:51:44,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:51:46,365 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:51:51,366 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:51:52,438 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:51:56,440 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:51:59,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:51:59,006 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:51:59,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:52:02,406 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:52:07,407 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:52:12,408 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:52:14,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:52:14,006 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:52:14,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:52:17,411 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:52:22,412 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:52:22,440 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:52:27,442 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:52:29,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:52:29,006 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:52:29,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:52:33,371 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:52:38,372 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:52:43,373 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:52:44,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:52:44,007 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:52:44,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:52:48,492 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:52:52,442 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:52:54,444 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:52:59,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:52:59,007 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:52:59,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:53:00,404 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:53:05,405 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:53:10,405 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:53:14,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:53:14,007 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:53:14,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:53:15,429 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:53:20,430 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:53:22,444 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:53:25,446 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:53:29,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:53:29,007 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:53:29,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:53:30,463 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:53:35,464 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:53:40,464 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:53:44,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:53:44,007 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:53:44,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:53:46,364 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:53:51,365 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:53:52,446 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:53:56,447 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:53:59,006 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:53:59,007 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:53:59,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:54:02,380 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:54:07,381 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:54:12,382 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:54:14,007 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:54:14,007 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:54:14,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:54:17,456 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:54:22,449 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:54:23,450 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:54:28,451 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:54:29,007 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:54:29,007 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:54:29,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:54:34,428 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:54:39,429 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:54:44,007 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:54:44,008 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:54:44,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:54:45,403 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:54:50,404 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:54:52,451 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:54:55,453 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:54:59,007 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:54:59,008 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:54:59,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:55:01,367 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:55:06,368 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:55:11,369 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:55:14,007 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:55:14,008 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:55:14,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:55:16,454 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:55:21,454 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:55:22,454 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:55:26,456 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:55:29,007 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:55:29,008 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:55:29,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:55:32,410 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:55:37,411 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:55:42,412 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:55:44,007 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:55:44,008 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:55:44,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:55:47,432 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:55:52,433 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:55:52,456 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:55:57,458 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:55:59,007 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:55:59,008 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:55:59,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:56:03,382 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:56:08,383 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:56:13,384 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:56:14,008 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:56:14,008 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:56:14,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:56:18,406 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:56:22,458 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:56:23,460 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:56:28,460 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:56:29,008 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:56:29,008 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:56:29,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:56:34,421 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:56:39,421 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:56:44,008 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:56:44,008 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:56:44,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:56:45,398 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:56:50,399 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:56:52,461 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:56:55,462 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:56:59,008 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:56:59,009 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:56:59,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:57:01,443 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:57:06,443 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:57:11,444 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:57:14,008 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:57:14,009 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:57:14,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:57:17,376 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:57:22,376 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:57:22,463 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:57:27,465 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:57:29,009 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:57:29,009 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:57:29,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:57:33,357 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:57:38,358 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:57:43,359 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:57:44,008 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:57:44,009 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:57:44,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:57:48,431 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:57:52,465 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:57:53,467 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:57:58,467 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:57:59,009 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:57:59,009 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:57:59,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:58:04,399 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:58:09,400 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:58:14,009 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:58:14,009 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:58:14,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:58:14,406 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:58:19,406 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:58:22,467 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:58:24,468 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:58:29,009 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:58:29,009 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:58:29,049 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:58:30,381 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:58:35,382 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:58:40,383 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:58:44,009 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:58:44,009 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:58:44,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:58:45,458 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:58:50,458 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:58:52,470 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:58:55,471 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:58:59,009 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:58:59,010 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:58:59,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:59:01,452 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:59:06,452 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:59:11,453 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:59:14,009 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:59:14,009 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:59:14,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:59:17,436 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:59:22,437 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:59:22,472 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:59:27,473 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:59:29,009 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:59:29,010 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:59:29,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:59:33,352 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:59:38,353 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:59:43,354 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:59:44,009 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:59:44,010 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:59:44,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 19:59:48,406 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:59:52,473 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 19:59:53,474 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:59:58,475 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 19:59:59,009 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 19:59:59,010 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 19:59:59,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:00:03,486 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:00:08,487 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:00:13,488 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:00:14,009 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:00:14,010 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:00:14,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:00:19,367 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:00:22,475 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:00:24,477 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:00:29,010 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:00:29,010 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:00:29,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:00:30,424 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:00:35,425 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:00:40,425 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:00:44,010 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:00:44,010 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:00:44,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:00:45,460 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:00:50,460 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:00:52,478 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:00:55,479 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:00:59,010 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:00:59,010 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:00:59,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:01:01,410 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:01:06,411 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:01:11,411 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:01:14,010 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:01:14,010 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:01:14,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:01:16,423 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:01:21,423 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:01:22,479 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:01:26,482 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:01:29,010 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:01:29,010 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:01:29,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:01:32,426 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:01:37,426 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:01:42,427 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:01:44,010 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:01:44,011 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:01:44,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:01:48,360 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:01:52,481 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:01:53,483 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:01:58,483 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:01:59,010 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:01:59,011 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:01:59,053 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:01:59,856 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: partial_history +2023-10-31 20:01:59,859 DEBUG SenderThread:33870 [sender.py:send():380] send: history +2023-10-31 20:01:59,859 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: summary_record +2023-10-31 20:01:59,860 INFO SenderThread:33870 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 20:01:59,897 DEBUG SenderThread:33870 [sender.py:send():380] send: telemetry +2023-10-31 20:01:59,898 DEBUG SenderThread:33870 [sender.py:send():380] send: exit +2023-10-31 20:01:59,898 INFO SenderThread:33870 [sender.py:send_exit():585] handling exit code: 0 +2023-10-31 20:01:59,898 INFO SenderThread:33870 [sender.py:send_exit():587] handling runtime: 1957 +2023-10-31 20:01:59,898 INFO SenderThread:33870 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 20:01:59,899 INFO SenderThread:33870 [sender.py:send_exit():593] send defer +2023-10-31 20:01:59,899 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:01:59,899 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 0 +2023-10-31 20:01:59,899 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:01:59,899 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-10-31 20:01:59,900 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 1 +2023-10-31 20:01:59,900 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:01:59,900 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 1 +2023-10-31 20:01:59,900 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:01:59,900 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-10-31 20:01:59,900 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 2 +2023-10-31 20:01:59,901 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:01:59,901 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 2 +2023-10-31 20:01:59,901 INFO HandlerThread:33870 [system_monitor.py:finish():203] Stopping system monitor +2023-10-31 20:01:59,901 DEBUG SystemMonitor:33870 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-10-31 20:01:59,902 INFO HandlerThread:33870 [interfaces.py:finish():202] Joined cpu monitor +2023-10-31 20:01:59,902 DEBUG SystemMonitor:33870 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-10-31 20:01:59,902 INFO HandlerThread:33870 [interfaces.py:finish():202] Joined disk monitor +2023-10-31 20:02:00,551 INFO Thread-12 :33870 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/wandb-summary.json +2023-10-31 20:02:01,647 INFO HandlerThread:33870 [interfaces.py:finish():202] Joined gpu monitor +2023-10-31 20:02:01,647 INFO HandlerThread:33870 [interfaces.py:finish():202] Joined memory monitor +2023-10-31 20:02:01,647 INFO HandlerThread:33870 [interfaces.py:finish():202] Joined network monitor +2023-10-31 20:02:01,648 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:02:01,648 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:01,649 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-10-31 20:02:01,649 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 3 +2023-10-31 20:02:01,649 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:02:01,650 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:02:01,651 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:02:01,651 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 3 +2023-10-31 20:02:01,652 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:01,652 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-10-31 20:02:01,652 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 4 +2023-10-31 20:02:01,652 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:02:01,652 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 4 +2023-10-31 20:02:01,653 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:01,653 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-10-31 20:02:01,653 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 5 +2023-10-31 20:02:01,653 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:02:01,653 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 5 +2023-10-31 20:02:01,654 DEBUG SenderThread:33870 [sender.py:send():380] send: summary +2023-10-31 20:02:01,655 INFO SenderThread:33870 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 20:02:01,655 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:01,655 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-10-31 20:02:01,655 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 6 +2023-10-31 20:02:01,656 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:02:01,656 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 6 +2023-10-31 20:02:01,656 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:01,656 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-10-31 20:02:01,662 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:02:01,858 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:02:02,097 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 7 +2023-10-31 20:02:02,097 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:02:02,097 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:02:02,098 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 7 +2023-10-31 20:02:02,098 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:02,099 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-10-31 20:02:02,552 INFO Thread-12 :33870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/wandb-summary.json +2023-10-31 20:02:02,552 INFO Thread-12 :33870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/config.yaml +2023-10-31 20:02:02,858 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:02:02,896 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 8 +2023-10-31 20:02:02,897 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:02:02,897 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:02:02,897 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 8 +2023-10-31 20:02:02,898 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:02,898 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-10-31 20:02:02,898 INFO SenderThread:33870 [job_builder.py:build():281] Attempting to build job artifact +2023-10-31 20:02:02,899 INFO SenderThread:33870 [job_builder.py:_get_source_type():404] no source found +2023-10-31 20:02:02,899 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 9 +2023-10-31 20:02:02,899 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:02:02,899 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 9 +2023-10-31 20:02:02,900 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:02,900 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-10-31 20:02:02,900 INFO SenderThread:33870 [dir_watcher.py:finish():358] shutting down directory watcher +2023-10-31 20:02:03,552 INFO Thread-12 :33870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/output.log +2023-10-31 20:02:03,553 INFO SenderThread:33870 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files +2023-10-31 20:02:03,554 INFO SenderThread:33870 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/output.log output.log +2023-10-31 20:02:03,554 INFO SenderThread:33870 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/requirements.txt requirements.txt +2023-10-31 20:02:03,561 INFO SenderThread:33870 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/conda-environment.yaml conda-environment.yaml +2023-10-31 20:02:03,569 INFO SenderThread:33870 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/wandb-summary.json wandb-summary.json +2023-10-31 20:02:03,570 INFO SenderThread:33870 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/config.yaml config.yaml +2023-10-31 20:02:03,577 INFO SenderThread:33870 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/wandb-metadata.json wandb-metadata.json +2023-10-31 20:02:03,582 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 10 +2023-10-31 20:02:03,585 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:02:03,590 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 10 +2023-10-31 20:02:03,592 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:03,593 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-10-31 20:02:03,593 INFO SenderThread:33870 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 20:02:03,859 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:02:03,860 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:02:04,484 INFO wandb-upload_0:33870 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/output.log +2023-10-31 20:02:04,804 INFO wandb-upload_4:33870 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/config.yaml +2023-10-31 20:02:04,819 INFO wandb-upload_1:33870 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/requirements.txt +2023-10-31 20:02:04,821 INFO wandb-upload_2:33870 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/conda-environment.yaml +2023-10-31 20:02:04,845 INFO wandb-upload_3:33870 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/files/wandb-summary.json +2023-10-31 20:02:04,860 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:02:04,861 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:02:05,046 INFO Thread-11 :33870 [sender.py:transition_state():613] send defer: 11 +2023-10-31 20:02:05,046 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:02:05,046 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 11 +2023-10-31 20:02:05,047 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:05,047 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-10-31 20:02:05,047 INFO SenderThread:33870 [file_pusher.py:join():181] waiting for file pusher +2023-10-31 20:02:05,047 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 12 +2023-10-31 20:02:05,048 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:02:05,048 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 12 +2023-10-31 20:02:05,048 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:05,048 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-10-31 20:02:05,048 INFO SenderThread:33870 [file_stream.py:finish():595] file stream finish called +2023-10-31 20:02:05,861 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:02:06,241 INFO SenderThread:33870 [file_stream.py:finish():599] file stream finish is done +2023-10-31 20:02:06,242 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 13 +2023-10-31 20:02:06,242 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:02:06,242 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:02:06,242 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 13 +2023-10-31 20:02:06,243 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:06,243 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-10-31 20:02:06,243 INFO SenderThread:33870 [sender.py:transition_state():613] send defer: 14 +2023-10-31 20:02:06,244 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:02:06,244 DEBUG SenderThread:33870 [sender.py:send():380] send: final +2023-10-31 20:02:06,244 INFO HandlerThread:33870 [handler.py:handle_request_defer():172] handle defer: 14 +2023-10-31 20:02:06,245 DEBUG SenderThread:33870 [sender.py:send():380] send: footer +2023-10-31 20:02:06,245 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: defer +2023-10-31 20:02:06,245 INFO SenderThread:33870 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-10-31 20:02:06,246 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:02:06,247 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:02:06,248 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:02:06,249 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: server_info +2023-10-31 20:02:06,249 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: get_summary +2023-10-31 20:02:06,249 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: sampled_history +2023-10-31 20:02:06,250 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: server_info +2023-10-31 20:02:06,250 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: job_info +2023-10-31 20:02:06,584 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: job_info +2023-10-31 20:02:06,585 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: shutdown +2023-10-31 20:02:06,586 INFO HandlerThread:33870 [handler.py:finish():866] shutting down handler +2023-10-31 20:02:07,256 INFO WriterThread:33870 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/run-ss558q6i.wandb +2023-10-31 20:02:07,584 INFO SenderThread:33870 [sender.py:finish():1534] shutting down sender +2023-10-31 20:02:07,585 INFO SenderThread:33870 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 20:02:07,585 INFO SenderThread:33870 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231031_192920-ss558q6i/logs/debug.log b/wandb/run-20231031_192920-ss558q6i/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..88da9bc63bb0e9cf3b11f7889cf33a323ff7cab3 --- /dev/null +++ b/wandb/run-20231031_192920-ss558q6i/logs/debug.log @@ -0,0 +1,34 @@ +2023-10-31 19:29:20,930 INFO MainThread:33719 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 19:29:20,930 INFO MainThread:33719 [wandb_setup.py:_flush():76] Configure stats pid to 33719 +2023-10-31 19:29:20,930 INFO MainThread:33719 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 19:29:20,930 INFO MainThread:33719 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 19:29:20,930 INFO MainThread:33719 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 19:29:20,930 INFO MainThread:33719 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 19:29:20,930 INFO MainThread:33719 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 19:29:20,930 INFO MainThread:33719 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 19:29:20,930 INFO MainThread:33719 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/logs/debug.log +2023-10-31 19:29:20,930 INFO MainThread:33719 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_192920-ss558q6i/logs/debug-internal.log +2023-10-31 19:29:20,930 INFO MainThread:33719 [wandb_init.py:init():568] calling init triggers +2023-10-31 19:29:20,931 INFO MainThread:33719 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'RMSprop', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 47} +2023-10-31 19:29:20,931 INFO MainThread:33719 [wandb_init.py:init():618] starting backend +2023-10-31 19:29:20,931 INFO MainThread:33719 [wandb_init.py:init():622] setting up manager +2023-10-31 19:29:20,932 INFO MainThread:33719 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 19:29:20,934 INFO MainThread:33719 [wandb_init.py:init():628] backend started and connected +2023-10-31 19:29:20,938 INFO MainThread:33719 [wandb_init.py:init():720] updated telemetry +2023-10-31 19:29:20,938 INFO MainThread:33719 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 19:29:21,851 INFO MainThread:33719 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 19:29:22,299 INFO MainThread:33719 [wandb_run.py:_on_init():2229] got version response +2023-10-31 19:29:22,299 INFO MainThread:33719 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 19:29:28,996 INFO MainThread:33719 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 19:29:28,997 INFO MainThread:33719 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 19:29:28,997 INFO MainThread:33719 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 19:29:28,997 INFO MainThread:33719 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 19:29:28,998 INFO MainThread:33719 [wandb_init.py:init():845] run started, returning control to user process +2023-10-31 20:01:59,856 INFO MainThread:33719 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/ss558q6i +2023-10-31 20:01:59,856 INFO MainThread:33719 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0 +2023-10-31 20:01:59,856 INFO MainThread:33719 [wandb_run.py:_restore():2151] restore +2023-10-31 20:01:59,856 INFO MainThread:33719 [wandb_run.py:_restore():2157] restore done +2023-10-31 20:02:07,588 INFO MainThread:33719 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-10-31 20:02:07,588 INFO MainThread:33719 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-10-31 20:02:07,596 INFO MainThread:33719 [wandb_run.py:_footer_sync_info():3558] logging synced files diff --git a/wandb/run-20231031_192920-ss558q6i/run-ss558q6i.wandb b/wandb/run-20231031_192920-ss558q6i/run-ss558q6i.wandb new file mode 100644 index 0000000000000000000000000000000000000000..abe2b8aec9b15226ca6bb5d0cc2c23c8b21e79fd Binary files /dev/null and b/wandb/run-20231031_192920-ss558q6i/run-ss558q6i.wandb differ diff --git a/wandb/run-20231031_200208-lw42gge0/files/conda-environment.yaml b/wandb/run-20231031_200208-lw42gge0/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231031_200208-lw42gge0/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231031_200208-lw42gge0/files/config.yaml b/wandb/run-20231031_200208-lw42gge0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..958ff8c2ea04027a06ff91216bd186f2f0fd0004 --- /dev/null +++ b/wandb/run-20231031_200208-lw42gge0/files/config.yaml @@ -0,0 +1,52 @@ +wandb_version: 1 + +optimizer: + desc: null + value: SGD +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 45 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698768128.10801 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231031_200208-lw42gge0/files/output.log b/wandb/run-20231031_200208-lw42gge0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/wandb/run-20231031_200208-lw42gge0/files/output.log @@ -0,0 +1 @@ + diff --git a/wandb/run-20231031_200208-lw42gge0/files/requirements.txt b/wandb/run-20231031_200208-lw42gge0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231031_200208-lw42gge0/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_200208-lw42gge0/files/wandb-metadata.json b/wandb/run-20231031_200208-lw42gge0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..062aebde0807ee5693ce49f9b1eb815f474b5211 --- /dev/null +++ b/wandb/run-20231031_200208-lw42gge0/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T16:02:10.017803", + "startedAt": "2023-10-31T16:02:08.103590", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.4666249999999998, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.4, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.428, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.202, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.424, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.495, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.399, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.304, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.204, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.207, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.398, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.702, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.203, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.497, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.208, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.448, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.385, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.211, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.306, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.206, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.9234085083008 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_200208-lw42gge0/logs/debug-internal.log b/wandb/run-20231031_200208-lw42gge0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1a691bde906a8dc7e129e3a11873fb6737802680 --- /dev/null +++ b/wandb/run-20231031_200208-lw42gge0/logs/debug-internal.log @@ -0,0 +1,195 @@ +2023-10-31 20:02:08,590 INFO StreamThr :33870 [internal.py:wandb_internal():86] W&B internal server running at pid: 33870, started at: 2023-10-31 20:02:08.589249 +2023-10-31 20:02:08,592 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status +2023-10-31 20:02:08,594 INFO WriterThread:33870 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_200208-lw42gge0/run-lw42gge0.wandb +2023-10-31 20:02:08,597 DEBUG SenderThread:33870 [sender.py:send():380] send: header +2023-10-31 20:02:08,597 DEBUG SenderThread:33870 [sender.py:send():380] send: run +2023-10-31 20:02:09,538 INFO SenderThread:33870 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_200208-lw42gge0/files +2023-10-31 20:02:09,538 INFO SenderThread:33870 [sender.py:_start_run_threads():1122] run started: lw42gge0 with start time 1698768128.10801 +2023-10-31 20:02:09,559 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 20:02:09,560 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: check_version +2023-10-31 20:02:09,951 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 20:02:09,952 DEBUG HandlerThread:33870 [system_info.py:__init__():32] System info init +2023-10-31 20:02:09,952 DEBUG HandlerThread:33870 [system_info.py:__init__():47] System info init done +2023-10-31 20:02:09,953 INFO HandlerThread:33870 [system_monitor.py:start():194] Starting system monitor +2023-10-31 20:02:09,953 INFO SystemMonitor:33870 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 20:02:09,953 INFO HandlerThread:33870 [system_monitor.py:probe():214] Collecting system info +2023-10-31 20:02:09,954 INFO SystemMonitor:33870 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 20:02:09,955 INFO SystemMonitor:33870 [interfaces.py:start():190] Started disk monitoring +2023-10-31 20:02:09,955 INFO SystemMonitor:33870 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 20:02:09,956 INFO SystemMonitor:33870 [interfaces.py:start():190] Started memory monitoring +2023-10-31 20:02:09,958 INFO SystemMonitor:33870 [interfaces.py:start():190] Started network monitoring +2023-10-31 20:02:10,017 DEBUG HandlerThread:33870 [system_info.py:probe():196] Probing system +2023-10-31 20:02:10,018 DEBUG HandlerThread:33870 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 20:02:10,018 DEBUG HandlerThread:33870 [system_info.py:probe():244] Probing system done +2023-10-31 20:02:10,018 DEBUG HandlerThread:33870 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T16:02:10.017803', 'startedAt': '2023-10-31T16:02:08.103590', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.4666249999999998, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.4, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 2.428, 'min': 1200.0, 'max': 3600.0}, {'current': 1.202, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.424, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.495, 'min': 1200.0, 'max': 3600.0}, {'current': 1.399, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.304, 'min': 1200.0, 'max': 3600.0}, {'current': 1.204, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.207, 'min': 1200.0, 'max': 3600.0}, {'current': 1.398, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.702, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.203, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.497, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.208, 'min': 1200.0, 'max': 3600.0}, {'current': 1.448, 'min': 1200.0, 'max': 3600.0}, {'current': 1.385, 'min': 1200.0, 'max': 3600.0}, {'current': 2.211, 'min': 1200.0, 'max': 3600.0}, {'current': 1.306, 'min': 1200.0, 'max': 3600.0}, {'current': 1.206, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9234085083008}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 20:02:10,018 INFO HandlerThread:33870 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 20:02:10,018 INFO HandlerThread:33870 [system_monitor.py:probe():227] Publishing system info +2023-10-31 20:02:10,018 DEBUG HandlerThread:33870 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 20:02:10,019 DEBUG HandlerThread:33870 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 20:02:10,019 DEBUG HandlerThread:33870 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 20:02:10,542 INFO Thread-25 :33870 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_200208-lw42gge0/files/conda-environment.yaml +2023-10-31 20:02:10,543 INFO Thread-25 :33870 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_200208-lw42gge0/files/requirements.txt +2023-10-31 20:02:16,532 DEBUG HandlerThread:33870 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 20:02:16,534 INFO HandlerThread:33870 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 20:02:16,535 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:02:16,535 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 20:02:16,536 DEBUG SenderThread:33870 [sender.py:send():380] send: files +2023-10-31 20:02:16,537 INFO SenderThread:33870 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 20:02:16,547 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:02:16,548 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:02:16,550 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:02:16,556 INFO Thread-25 :33870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_200208-lw42gge0/files/conda-environment.yaml +2023-10-31 20:02:16,557 INFO Thread-25 :33870 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_200208-lw42gge0/files/wandb-metadata.json +2023-10-31 20:02:17,032 DEBUG SenderThread:33870 [sender.py:send():380] send: telemetry +2023-10-31 20:02:17,786 INFO wandb-upload_0:33870 [upload_job.py:push():131] Uploaded file /tmp/tmpujiufwrywandb/89o92jth-wandb-metadata.json +2023-10-31 20:02:19,033 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:02:24,034 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:02:29,034 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:02:31,544 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:02:31,545 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:02:31,585 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:02:34,967 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:02:39,978 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:02:40,565 INFO Thread-25 :33870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_200208-lw42gge0/files/config.yaml +2023-10-31 20:02:45,463 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:02:46,544 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:02:46,545 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:02:46,585 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:02:50,970 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:02:55,971 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:03:00,971 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:03:01,544 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:03:01,545 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:03:01,585 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:03:06,935 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:03:09,958 DEBUG SystemMonitor:33870 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 20:03:09,961 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:03:11,962 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:03:16,544 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:03:16,545 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:03:16,585 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:03:17,953 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:03:22,954 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:03:27,954 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:03:31,544 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:03:31,545 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:03:31,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:03:33,905 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:03:38,906 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:03:39,964 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:03:43,966 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:03:46,545 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:03:46,545 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:03:46,585 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:03:48,972 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:03:53,973 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:03:58,974 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:04:01,545 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:04:01,546 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:04:01,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:04:04,953 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:04:09,954 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:04:09,966 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:04:14,967 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:04:16,545 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:04:16,546 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:04:16,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:04:20,941 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:04:25,942 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:04:30,943 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:04:31,546 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:04:31,547 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:04:31,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:04:35,961 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:04:39,969 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:04:40,970 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:04:45,971 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:04:46,545 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:04:46,546 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:04:46,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:04:51,942 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:04:56,943 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:05:01,546 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:05:01,546 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:05:01,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:05:01,957 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:05:06,957 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:05:09,972 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:05:11,973 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:05:16,546 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:05:16,546 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:05:16,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:05:17,908 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:05:22,908 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:05:27,909 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:05:31,545 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:05:31,546 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:05:31,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:05:32,933 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:05:37,933 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:05:39,974 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:05:42,975 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:05:46,546 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:05:46,546 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:05:46,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:05:48,939 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:05:53,940 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:05:58,941 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:06:01,546 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:06:01,547 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:06:01,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:06:04,921 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:06:09,921 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:06:09,976 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:06:14,978 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:06:16,546 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:06:16,547 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:06:16,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:06:20,941 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:06:25,645 INFO Thread-25 :33870 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_200208-lw42gge0/files/output.log +2023-10-31 20:06:26,273 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:06:27,646 INFO Thread-25 :33870 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_200208-lw42gge0/files/output.log +2023-10-31 20:06:31,274 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:06:31,547 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:06:31,547 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:06:31,548 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:06:36,920 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:06:39,979 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:06:41,980 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:06:46,546 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:06:46,547 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:06:46,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:06:47,904 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:06:52,904 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:06:57,905 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:07:01,546 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:07:01,547 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:07:01,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:07:02,942 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:07:07,943 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:07:09,981 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:07:12,983 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:07:16,547 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:07:16,548 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:07:16,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:07:18,978 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:07:23,979 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:07:28,980 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:07:31,547 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:07:31,547 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:07:31,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:07:34,964 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:07:39,964 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:07:39,983 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:07:44,985 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:07:46,547 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:07:46,547 DEBUG SenderThread:33870 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:07:46,589 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:07:50,934 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:07:55,935 DEBUG HandlerThread:33870 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:07:58,075 INFO memory :33870 [interfaces.py:monitor():140] Process proc.memory.rssMB has exited. +2023-10-31 20:07:58,075 DEBUG SystemMonitor:33870 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-10-31 20:07:58,076 DEBUG SystemMonitor:33870 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-10-31 20:07:58,079 DEBUG SenderThread:33870 [sender.py:send():380] send: stats +2023-10-31 20:07:59,014 INFO MainThread:33870 [internal.py:handle_exit():76] Internal process exited +2023-10-31 20:07:59,014 INFO MainThread:33870 [internal.py:handle_exit():76] Internal process exited diff --git a/wandb/run-20231031_200208-lw42gge0/logs/debug.log b/wandb/run-20231031_200208-lw42gge0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9846021323c8c72bc38a62c41c40a7c4d425b59c --- /dev/null +++ b/wandb/run-20231031_200208-lw42gge0/logs/debug.log @@ -0,0 +1,27 @@ +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_setup.py:_flush():76] Configure stats pid to 33719 +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_200208-lw42gge0/logs/debug.log +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_200208-lw42gge0/logs/debug-internal.log +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_init.py:init():568] calling init triggers +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'SGD', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 45} +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_init.py:init():618] starting backend +2023-10-31 20:02:08,105 INFO MainThread:33719 [wandb_init.py:init():622] setting up manager +2023-10-31 20:02:08,107 INFO MainThread:33719 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 20:02:08,107 INFO MainThread:33719 [wandb_init.py:init():628] backend started and connected +2023-10-31 20:02:08,110 INFO MainThread:33719 [wandb_init.py:init():720] updated telemetry +2023-10-31 20:02:08,110 INFO MainThread:33719 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 20:02:09,559 INFO MainThread:33719 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 20:02:09,938 INFO MainThread:33719 [wandb_run.py:_on_init():2229] got version response +2023-10-31 20:02:09,938 INFO MainThread:33719 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 20:02:16,544 INFO MainThread:33719 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 20:02:16,545 INFO MainThread:33719 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 20:02:16,546 INFO MainThread:33719 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 20:02:16,546 INFO MainThread:33719 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 20:02:16,547 INFO MainThread:33719 [wandb_init.py:init():845] run started, returning control to user process diff --git a/wandb/run-20231031_200208-lw42gge0/run-lw42gge0.wandb b/wandb/run-20231031_200208-lw42gge0/run-lw42gge0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..bae4091af96573c9d1df001a53c2af32d60e2303 Binary files /dev/null and b/wandb/run-20231031_200208-lw42gge0/run-lw42gge0.wandb differ diff --git a/wandb/run-20231031_200845-8vkss7as/files/conda-environment.yaml b/wandb/run-20231031_200845-8vkss7as/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231031_200845-8vkss7as/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231031_200845-8vkss7as/files/config.yaml b/wandb/run-20231031_200845-8vkss7as/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1af55e480ae2dd08e4faf96743901257ed0651f --- /dev/null +++ b/wandb/run-20231031_200845-8vkss7as/files/config.yaml @@ -0,0 +1,53 @@ +wandb_version: 1 + +optimizer: + desc: null + value: SGD +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 31 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698768525.067805 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 2 + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231031_200845-8vkss7as/files/output.log b/wandb/run-20231031_200845-8vkss7as/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ffe8419abb7fcad674b4fde316501b4ab7e97bc6 --- /dev/null +++ b/wandb/run-20231031_200845-8vkss7as/files/output.log @@ -0,0 +1,4 @@ + +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth \ No newline at end of file diff --git a/wandb/run-20231031_200845-8vkss7as/files/requirements.txt b/wandb/run-20231031_200845-8vkss7as/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231031_200845-8vkss7as/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_200845-8vkss7as/files/wandb-metadata.json b/wandb/run-20231031_200845-8vkss7as/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e032f1412a920494756df10982ed66eb9eda43a9 --- /dev/null +++ b/wandb/run-20231031_200845-8vkss7as/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T16:08:46.507382", + "startedAt": "2023-10-31T16:08:45.061072", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.5015999999999998, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.318, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.295, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.205, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.615, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.497, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.295, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.202, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.203, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.42, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.207, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.579, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.201, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.206, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.693, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.425, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.605, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.695, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.207, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.9234428405762 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_200845-8vkss7as/files/wandb-summary.json b/wandb/run-20231031_200845-8vkss7as/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5f189ebb76c326112e5863a20d9ae49c1bb22bdf --- /dev/null +++ b/wandb/run-20231031_200845-8vkss7as/files/wandb-summary.json @@ -0,0 +1 @@ +{"Train Loss": 1.5229237079620361, "Train Accuracy": 0.4761870801448822, "Validation Loss": 1.3105363845825195, "Validation Accuracy": 0.5369605422019958, "_timestamp": 1698771211.5781732, "_runtime": 2686.5103681087494, "_step": 0, "_wandb": {"runtime": 2685}} \ No newline at end of file diff --git a/wandb/run-20231031_200845-8vkss7as/logs/debug-internal.log b/wandb/run-20231031_200845-8vkss7as/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1ee438a59ce9d4c4e70a7b5582d453e3a0bd994b --- /dev/null +++ b/wandb/run-20231031_200845-8vkss7as/logs/debug-internal.log @@ -0,0 +1,1328 @@ +2023-10-31 20:08:45,070 INFO StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-10-31 20:08:45.068914 +2023-10-31 20:08:45,072 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status +2023-10-31 20:08:45,075 INFO WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/run-8vkss7as.wandb +2023-10-31 20:08:45,077 DEBUG SenderThread:34227 [sender.py:send():380] send: header +2023-10-31 20:08:45,077 DEBUG SenderThread:34227 [sender.py:send():380] send: run +2023-10-31 20:08:45,925 INFO SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files +2023-10-31 20:08:45,926 INFO SenderThread:34227 [sender.py:_start_run_threads():1122] run started: 8vkss7as with start time 1698768525.067805 +2023-10-31 20:08:45,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 20:08:45,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: check_version +2023-10-31 20:08:46,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 20:08:46,408 DEBUG HandlerThread:34227 [system_info.py:__init__():32] System info init +2023-10-31 20:08:46,408 DEBUG HandlerThread:34227 [system_info.py:__init__():47] System info init done +2023-10-31 20:08:46,408 INFO HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor +2023-10-31 20:08:46,408 INFO SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 20:08:46,408 INFO HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info +2023-10-31 20:08:46,409 INFO SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 20:08:46,410 INFO SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring +2023-10-31 20:08:46,412 INFO SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 20:08:46,413 INFO SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring +2023-10-31 20:08:46,413 INFO SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring +2023-10-31 20:08:46,507 DEBUG HandlerThread:34227 [system_info.py:probe():196] Probing system +2023-10-31 20:08:46,515 DEBUG HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 20:08:46,516 DEBUG HandlerThread:34227 [system_info.py:probe():244] Probing system done +2023-10-31 20:08:46,516 DEBUG HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T16:08:46.507382', 'startedAt': '2023-10-31T16:08:45.061072', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.5015999999999998, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.318, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.295, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 1.615, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.497, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.599, 'min': 1200.0, 'max': 3600.0}, {'current': 1.295, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.202, 'min': 1200.0, 'max': 3600.0}, {'current': 1.203, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.42, 'min': 1200.0, 'max': 3600.0}, {'current': 2.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.207, 'min': 1200.0, 'max': 3600.0}, {'current': 1.579, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.201, 'min': 1200.0, 'max': 3600.0}, {'current': 1.206, 'min': 1200.0, 'max': 3600.0}, {'current': 2.693, 'min': 1200.0, 'max': 3600.0}, {'current': 1.425, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.605, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.695, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.207, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9234428405762}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 20:08:46,516 INFO HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 20:08:46,516 INFO HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info +2023-10-31 20:08:46,516 DEBUG HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 20:08:46,517 DEBUG HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 20:08:46,517 DEBUG HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 20:08:46,929 INFO Thread-12 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/requirements.txt +2023-10-31 20:08:46,930 INFO Thread-12 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/conda-environment.yaml +2023-10-31 20:08:53,191 DEBUG HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 20:08:53,194 INFO HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 20:08:53,198 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:08:53,199 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 20:08:53,200 DEBUG SenderThread:34227 [sender.py:send():380] send: files +2023-10-31 20:08:53,200 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 20:08:53,214 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 20:08:53,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:08:53,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:08:53,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:08:53,929 INFO Thread-12 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/conda-environment.yaml +2023-10-31 20:08:53,930 INFO Thread-12 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/wandb-metadata.json +2023-10-31 20:08:54,541 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmpgwx01lwdwandb/yfbnlltg-wandb-metadata.json +2023-10-31 20:08:55,698 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:09:00,699 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:09:05,699 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:09:08,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:09:08,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:09:08,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:09:11,564 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:09:16,570 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:09:17,938 INFO Thread-12 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/config.yaml +2023-10-31 20:09:21,974 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:09:23,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:09:23,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:09:23,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:09:27,563 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:09:32,564 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:09:37,565 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:09:38,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:09:38,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:09:38,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:09:42,571 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:09:46,414 DEBUG SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 20:09:46,416 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:09:48,418 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:09:53,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:09:53,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:09:53,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:09:53,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:09:58,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:10:03,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:10:08,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:10:08,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:10:08,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:10:08,650 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:10:13,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:10:16,419 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:10:19,421 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:10:23,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:10:23,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:10:23,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:10:24,607 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:10:29,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:10:34,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:10:38,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:10:38,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:10:38,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:10:40,603 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:10:45,604 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:10:46,421 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:10:51,423 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:10:53,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:10:53,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:10:53,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:10:56,678 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:11:01,679 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:11:06,680 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:11:08,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:11:08,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:11:08,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:11:12,595 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:11:16,425 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:11:18,426 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:11:23,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:11:23,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:11:23,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:11:23,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:11:28,590 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:11:33,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:11:38,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:11:38,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:11:38,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:11:39,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:11:44,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:11:46,431 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:11:50,432 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:11:53,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:11:53,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:11:53,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:11:55,583 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:00,584 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:05,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:08,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:12:08,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:12:08,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:12:10,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:15,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:16,434 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:12:21,436 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:23,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:12:23,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:12:23,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:12:26,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:31,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:36,590 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:38,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:12:38,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:12:38,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:12:41,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:46,437 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:12:47,438 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:52,439 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:53,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:12:53,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:12:53,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:12:56,012 INFO Thread-12 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/output.log +2023-10-31 20:12:57,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:12:58,013 INFO Thread-12 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/output.log +2023-10-31 20:13:02,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:13:07,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:13:08,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:13:08,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:13:08,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:13:13,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:13:16,439 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:13:19,441 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:13:23,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:13:23,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:13:23,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:13:24,587 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:13:29,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:13:34,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:13:38,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:13:38,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:13:38,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:13:39,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:13:44,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:13:46,442 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:13:50,443 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:13:53,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:13:53,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:13:53,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:13:55,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:14:00,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:14:05,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:14:08,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:14:08,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:14:08,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:14:10,668 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:14:15,669 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:14:16,443 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:14:21,445 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:14:23,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:14:23,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:14:23,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:14:26,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:14:31,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:14:36,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:14:38,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:14:38,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:14:38,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:14:41,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:14:46,445 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:14:47,446 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:14:52,446 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:14:53,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:14:53,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:14:53,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:14:57,632 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:15:02,632 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:15:07,633 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:15:08,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:15:08,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:15:08,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:15:12,695 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:15:16,447 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:15:18,449 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:15:23,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:15:23,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:15:23,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:15:23,630 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:15:28,631 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:15:33,631 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:15:38,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:15:38,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:15:38,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:15:38,640 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:15:43,641 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:15:46,449 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:15:49,451 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:15:53,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:15:53,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:15:53,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:15:54,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:15:59,583 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:16:04,583 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:16:08,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:16:08,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:16:08,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:16:10,568 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:16:15,569 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:16:16,452 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:16:21,453 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:16:23,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:16:23,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:16:23,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:16:26,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:16:31,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:16:36,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:16:38,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:16:38,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:16:38,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:16:41,633 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:16:46,454 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:16:47,455 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:16:52,456 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:16:53,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:16:53,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:16:53,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:16:57,602 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:17:03,104 INFO Thread-12 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/output.log +2023-10-31 20:17:03,185 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:17:08,186 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:17:08,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:17:08,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:17:08,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:17:13,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:17:16,456 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:17:19,458 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:17:23,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:17:23,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:17:23,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:17:24,639 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:17:29,640 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:17:34,641 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:17:38,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:17:38,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:17:38,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:17:39,730 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:17:44,731 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:17:46,458 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:17:50,459 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:17:53,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:17:53,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:17:53,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:17:55,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:18:00,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:18:05,559 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:18:08,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:18:08,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:18:08,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:18:10,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:18:15,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:18:16,459 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:18:21,462 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:18:23,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:18:23,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:18:23,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:18:26,770 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:18:31,771 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:18:36,771 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:18:38,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:18:38,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:18:38,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:18:42,641 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:18:46,461 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:18:48,463 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:18:53,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:18:53,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:18:53,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:18:53,631 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:18:58,631 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:19:03,632 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:19:08,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:19:08,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:19:08,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:19:09,590 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:19:14,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:19:16,464 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:19:20,467 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:19:23,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:19:23,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:19:23,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:19:25,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:19:30,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:19:35,590 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:19:38,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:19:38,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:19:38,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:19:40,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:19:45,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:19:46,466 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:19:51,468 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:19:53,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:19:53,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:19:53,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:19:56,668 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:20:01,669 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:20:06,670 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:20:08,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:20:08,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:20:08,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:20:12,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:20:16,468 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:20:18,470 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:20:23,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:20:23,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:20:23,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:20:23,656 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:20:28,656 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:20:33,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:20:38,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:20:38,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:20:38,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:20:39,587 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:20:44,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:20:46,469 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:20:50,471 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:20:53,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:20:53,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:20:53,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:20:55,607 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:21:00,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:21:05,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:21:08,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:21:08,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:21:08,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:21:10,622 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:21:15,623 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:21:16,471 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:21:21,473 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:21:23,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:21:23,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:21:23,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:21:26,603 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:21:31,604 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:21:36,605 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:21:38,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:21:38,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:21:38,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:21:41,726 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:21:46,473 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:21:47,475 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:21:52,476 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:21:53,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:21:53,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:21:53,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:21:57,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:22:02,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:22:07,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:22:08,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:22:08,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:22:08,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:22:12,625 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:22:16,475 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:22:18,477 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:22:23,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:22:23,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:22:23,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:22:23,771 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:22:28,772 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:22:33,773 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:22:38,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:22:38,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:22:38,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:22:39,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:22:44,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:22:46,477 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:22:50,478 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:22:53,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:22:53,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:22:53,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:22:55,660 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:23:00,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:23:05,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:23:08,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:23:08,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:23:08,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:23:11,562 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:23:16,479 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:23:17,481 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:23:22,482 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:23:23,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:23:23,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:23:23,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:23:27,654 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:23:32,655 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:23:37,655 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:23:38,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:23:38,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:23:38,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:23:42,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:23:46,482 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:23:48,484 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:23:53,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:23:53,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:23:53,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:23:53,649 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:23:58,650 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:24:03,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:24:08,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:24:08,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:24:08,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:24:08,666 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:24:13,667 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:24:16,485 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:24:19,487 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:24:23,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:24:23,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:24:23,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:24:24,691 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:24:29,692 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:24:34,692 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:24:38,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:24:38,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:24:38,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:24:40,566 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:24:45,567 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:24:46,487 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:24:51,489 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:24:53,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:24:53,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:24:53,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:24:56,622 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:25:01,623 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:25:06,623 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:25:08,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:25:08,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:25:08,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:25:11,633 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:25:16,489 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:25:17,490 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:25:22,491 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:25:23,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:25:23,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:25:23,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:25:27,605 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:25:32,606 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:25:37,606 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:25:38,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:25:38,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:25:38,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:25:42,739 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:25:46,491 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:25:48,492 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:25:53,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:25:53,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:25:53,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:25:53,757 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:25:58,758 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:26:03,759 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:26:08,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:26:08,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:26:08,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:26:09,626 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:26:14,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:26:16,493 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:26:20,495 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:26:23,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:26:23,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:26:23,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:26:25,609 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:26:30,609 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:26:35,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:26:38,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:26:38,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:26:38,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:26:40,622 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:26:45,623 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:26:46,495 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:26:51,496 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:26:53,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:26:53,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:26:53,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:26:56,649 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:27:01,650 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:27:06,650 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:27:08,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:27:08,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:27:08,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:27:11,688 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:27:16,497 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:27:17,498 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:27:22,499 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:27:23,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:27:23,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:27:23,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:27:27,623 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:27:32,624 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:27:37,625 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:27:38,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:27:38,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:27:38,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:27:43,609 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:27:46,499 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:27:49,501 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:27:53,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:27:53,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:27:53,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:27:54,607 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:27:59,607 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:28:04,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:28:08,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:28:08,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:28:08,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:28:10,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:28:15,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:28:16,501 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:28:21,503 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:28:23,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:28:23,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:28:23,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:28:26,567 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:28:31,567 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:28:36,568 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:28:38,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:28:38,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:28:38,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:28:41,635 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:28:46,503 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:28:47,505 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:28:52,505 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:28:53,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:28:53,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:28:53,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:28:57,821 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:29:02,822 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:29:07,822 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:29:08,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:29:08,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:29:08,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:29:13,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:29:16,505 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:29:19,507 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:29:23,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:29:23,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:29:23,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:29:24,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:29:29,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:29:34,586 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:29:38,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:29:38,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:29:38,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:29:39,718 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:29:44,719 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:29:46,507 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:29:50,509 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:29:53,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:29:53,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:29:53,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:29:55,706 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:30:00,707 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:30:05,708 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:30:08,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:30:08,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:30:08,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:30:11,642 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:30:16,510 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:30:17,512 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:30:22,512 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:30:23,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:30:23,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:30:23,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:30:27,587 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:30:32,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:30:37,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:30:38,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:30:38,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:30:38,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:30:42,674 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:30:46,515 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:30:48,516 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:30:53,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:30:53,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:30:53,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:30:53,696 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:30:58,697 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:31:03,697 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:31:08,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:31:08,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:31:08,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:31:08,718 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:31:13,718 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:31:16,517 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:31:19,519 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:31:23,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:31:23,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:31:23,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:31:24,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:31:29,629 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:31:34,629 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:31:38,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:31:38,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:31:38,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:31:39,760 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:31:44,760 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:31:46,519 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:31:50,521 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:31:53,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:31:53,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:31:53,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:31:55,712 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:32:00,713 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:32:05,714 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:32:08,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:32:08,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:32:08,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:32:11,622 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:32:16,522 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:32:17,524 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:32:22,525 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:32:23,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:32:23,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:32:23,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:32:27,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:32:32,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:32:37,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:32:38,223 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:32:38,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:32:38,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:32:42,606 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:32:46,524 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:32:48,525 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:32:53,223 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:32:53,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:32:53,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:32:53,572 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:32:58,573 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:33:03,574 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:33:08,223 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:33:08,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:33:08,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:33:08,594 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:33:11,456 INFO Thread-12 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/output.log +2023-10-31 20:33:13,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:33:16,525 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:33:19,527 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:33:23,223 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:33:23,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:33:23,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:33:24,614 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:33:29,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:33:34,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:33:38,223 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:33:38,224 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:33:38,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:33:39,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:33:44,621 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:33:46,528 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:33:50,530 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:33:53,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:33:53,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:33:53,226 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:33:55,604 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:34:00,605 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:34:05,606 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:34:08,224 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:34:08,225 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:34:08,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:34:10,720 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:34:15,721 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:34:16,531 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:34:21,533 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:34:23,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:34:23,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:34:23,226 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:34:26,590 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:34:31,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:34:36,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:34:38,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:34:38,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:34:38,227 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:34:42,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:34:46,533 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:34:47,535 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:34:52,535 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:34:53,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:34:53,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:34:53,227 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:34:58,187 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:35:03,187 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:35:08,188 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:35:08,225 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:35:08,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:35:08,226 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:35:13,671 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:35:16,536 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:35:19,538 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:35:23,225 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:35:23,226 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:35:23,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:35:24,590 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:35:29,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:35:34,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:35:38,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:35:38,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:35:38,226 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:35:39,655 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:35:44,656 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:35:46,538 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:35:50,540 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:35:53,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:35:53,226 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:35:53,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:35:55,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:36:00,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:36:05,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:36:08,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:36:08,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:36:08,227 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:36:10,696 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:36:15,697 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:36:16,540 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:36:21,542 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:36:23,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:36:23,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:36:23,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:36:27,004 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:36:32,005 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:36:37,005 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:36:38,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:36:38,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:36:38,227 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:36:42,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:36:46,541 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:36:48,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:36:53,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:36:53,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:36:53,227 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:36:53,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:36:58,654 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:37:03,655 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:37:08,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:37:08,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:37:08,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:37:08,669 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:37:13,670 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:37:16,544 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:37:19,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:37:23,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:37:23,227 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:37:23,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:37:24,638 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:37:29,638 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:37:34,639 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:37:38,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:37:38,227 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:37:38,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:37:40,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:37:45,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:37:46,546 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:37:51,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:37:53,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:37:53,227 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:37:53,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:37:56,642 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:38:01,643 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:38:06,644 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:38:08,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:38:08,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:38:08,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:38:12,625 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:38:16,548 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:38:18,550 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:38:23,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:38:23,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:38:23,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:38:23,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:38:28,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:38:33,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:38:38,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:38:38,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:38:38,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:38:38,650 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:38:43,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:38:46,550 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:38:49,551 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:38:53,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:38:53,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:38:53,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:38:54,590 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:38:59,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:39:04,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:39:08,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:39:08,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:39:08,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:39:09,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:39:14,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:39:16,552 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:39:20,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:39:23,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:39:23,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:39:23,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:39:25,682 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:39:30,682 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:39:35,683 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:39:38,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:39:38,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:39:38,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:39:41,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:39:46,555 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:39:47,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:39:52,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:39:53,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:39:53,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:39:53,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:39:57,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:40:02,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:40:07,635 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:40:08,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:40:08,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:40:08,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:40:13,635 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:40:16,557 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:40:19,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:40:23,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:40:23,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:40:23,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:40:24,675 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:40:29,676 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:40:34,676 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:40:38,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:40:38,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:40:38,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:40:40,641 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:40:45,642 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:40:46,559 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:40:51,560 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:40:53,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:40:53,229 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:40:53,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:40:56,622 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:41:01,623 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:41:06,624 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:41:08,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:41:08,229 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:41:08,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:41:12,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:41:16,561 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:41:18,562 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:41:23,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:41:23,229 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:41:23,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:41:23,660 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:41:28,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:41:33,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:41:38,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:41:38,229 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:41:38,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:41:38,747 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:41:43,748 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:41:46,563 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:41:49,565 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:41:53,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:41:53,229 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:41:53,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:41:54,647 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:41:59,648 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:42:04,648 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:42:08,229 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:42:08,229 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:42:08,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:42:10,584 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:42:15,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:42:16,565 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:42:21,567 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:42:23,229 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:42:23,229 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:42:23,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:42:26,578 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:42:31,579 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:42:36,580 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:42:38,230 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:42:38,230 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:42:38,231 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:42:41,708 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:42:46,568 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:42:47,569 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:42:52,570 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:42:53,229 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:42:53,230 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:42:53,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:42:58,171 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:43:03,172 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:43:08,173 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:43:08,229 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:43:08,230 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:43:08,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:43:13,288 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:43:16,570 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:43:18,571 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:43:23,229 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:43:23,230 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:43:23,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:43:23,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:43:28,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:43:33,602 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:43:38,229 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:43:38,230 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:43:38,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:43:38,646 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:43:43,646 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:43:46,571 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:43:49,574 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:43:53,230 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:43:53,230 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:43:53,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:43:54,682 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:43:59,683 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:44:04,683 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:44:08,230 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:44:08,230 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:44:08,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:44:10,643 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:44:15,644 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:44:16,574 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:44:21,576 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:44:23,230 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:44:23,231 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:44:23,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:44:26,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:44:31,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:44:36,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:44:38,230 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:44:38,231 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:44:38,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:44:41,675 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:44:46,577 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:44:47,578 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:44:52,579 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:44:53,230 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:44:53,231 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:44:53,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:44:57,698 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:45:02,699 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:45:07,699 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:45:08,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:45:08,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:45:08,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:45:13,579 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:45:16,579 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:45:18,580 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:45:23,230 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:45:23,231 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:45:23,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:45:23,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:45:28,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:45:33,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:45:38,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:45:38,231 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:45:38,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:45:38,637 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:45:43,638 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:45:46,581 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:45:49,583 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:45:53,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:45:53,231 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:45:53,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:45:54,662 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:45:59,663 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:46:04,664 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:46:08,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:46:08,231 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:46:08,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:46:10,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:46:15,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:46:16,583 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:46:21,584 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:46:23,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:46:23,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:46:23,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:46:26,676 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:46:31,677 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:46:36,678 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:46:38,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:46:38,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:46:38,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:46:42,631 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:46:46,585 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:46:48,586 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:46:53,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:46:53,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:46:53,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:46:53,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:46:58,590 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:47:03,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:47:08,232 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:47:08,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:47:08,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:47:08,641 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:47:13,642 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:47:16,587 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:47:19,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:47:23,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:47:23,231 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:47:23,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:47:25,569 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:47:30,570 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:47:35,570 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:47:38,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:47:38,231 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:47:38,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:47:40,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:47:45,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:47:46,588 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:47:51,590 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:47:53,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:47:53,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:47:53,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:47:57,579 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:48:02,580 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:48:07,580 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:48:08,232 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:48:08,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:48:08,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:48:12,594 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:48:16,590 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:48:18,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:48:23,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:48:23,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:48:23,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:48:23,674 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:48:28,675 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:48:33,675 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:48:38,232 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:48:38,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:48:38,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:48:39,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:48:44,662 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:48:46,593 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:48:50,595 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:48:53,232 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:48:53,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:48:53,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:48:55,599 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:49:00,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:49:05,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:49:08,232 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:49:08,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:49:08,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:49:10,645 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:49:15,646 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:49:16,596 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:49:21,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:49:23,232 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:49:23,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:49:23,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:49:27,479 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:49:32,480 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:49:37,481 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:49:38,232 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:49:38,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:49:38,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:49:43,446 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:49:46,599 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:49:48,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:49:53,232 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:49:53,232 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:49:53,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:49:54,406 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:49:59,407 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:50:04,408 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:50:08,232 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:50:08,233 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:50:08,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:50:09,432 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:50:14,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:50:16,601 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:50:19,603 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:50:23,232 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:50:23,233 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:50:23,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:50:24,631 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:50:29,632 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:50:34,633 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:50:38,233 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:50:38,233 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:50:38,281 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:50:40,161 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:50:45,162 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:50:46,604 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:50:50,606 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:50:53,232 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:50:53,233 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:50:53,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:50:56,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:51:01,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:51:06,258 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:51:08,233 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:51:08,233 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:51:08,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:51:12,121 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:51:16,605 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:51:17,606 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:51:22,607 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:51:23,233 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:51:23,233 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:51:23,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:51:28,570 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:51:33,571 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:51:38,233 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:51:38,233 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:51:38,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:51:39,190 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:51:44,191 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:51:46,607 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:51:49,609 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:51:53,233 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:51:53,233 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:51:53,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:51:54,668 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:51:59,668 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:52:04,669 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:52:08,233 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:52:08,234 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:52:08,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:52:10,031 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:52:15,032 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:52:16,609 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:52:20,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:52:23,233 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:52:23,234 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:52:23,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:52:26,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:52:31,594 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:52:36,595 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:52:38,233 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:52:38,234 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:52:38,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:52:41,699 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:52:46,612 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:52:47,613 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:52:52,614 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:52:53,233 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:52:53,234 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:52:53,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:52:57,677 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:53:02,677 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:53:07,678 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:53:08,233 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:53:08,234 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:53:08,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:53:13,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:53:16,613 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:53:18,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:53:23,234 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:53:23,234 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:53:23,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:53:24,575 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:53:29,576 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:53:31,578 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history +2023-10-31 20:53:31,580 DEBUG SenderThread:34227 [sender.py:send():380] send: history +2023-10-31 20:53:31,581 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: summary_record +2023-10-31 20:53:31,581 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 20:53:31,621 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 20:53:31,621 DEBUG SenderThread:34227 [sender.py:send():380] send: exit +2023-10-31 20:53:31,621 INFO SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0 +2023-10-31 20:53:31,622 INFO SenderThread:34227 [sender.py:send_exit():587] handling runtime: 2685 +2023-10-31 20:53:31,622 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 20:53:31,622 INFO SenderThread:34227 [sender.py:send_exit():593] send defer +2023-10-31 20:53:31,622 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:31,623 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0 +2023-10-31 20:53:31,623 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:31,623 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-10-31 20:53:31,623 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 1 +2023-10-31 20:53:31,623 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:31,623 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1 +2023-10-31 20:53:31,623 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:31,624 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-10-31 20:53:31,624 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 2 +2023-10-31 20:53:31,624 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:31,624 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2 +2023-10-31 20:53:31,624 INFO HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor +2023-10-31 20:53:31,624 DEBUG SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-10-31 20:53:31,625 DEBUG SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-10-31 20:53:31,625 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor +2023-10-31 20:53:31,628 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor +2023-10-31 20:53:31,906 INFO Thread-12 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/wandb-summary.json +2023-10-31 20:53:33,239 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor +2023-10-31 20:53:33,240 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor +2023-10-31 20:53:33,240 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor +2023-10-31 20:53:33,240 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:53:33,241 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:33,242 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-10-31 20:53:33,242 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 3 +2023-10-31 20:53:33,242 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:53:33,242 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:33,243 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:53:33,243 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3 +2023-10-31 20:53:33,244 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:33,244 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-10-31 20:53:33,244 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 4 +2023-10-31 20:53:33,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:33,245 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4 +2023-10-31 20:53:33,245 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:33,245 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-10-31 20:53:33,245 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 5 +2023-10-31 20:53:33,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:33,245 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5 +2023-10-31 20:53:33,246 DEBUG SenderThread:34227 [sender.py:send():380] send: summary +2023-10-31 20:53:33,247 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 20:53:33,247 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:33,247 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-10-31 20:53:33,247 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 6 +2023-10-31 20:53:33,247 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:33,247 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6 +2023-10-31 20:53:33,248 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:33,248 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-10-31 20:53:33,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:53:33,580 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:53:33,635 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 7 +2023-10-31 20:53:33,635 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:53:33,635 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:33,636 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7 +2023-10-31 20:53:33,636 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:33,637 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-10-31 20:53:33,907 INFO Thread-12 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/wandb-summary.json +2023-10-31 20:53:33,907 INFO Thread-12 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/config.yaml +2023-10-31 20:53:34,581 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:53:34,583 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 8 +2023-10-31 20:53:34,583 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:53:34,583 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:34,584 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8 +2023-10-31 20:53:34,584 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:34,584 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-10-31 20:53:34,584 INFO SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact +2023-10-31 20:53:34,585 INFO SenderThread:34227 [job_builder.py:_get_source_type():404] no source found +2023-10-31 20:53:34,585 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 9 +2023-10-31 20:53:34,586 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:34,586 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9 +2023-10-31 20:53:34,586 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:34,586 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-10-31 20:53:34,586 INFO SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher +2023-10-31 20:53:34,907 INFO Thread-12 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/output.log +2023-10-31 20:53:34,908 INFO SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files +2023-10-31 20:53:34,909 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/output.log output.log +2023-10-31 20:53:34,909 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/requirements.txt requirements.txt +2023-10-31 20:53:34,909 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/conda-environment.yaml conda-environment.yaml +2023-10-31 20:53:34,915 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/wandb-summary.json wandb-summary.json +2023-10-31 20:53:34,921 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/config.yaml config.yaml +2023-10-31 20:53:34,938 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/wandb-metadata.json wandb-metadata.json +2023-10-31 20:53:34,938 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 10 +2023-10-31 20:53:34,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:34,944 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10 +2023-10-31 20:53:34,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:34,946 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-10-31 20:53:34,946 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 20:53:35,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:53:35,582 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:53:35,829 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/output.log +2023-10-31 20:53:36,156 INFO wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/wandb-summary.json +2023-10-31 20:53:36,163 INFO wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/config.yaml +2023-10-31 20:53:36,178 INFO wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/conda-environment.yaml +2023-10-31 20:53:36,355 INFO wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/files/requirements.txt +2023-10-31 20:53:36,556 INFO Thread-11 :34227 [sender.py:transition_state():613] send defer: 11 +2023-10-31 20:53:36,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:36,557 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11 +2023-10-31 20:53:36,557 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:36,557 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-10-31 20:53:36,557 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher +2023-10-31 20:53:36,558 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 12 +2023-10-31 20:53:36,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:36,558 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12 +2023-10-31 20:53:36,558 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:36,558 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-10-31 20:53:36,558 INFO SenderThread:34227 [file_stream.py:finish():595] file stream finish called +2023-10-31 20:53:36,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:53:37,879 INFO SenderThread:34227 [file_stream.py:finish():599] file stream finish is done +2023-10-31 20:53:37,880 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 13 +2023-10-31 20:53:37,880 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:53:37,880 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:37,881 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13 +2023-10-31 20:53:37,881 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:37,881 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-10-31 20:53:37,881 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 14 +2023-10-31 20:53:37,882 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 20:53:37,882 DEBUG SenderThread:34227 [sender.py:send():380] send: final +2023-10-31 20:53:37,882 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14 +2023-10-31 20:53:37,883 DEBUG SenderThread:34227 [sender.py:send():380] send: footer +2023-10-31 20:53:37,883 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 20:53:37,883 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-10-31 20:53:37,884 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:53:37,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 20:53:37,886 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 20:53:37,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info +2023-10-31 20:53:37,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary +2023-10-31 20:53:37,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history +2023-10-31 20:53:37,887 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: server_info +2023-10-31 20:53:37,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info +2023-10-31 20:53:38,263 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: job_info +2023-10-31 20:53:38,264 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown +2023-10-31 20:53:38,264 INFO HandlerThread:34227 [handler.py:finish():866] shutting down handler +2023-10-31 20:53:38,894 INFO WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/run-8vkss7as.wandb +2023-10-31 20:53:39,263 INFO SenderThread:34227 [sender.py:finish():1534] shutting down sender +2023-10-31 20:53:39,263 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 20:53:39,264 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231031_200845-8vkss7as/logs/debug.log b/wandb/run-20231031_200845-8vkss7as/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..de6347215471e711ecf76862905bf3af1da3f7f1 --- /dev/null +++ b/wandb/run-20231031_200845-8vkss7as/logs/debug.log @@ -0,0 +1,34 @@ +2023-10-31 20:08:45,063 INFO MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 20:08:45,063 INFO MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070 +2023-10-31 20:08:45,063 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 20:08:45,063 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 20:08:45,063 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 20:08:45,063 INFO MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 20:08:45,063 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 20:08:45,063 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 20:08:45,063 INFO MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/logs/debug.log +2023-10-31 20:08:45,064 INFO MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as/logs/debug-internal.log +2023-10-31 20:08:45,064 INFO MainThread:34070 [wandb_init.py:init():568] calling init triggers +2023-10-31 20:08:45,064 INFO MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'SGD', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 31} +2023-10-31 20:08:45,064 INFO MainThread:34070 [wandb_init.py:init():618] starting backend +2023-10-31 20:08:45,064 INFO MainThread:34070 [wandb_init.py:init():622] setting up manager +2023-10-31 20:08:45,066 INFO MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 20:08:45,067 INFO MainThread:34070 [wandb_init.py:init():628] backend started and connected +2023-10-31 20:08:45,071 INFO MainThread:34070 [wandb_init.py:init():720] updated telemetry +2023-10-31 20:08:45,072 INFO MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 20:08:45,944 INFO MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 20:08:46,378 INFO MainThread:34070 [wandb_run.py:_on_init():2229] got version response +2023-10-31 20:08:46,378 INFO MainThread:34070 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 20:08:53,212 INFO MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 20:08:53,212 INFO MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 20:08:53,212 INFO MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 20:08:53,212 INFO MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 20:08:53,213 INFO MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process +2023-10-31 20:53:31,578 INFO MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/8vkss7as +2023-10-31 20:53:31,578 INFO MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0 +2023-10-31 20:53:31,579 INFO MainThread:34070 [wandb_run.py:_restore():2151] restore +2023-10-31 20:53:31,579 INFO MainThread:34070 [wandb_run.py:_restore():2157] restore done +2023-10-31 20:53:39,266 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-10-31 20:53:39,267 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-10-31 20:53:39,274 INFO MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files diff --git a/wandb/run-20231031_200845-8vkss7as/run-8vkss7as.wandb b/wandb/run-20231031_200845-8vkss7as/run-8vkss7as.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a6857160bda1a344323e592661b8c022caa19172 Binary files /dev/null and b/wandb/run-20231031_200845-8vkss7as/run-8vkss7as.wandb differ diff --git a/wandb/run-20231031_205340-97cre3xd/files/conda-environment.yaml b/wandb/run-20231031_205340-97cre3xd/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231031_205340-97cre3xd/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231031_205340-97cre3xd/files/config.yaml b/wandb/run-20231031_205340-97cre3xd/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0b936af5bec1fd4298f1cd4ff6976496e2624ae --- /dev/null +++ b/wandb/run-20231031_205340-97cre3xd/files/config.yaml @@ -0,0 +1,53 @@ +wandb_version: 1 + +optimizer: + desc: null + value: RMSprop +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 31 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698771220.314322 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 2 + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231031_205340-97cre3xd/files/output.log b/wandb/run-20231031_205340-97cre3xd/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3e3243d813ebee0d393d6b3779843d64e779d465 --- /dev/null +++ b/wandb/run-20231031_205340-97cre3xd/files/output.log @@ -0,0 +1,3 @@ + +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth \ No newline at end of file diff --git a/wandb/run-20231031_205340-97cre3xd/files/requirements.txt b/wandb/run-20231031_205340-97cre3xd/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231031_205340-97cre3xd/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_205340-97cre3xd/files/wandb-metadata.json b/wandb/run-20231031_205340-97cre3xd/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..df3449407e53d927d2808e987230e9e2e4c925ab --- /dev/null +++ b/wandb/run-20231031_205340-97cre3xd/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T16:53:42.254778", + "startedAt": "2023-10-31T16:53:40.308513", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.4536250000000002, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.298, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.653, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.297, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.528, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.216, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.548, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.211, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.61, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.296, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.549, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.241, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.597, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.923469543457 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_205340-97cre3xd/files/wandb-summary.json b/wandb/run-20231031_205340-97cre3xd/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8f3dc8dc067386809221fac0db6bd0006c948fbb --- /dev/null +++ b/wandb/run-20231031_205340-97cre3xd/files/wandb-summary.json @@ -0,0 +1 @@ +{"Train Loss": 1.4997429847717285, "Train Accuracy": 0.4841685891151428, "Validation Loss": 1.3094165325164795, "Validation Accuracy": 0.5346873998641968, "_timestamp": 1698773156.7494874, "_runtime": 1936.4351654052734, "_step": 0, "_wandb": {"runtime": 1934}} \ No newline at end of file diff --git a/wandb/run-20231031_205340-97cre3xd/logs/debug-internal.log b/wandb/run-20231031_205340-97cre3xd/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1d8f562f4f45f4dc9571bc7eb9152ec8c9ca2bc5 --- /dev/null +++ b/wandb/run-20231031_205340-97cre3xd/logs/debug-internal.log @@ -0,0 +1,1004 @@ +2023-10-31 20:53:40,315 INFO StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-10-31 20:53:40.314667 +2023-10-31 20:53:40,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status +2023-10-31 20:53:40,319 INFO WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/run-97cre3xd.wandb +2023-10-31 20:53:40,322 DEBUG SenderThread:34227 [sender.py:send():380] send: header +2023-10-31 20:53:40,322 DEBUG SenderThread:34227 [sender.py:send():380] send: run +2023-10-31 20:53:41,609 INFO SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files +2023-10-31 20:53:41,610 INFO SenderThread:34227 [sender.py:_start_run_threads():1122] run started: 97cre3xd with start time 1698771220.314322 +2023-10-31 20:53:41,633 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 20:53:41,634 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: check_version +2023-10-31 20:53:42,158 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 20:53:42,159 DEBUG HandlerThread:34227 [system_info.py:__init__():32] System info init +2023-10-31 20:53:42,159 DEBUG HandlerThread:34227 [system_info.py:__init__():47] System info init done +2023-10-31 20:53:42,159 INFO HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor +2023-10-31 20:53:42,159 INFO SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 20:53:42,160 INFO HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info +2023-10-31 20:53:42,160 INFO SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 20:53:42,161 INFO SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring +2023-10-31 20:53:42,162 INFO SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 20:53:42,163 INFO SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring +2023-10-31 20:53:42,164 INFO SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring +2023-10-31 20:53:42,254 DEBUG HandlerThread:34227 [system_info.py:probe():196] Probing system +2023-10-31 20:53:42,255 DEBUG HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 20:53:42,255 DEBUG HandlerThread:34227 [system_info.py:probe():244] Probing system done +2023-10-31 20:53:42,255 DEBUG HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T16:53:42.254778', 'startedAt': '2023-10-31T16:53:40.308513', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.4536250000000002, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.298, 'min': 1200.0, 'max': 3600.0}, {'current': 2.653, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.297, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.528, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.216, 'min': 1200.0, 'max': 3600.0}, {'current': 1.548, 'min': 1200.0, 'max': 3600.0}, {'current': 2.211, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 2.61, 'min': 1200.0, 'max': 3600.0}, {'current': 1.296, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.549, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.241, 'min': 1200.0, 'max': 3600.0}, {'current': 1.597, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.923469543457}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 20:53:42,255 INFO HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 20:53:42,256 INFO HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info +2023-10-31 20:53:42,256 DEBUG HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 20:53:42,256 DEBUG HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 20:53:42,256 DEBUG HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 20:53:42,613 INFO Thread-25 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/conda-environment.yaml +2023-10-31 20:53:42,613 INFO Thread-25 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/requirements.txt +2023-10-31 20:53:48,878 DEBUG HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 20:53:48,880 INFO HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 20:53:48,880 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:53:48,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 20:53:48,882 DEBUG SenderThread:34227 [sender.py:send():380] send: files +2023-10-31 20:53:48,882 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 20:53:48,897 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:53:48,897 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:53:48,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:53:49,340 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 20:53:49,613 INFO Thread-25 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/conda-environment.yaml +2023-10-31 20:53:49,613 INFO Thread-25 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/wandb-metadata.json +2023-10-31 20:53:50,154 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmpb4bnucxwwandb/kv4n1opa-wandb-metadata.json +2023-10-31 20:53:51,340 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:53:56,341 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:54:01,342 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:54:03,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:54:03,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:54:03,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:54:07,332 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:54:12,338 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:54:13,621 INFO Thread-25 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/config.yaml +2023-10-31 20:54:17,745 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:54:18,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:54:18,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:54:18,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:54:23,279 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:54:28,279 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:54:33,280 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:54:33,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:54:33,895 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:54:33,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:54:38,330 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:54:42,164 DEBUG SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 20:54:42,167 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:54:44,168 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:54:48,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:54:48,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:54:48,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:54:49,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:54:54,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:54:59,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:55:03,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:55:03,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:55:03,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:55:04,518 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:55:09,519 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:55:12,169 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:55:15,170 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:55:18,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:55:18,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:55:18,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:55:20,340 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:55:25,341 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:55:30,342 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:55:33,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:55:33,895 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:55:33,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:55:36,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:55:41,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:55:42,171 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:55:47,173 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:55:48,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:55:48,895 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:55:48,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:55:52,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:55:57,262 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:56:02,263 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:56:03,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:56:03,895 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:56:03,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:56:08,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:56:12,174 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:56:14,175 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:56:18,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:56:18,895 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:56:18,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:56:19,285 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:56:24,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:56:29,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:56:33,895 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:56:33,895 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:56:33,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:56:34,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:56:39,329 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:56:42,176 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:56:45,178 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:56:48,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:56:48,895 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:56:48,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:56:50,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:56:55,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:57:00,262 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:57:03,895 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:57:03,895 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:57:03,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:57:06,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:57:11,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:57:12,178 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:57:17,180 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:57:18,895 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:57:18,895 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:57:18,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:57:22,290 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:57:27,291 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:57:32,292 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:57:33,895 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:57:33,895 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:57:33,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:57:37,292 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:57:42,180 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:57:43,181 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:57:48,182 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:57:48,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:57:48,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:57:48,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:57:49,693 INFO Thread-25 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/output.log +2023-10-31 20:57:51,693 INFO Thread-25 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/output.log +2023-10-31 20:57:53,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:57:58,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:58:03,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:58:03,901 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:58:03,902 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:58:03,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:58:09,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:58:12,183 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:58:15,185 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:58:18,900 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:58:18,901 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:58:18,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:58:20,333 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:58:25,334 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:58:30,334 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:58:33,901 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:58:33,902 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:58:33,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:58:36,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:58:41,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:58:42,185 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:58:47,187 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:58:48,901 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:58:48,902 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:58:48,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:58:52,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:58:57,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:59:02,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:59:03,901 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:59:03,902 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:59:03,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:59:07,396 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:59:12,187 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:59:13,189 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:59:18,189 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:59:18,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:59:18,902 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:59:18,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:59:23,243 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:59:28,243 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:59:33,244 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:59:33,901 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:59:33,902 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:59:33,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:59:38,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:59:42,190 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 20:59:44,191 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:59:48,901 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 20:59:48,902 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 20:59:48,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 20:59:49,344 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:59:54,344 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 20:59:59,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:00:03,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:00:03,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:00:03,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:00:05,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:00:10,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:00:12,191 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:00:16,193 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:00:18,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:00:18,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:00:18,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:00:21,362 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:00:26,363 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:00:31,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:00:33,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:00:33,902 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:00:33,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:00:37,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:00:42,194 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:00:43,195 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:00:48,196 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:00:48,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:00:48,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:00:48,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:00:53,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:00:58,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:01:03,297 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:01:03,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:01:03,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:01:03,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:01:08,382 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:01:12,195 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:01:14,197 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:01:18,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:01:18,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:01:18,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:01:19,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:01:24,266 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:01:29,267 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:01:33,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:01:33,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:01:33,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:01:34,279 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:01:39,280 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:01:42,199 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:01:45,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:01:48,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:01:48,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:01:48,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:01:50,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:01:55,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:02:00,332 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:02:03,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:02:03,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:02:03,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:02:06,329 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:02:11,330 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:02:12,201 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:02:17,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:02:18,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:02:18,904 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:02:18,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:02:22,336 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:02:27,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:02:32,338 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:02:33,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:02:33,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:02:33,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:02:37,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:02:42,203 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:02:43,204 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:02:48,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:02:48,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:02:48,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:02:48,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:02:53,340 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:02:58,341 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:03:03,341 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:03:03,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:03:03,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:03:03,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:03:09,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:03:12,206 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:03:15,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:03:18,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:03:18,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:03:18,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:03:20,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:03:25,329 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:03:30,330 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:03:33,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:03:33,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:03:33,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:03:35,330 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:03:40,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:03:42,208 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:03:46,210 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:03:48,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:03:48,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:03:48,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:03:51,254 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:03:56,255 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:04:01,256 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:04:03,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:04:03,904 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:04:03,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:04:06,391 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:04:11,391 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:04:12,211 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:04:17,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:04:18,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:04:18,904 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:04:18,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:04:22,290 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:04:27,291 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:04:32,292 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:04:33,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:04:33,904 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:04:33,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:04:37,360 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:04:42,213 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:04:43,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:04:48,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:04:48,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:04:48,904 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:04:48,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:04:53,338 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:04:58,339 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:05:03,340 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:05:03,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:05:03,904 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:05:03,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:05:09,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:05:12,216 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:05:15,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:05:18,904 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:05:18,904 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:05:18,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:05:20,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:05:25,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:05:30,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:05:33,904 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:05:33,905 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:05:33,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:05:36,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:05:41,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:05:42,218 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:05:47,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:05:48,904 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:05:48,904 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:05:48,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:05:52,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:05:57,872 INFO Thread-25 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/output.log +2023-10-31 21:05:58,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:06:03,278 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:06:03,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:06:03,906 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:06:03,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:06:08,378 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:06:12,220 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:06:14,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:06:18,904 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:06:18,904 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:06:18,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:06:19,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:06:24,262 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:06:29,263 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:06:33,904 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:06:33,904 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:06:33,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:06:34,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:06:39,354 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:06:42,221 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:06:45,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:06:48,904 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:06:48,904 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:06:48,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:06:50,334 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:06:55,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:07:00,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:07:03,904 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:07:03,905 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:07:03,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:07:06,323 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:07:11,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:07:12,223 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:07:17,224 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:07:18,904 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:07:18,905 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:07:18,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:07:22,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:07:27,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:07:32,323 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:07:33,904 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:07:33,905 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:07:33,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:07:37,370 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:07:42,225 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:07:43,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:07:48,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:07:48,904 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:07:48,905 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:07:48,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:07:53,383 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:07:58,384 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:08:03,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:08:03,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:08:03,905 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:08:03,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:08:09,358 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:08:12,226 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:08:15,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:08:18,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:08:18,905 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:08:18,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:08:20,285 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:08:25,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:08:30,287 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:08:33,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:08:33,905 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:08:33,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:08:35,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:08:40,350 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:08:42,229 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:08:46,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:08:48,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:08:48,905 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:08:48,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:08:51,343 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:08:56,344 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:09:01,344 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:09:03,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:09:03,905 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:09:03,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:09:06,362 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:09:11,363 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:09:12,231 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:09:17,232 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:09:18,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:09:18,906 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:09:18,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:09:22,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:09:27,336 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:09:32,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:09:33,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:09:33,906 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:09:33,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:09:38,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:09:42,232 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:09:44,234 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:09:48,906 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:09:48,906 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:09:48,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:09:49,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:09:54,362 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:09:59,363 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:10:03,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:10:03,906 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:10:03,953 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:10:05,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:10:10,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:10:12,234 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:10:16,236 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:10:18,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:10:18,906 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:10:18,953 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:10:21,343 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:10:26,343 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:10:31,344 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:10:33,906 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:10:33,906 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:10:33,953 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:10:37,336 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:10:42,236 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:10:43,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:10:48,238 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:10:48,906 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:10:48,906 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:10:48,953 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:10:53,284 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:10:58,285 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:11:03,285 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:11:03,906 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:11:03,906 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:11:03,953 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:11:09,254 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:11:12,238 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:11:15,240 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:11:18,906 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:11:18,906 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:11:18,953 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:11:20,299 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:11:25,300 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:11:30,300 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:11:33,906 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:11:33,907 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:11:33,953 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:11:35,377 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:11:40,378 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:11:42,240 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:11:46,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:11:48,906 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:11:48,907 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:11:48,953 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:11:51,336 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:11:56,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:12:01,338 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:12:03,906 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:12:03,907 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:12:03,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:12:07,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:12:12,242 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:12:13,243 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:12:18,244 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:12:18,906 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:12:18,907 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:12:18,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:12:23,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:12:28,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:12:33,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:12:33,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:12:33,907 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:12:33,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:12:39,299 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:12:42,245 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:12:45,247 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:12:48,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:12:48,907 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:12:48,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:12:50,252 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:12:55,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:13:00,254 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:13:03,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:13:03,907 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:13:03,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:13:05,350 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:13:10,350 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:13:12,248 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:13:16,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:13:18,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:13:18,907 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:13:18,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:13:21,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:13:26,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:13:31,332 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:13:33,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:13:33,907 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:13:33,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:13:37,264 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:13:42,249 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:13:43,251 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:13:48,252 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:13:48,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:13:48,907 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:13:48,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:13:53,334 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:13:58,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:14:03,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:14:03,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:14:03,908 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:14:03,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:14:09,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:14:12,251 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:14:15,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:14:18,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:14:18,908 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:14:18,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:14:20,336 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:14:25,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:14:30,338 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:14:33,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:14:33,908 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:14:33,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:14:36,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:14:41,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:14:42,254 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:14:47,255 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:14:48,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:14:48,908 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:14:48,955 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:14:52,288 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:14:57,289 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:15:02,290 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:15:03,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:15:03,908 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:15:03,955 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:15:07,347 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:15:12,257 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:15:13,258 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:15:18,259 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:15:18,908 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:15:18,908 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:15:18,955 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:15:23,375 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:15:28,376 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:15:33,377 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:15:33,908 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:15:33,908 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:15:33,955 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:15:39,256 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:15:42,259 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:15:44,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:15:48,908 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:15:48,908 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:15:48,955 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:15:49,347 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:15:54,348 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:15:59,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:16:03,908 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:16:03,908 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:16:03,955 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:16:05,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:16:10,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:16:12,266 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:16:16,269 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:16:18,908 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:16:18,908 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:16:18,955 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:16:21,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:16:26,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:16:31,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:16:33,908 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:16:33,909 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:16:33,955 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:16:36,359 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:16:41,359 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:16:42,265 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:16:47,267 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:16:48,908 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:16:48,909 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:16:48,955 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:16:52,294 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:16:57,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:17:02,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:17:03,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:17:03,909 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:17:03,956 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:17:08,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:17:12,271 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:17:14,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:17:18,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:17:18,910 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:17:18,956 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:17:19,360 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:17:24,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:17:29,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:17:33,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:17:33,910 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:17:33,956 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:17:35,333 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:17:40,334 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:17:42,270 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:17:46,272 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:17:48,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:17:48,909 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:17:48,956 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:17:51,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:17:56,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:18:01,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:18:03,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:18:03,910 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:18:03,956 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:18:06,420 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:18:11,421 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:18:12,272 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:18:17,274 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:18:18,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:18:18,910 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:18:18,956 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:18:22,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:18:27,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:18:32,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:18:33,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:18:33,910 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:18:33,957 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:18:38,274 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:18:42,274 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:18:43,276 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:18:48,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:18:48,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:18:48,910 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:18:48,957 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:18:53,289 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:18:58,289 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:19:03,290 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:19:03,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:19:03,910 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:19:03,957 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:19:09,258 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:19:12,276 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:19:14,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:19:18,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:19:18,910 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:19:18,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:19:19,355 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:19:24,356 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:19:29,356 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:19:33,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:19:33,910 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:19:33,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:19:35,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:19:40,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:19:42,278 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:19:46,280 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:19:48,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:19:48,910 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:19:48,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:19:51,300 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:19:56,301 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:20:01,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:20:03,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:20:03,910 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:20:03,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:20:06,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:20:11,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:20:12,280 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:20:17,282 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:20:18,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:20:18,911 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:20:18,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:20:23,281 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:20:28,282 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:20:33,282 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:20:33,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:20:33,910 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:20:33,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:20:38,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:20:42,282 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:20:44,284 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:20:48,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:20:48,911 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:20:48,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:20:49,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:20:54,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:20:59,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:21:03,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:21:03,911 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:21:03,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:21:04,342 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:21:09,343 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:21:12,284 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:21:15,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:21:18,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:21:18,911 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:21:18,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:21:20,841 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:21:25,842 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:21:30,843 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:21:33,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:21:33,911 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:21:33,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:21:36,354 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:21:41,355 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:21:42,286 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:21:47,288 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:21:48,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:21:48,911 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:21:48,959 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:21:52,415 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:21:57,416 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:22:02,416 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:22:03,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:22:03,911 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:22:03,959 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:22:08,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:22:12,288 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:22:13,290 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:22:18,291 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:22:18,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:22:18,911 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:22:18,959 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:22:24,271 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:22:29,272 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:22:33,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:22:33,911 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:22:33,959 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:22:34,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:22:39,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:22:42,290 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:22:45,292 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:22:48,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:22:48,911 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:22:48,960 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:22:50,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:22:55,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:23:00,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:23:03,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:23:03,911 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:23:03,959 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:23:05,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:23:10,336 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:23:12,294 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:23:16,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:23:18,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:23:18,912 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:23:18,960 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:23:22,291 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:23:27,292 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:23:32,293 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:23:33,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:23:33,912 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:23:33,961 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:23:38,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:23:42,295 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:23:43,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:23:48,297 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:23:48,914 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:23:48,914 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:23:48,961 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:23:53,363 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:23:58,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:24:03,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:24:03,914 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:24:03,914 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:24:03,961 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:24:09,272 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:24:12,297 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:24:14,299 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:24:18,914 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:24:18,915 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:24:18,961 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:24:20,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:24:25,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:24:30,297 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:24:33,913 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:24:33,914 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:24:33,961 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:24:35,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:24:40,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:24:42,298 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:24:46,300 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:24:48,913 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:24:48,914 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:24:48,961 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:24:52,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:24:57,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:02,297 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:03,914 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:25:03,914 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:25:03,961 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:25:07,352 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:12,301 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:25:13,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:18,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:18,914 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:25:18,915 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:25:18,961 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:25:23,330 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:28,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:33,332 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:33,914 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:25:33,915 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:25:33,961 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:25:39,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:42,303 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:25:45,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:48,914 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:25:48,915 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:25:48,962 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:25:51,256 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:56,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:56,750 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history +2023-10-31 21:25:56,751 DEBUG SenderThread:34227 [sender.py:send():380] send: history +2023-10-31 21:25:56,752 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: summary_record +2023-10-31 21:25:56,752 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 21:25:56,793 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 21:25:56,793 DEBUG SenderThread:34227 [sender.py:send():380] send: exit +2023-10-31 21:25:56,793 INFO SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0 +2023-10-31 21:25:56,793 INFO SenderThread:34227 [sender.py:send_exit():587] handling runtime: 1934 +2023-10-31 21:25:56,794 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 21:25:56,794 INFO SenderThread:34227 [sender.py:send_exit():593] send defer +2023-10-31 21:25:56,794 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:25:56,794 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0 +2023-10-31 21:25:56,795 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:25:56,795 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-10-31 21:25:56,795 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 1 +2023-10-31 21:25:56,795 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:25:56,795 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1 +2023-10-31 21:25:56,795 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:25:56,795 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-10-31 21:25:56,795 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 2 +2023-10-31 21:25:56,795 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:25:56,795 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2 +2023-10-31 21:25:56,796 INFO HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor +2023-10-31 21:25:56,796 DEBUG SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-10-31 21:25:56,796 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor +2023-10-31 21:25:56,796 DEBUG SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-10-31 21:25:56,797 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor +2023-10-31 21:25:57,320 INFO Thread-25 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/wandb-summary.json +2023-10-31 21:25:58,540 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor +2023-10-31 21:25:58,541 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor +2023-10-31 21:25:58,541 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor +2023-10-31 21:25:58,542 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 21:25:58,542 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:25:58,543 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-10-31 21:25:58,543 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 3 +2023-10-31 21:25:58,543 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:25:58,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:25:58,544 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 21:25:58,545 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3 +2023-10-31 21:25:58,546 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:25:58,546 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-10-31 21:25:58,546 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 4 +2023-10-31 21:25:58,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:25:58,546 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4 +2023-10-31 21:25:58,546 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:25:58,546 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-10-31 21:25:58,547 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 5 +2023-10-31 21:25:58,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:25:58,547 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5 +2023-10-31 21:25:58,548 DEBUG SenderThread:34227 [sender.py:send():380] send: summary +2023-10-31 21:25:58,548 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 21:25:58,548 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:25:58,549 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-10-31 21:25:58,549 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 6 +2023-10-31 21:25:58,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:25:58,549 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6 +2023-10-31 21:25:58,549 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:25:58,549 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-10-31 21:25:58,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:25:58,751 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 21:25:58,951 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 7 +2023-10-31 21:25:58,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 21:25:58,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:25:58,952 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7 +2023-10-31 21:25:58,952 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:25:58,952 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-10-31 21:25:59,321 INFO Thread-25 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/config.yaml +2023-10-31 21:25:59,321 INFO Thread-25 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/wandb-summary.json +2023-10-31 21:25:59,752 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 21:26:00,776 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 8 +2023-10-31 21:26:00,776 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 21:26:00,776 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:26:00,777 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8 +2023-10-31 21:26:00,777 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:26:00,778 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-10-31 21:26:00,778 INFO SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact +2023-10-31 21:26:00,779 INFO SenderThread:34227 [job_builder.py:_get_source_type():404] no source found +2023-10-31 21:26:00,779 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 9 +2023-10-31 21:26:00,779 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:26:00,779 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9 +2023-10-31 21:26:00,779 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:26:00,780 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-10-31 21:26:00,780 INFO SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher +2023-10-31 21:26:01,322 INFO Thread-25 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/output.log +2023-10-31 21:26:01,322 INFO SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files +2023-10-31 21:26:01,323 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/output.log output.log +2023-10-31 21:26:01,323 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/requirements.txt requirements.txt +2023-10-31 21:26:01,329 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/conda-environment.yaml conda-environment.yaml +2023-10-31 21:26:01,336 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/wandb-summary.json wandb-summary.json +2023-10-31 21:26:01,343 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/config.yaml config.yaml +2023-10-31 21:26:01,350 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/wandb-metadata.json wandb-metadata.json +2023-10-31 21:26:01,351 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 10 +2023-10-31 21:26:01,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:26:01,358 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10 +2023-10-31 21:26:01,358 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:26:01,359 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-10-31 21:26:01,359 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 21:26:02,240 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/output.log +2023-10-31 21:26:02,508 INFO wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/conda-environment.yaml +2023-10-31 21:26:02,544 INFO wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/config.yaml +2023-10-31 21:26:02,565 INFO wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/requirements.txt +2023-10-31 21:26:02,584 INFO wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/files/wandb-summary.json +2023-10-31 21:26:02,785 INFO Thread-24 :34227 [sender.py:transition_state():613] send defer: 11 +2023-10-31 21:26:02,785 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:26:02,786 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11 +2023-10-31 21:26:02,786 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:26:02,786 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-10-31 21:26:02,787 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher +2023-10-31 21:26:02,787 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 12 +2023-10-31 21:26:02,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:26:02,787 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12 +2023-10-31 21:26:02,787 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:26:02,788 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-10-31 21:26:02,788 INFO SenderThread:34227 [file_stream.py:finish():595] file stream finish called +2023-10-31 21:26:04,049 INFO SenderThread:34227 [file_stream.py:finish():599] file stream finish is done +2023-10-31 21:26:04,050 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 13 +2023-10-31 21:26:04,050 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:26:04,050 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13 +2023-10-31 21:26:04,050 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:26:04,051 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-10-31 21:26:04,051 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 14 +2023-10-31 21:26:04,051 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:26:04,052 DEBUG SenderThread:34227 [sender.py:send():380] send: final +2023-10-31 21:26:04,052 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14 +2023-10-31 21:26:04,052 DEBUG SenderThread:34227 [sender.py:send():380] send: footer +2023-10-31 21:26:04,052 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:26:04,052 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-10-31 21:26:04,054 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:26:04,055 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 21:26:04,055 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 21:26:04,056 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info +2023-10-31 21:26:04,056 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary +2023-10-31 21:26:04,057 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history +2023-10-31 21:26:04,057 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: server_info +2023-10-31 21:26:04,058 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info +2023-10-31 21:26:04,375 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: job_info +2023-10-31 21:26:04,376 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown +2023-10-31 21:26:04,377 INFO HandlerThread:34227 [handler.py:finish():866] shutting down handler +2023-10-31 21:26:05,064 INFO WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/run-97cre3xd.wandb +2023-10-31 21:26:05,376 INFO SenderThread:34227 [sender.py:finish():1534] shutting down sender +2023-10-31 21:26:05,376 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 21:26:05,376 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231031_205340-97cre3xd/logs/debug.log b/wandb/run-20231031_205340-97cre3xd/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..73c26f7c56e7b54d4574616c93284b1e59a19c89 --- /dev/null +++ b/wandb/run-20231031_205340-97cre3xd/logs/debug.log @@ -0,0 +1,34 @@ +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070 +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/logs/debug.log +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd/logs/debug-internal.log +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_init.py:init():568] calling init triggers +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'RMSprop', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 31} +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_init.py:init():618] starting backend +2023-10-31 20:53:40,310 INFO MainThread:34070 [wandb_init.py:init():622] setting up manager +2023-10-31 20:53:40,312 INFO MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 20:53:40,314 INFO MainThread:34070 [wandb_init.py:init():628] backend started and connected +2023-10-31 20:53:40,316 INFO MainThread:34070 [wandb_init.py:init():720] updated telemetry +2023-10-31 20:53:40,317 INFO MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 20:53:41,633 INFO MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 20:53:42,146 INFO MainThread:34070 [wandb_run.py:_on_init():2229] got version response +2023-10-31 20:53:42,146 INFO MainThread:34070 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 20:53:48,894 INFO MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 20:53:48,895 INFO MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 20:53:48,895 INFO MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 20:53:48,895 INFO MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 20:53:48,897 INFO MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process +2023-10-31 21:25:56,750 INFO MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/97cre3xd +2023-10-31 21:25:56,750 INFO MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0 +2023-10-31 21:25:56,750 INFO MainThread:34070 [wandb_run.py:_restore():2151] restore +2023-10-31 21:25:56,750 INFO MainThread:34070 [wandb_run.py:_restore():2157] restore done +2023-10-31 21:26:05,379 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-10-31 21:26:05,379 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-10-31 21:26:05,387 INFO MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files diff --git a/wandb/run-20231031_205340-97cre3xd/run-97cre3xd.wandb b/wandb/run-20231031_205340-97cre3xd/run-97cre3xd.wandb new file mode 100644 index 0000000000000000000000000000000000000000..7e649c869a0a0543da0d242175f49a53f685dd97 Binary files /dev/null and b/wandb/run-20231031_205340-97cre3xd/run-97cre3xd.wandb differ diff --git a/wandb/run-20231031_212605-2ork0ogz/files/conda-environment.yaml b/wandb/run-20231031_212605-2ork0ogz/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231031_212605-2ork0ogz/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231031_212605-2ork0ogz/files/config.yaml b/wandb/run-20231031_212605-2ork0ogz/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..724dbf87d3fe62dcdaeafd9493d79554755489e0 --- /dev/null +++ b/wandb/run-20231031_212605-2ork0ogz/files/config.yaml @@ -0,0 +1,53 @@ +wandb_version: 1 + +optimizer: + desc: null + value: RMSprop +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 43 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698773165.859383 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 2 + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231031_212605-2ork0ogz/files/output.log b/wandb/run-20231031_212605-2ork0ogz/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3e3243d813ebee0d393d6b3779843d64e779d465 --- /dev/null +++ b/wandb/run-20231031_212605-2ork0ogz/files/output.log @@ -0,0 +1,3 @@ + +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth \ No newline at end of file diff --git a/wandb/run-20231031_212605-2ork0ogz/files/requirements.txt b/wandb/run-20231031_212605-2ork0ogz/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231031_212605-2ork0ogz/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_212605-2ork0ogz/files/wandb-metadata.json b/wandb/run-20231031_212605-2ork0ogz/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..98c370868c01d5e6f2bbcddea1d16e10a41caa54 --- /dev/null +++ b/wandb/run-20231031_212605-2ork0ogz/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T17:26:07.749060", + "startedAt": "2023-10-31T17:26:05.854379", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.564025, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.203, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.499, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.203, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.3, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.499, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.117, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.191, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.204, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.207, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.696, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.499, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.693, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.3, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.498, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.03, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.9234962463379 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_212605-2ork0ogz/files/wandb-summary.json b/wandb/run-20231031_212605-2ork0ogz/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5e4fe7cfaf81bbedfe3a02d7596dcf021fd078a8 --- /dev/null +++ b/wandb/run-20231031_212605-2ork0ogz/files/wandb-summary.json @@ -0,0 +1 @@ +{"Train Loss": 1.4998910427093506, "Train Accuracy": 0.48503533005714417, "Validation Loss": 1.2999918460845947, "Validation Accuracy": 0.5385968685150146, "_timestamp": 1698774874.6565597, "_runtime": 1708.7971765995026, "_step": 0, "_wandb": {"runtime": 1707}} \ No newline at end of file diff --git a/wandb/run-20231031_212605-2ork0ogz/logs/debug-internal.log b/wandb/run-20231031_212605-2ork0ogz/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7c2a0401f03edf85d5e61c1da2304c1f24f51e01 --- /dev/null +++ b/wandb/run-20231031_212605-2ork0ogz/logs/debug-internal.log @@ -0,0 +1,909 @@ +2023-10-31 21:26:06,381 INFO StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-10-31 21:26:06.380823 +2023-10-31 21:26:06,383 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status +2023-10-31 21:26:06,385 INFO WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/run-2ork0ogz.wandb +2023-10-31 21:26:06,387 DEBUG SenderThread:34227 [sender.py:send():380] send: header +2023-10-31 21:26:06,388 DEBUG SenderThread:34227 [sender.py:send():380] send: run +2023-10-31 21:26:07,227 INFO SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files +2023-10-31 21:26:07,227 INFO SenderThread:34227 [sender.py:_start_run_threads():1122] run started: 2ork0ogz with start time 1698773165.859383 +2023-10-31 21:26:07,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 21:26:07,249 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: check_version +2023-10-31 21:26:07,682 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 21:26:07,683 DEBUG HandlerThread:34227 [system_info.py:__init__():32] System info init +2023-10-31 21:26:07,683 DEBUG HandlerThread:34227 [system_info.py:__init__():47] System info init done +2023-10-31 21:26:07,683 INFO HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor +2023-10-31 21:26:07,684 INFO SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 21:26:07,684 INFO HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info +2023-10-31 21:26:07,684 INFO SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 21:26:07,685 INFO SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring +2023-10-31 21:26:07,686 INFO SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 21:26:07,687 INFO SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring +2023-10-31 21:26:07,688 INFO SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring +2023-10-31 21:26:07,748 DEBUG HandlerThread:34227 [system_info.py:probe():196] Probing system +2023-10-31 21:26:07,749 DEBUG HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 21:26:07,749 DEBUG HandlerThread:34227 [system_info.py:probe():244] Probing system done +2023-10-31 21:26:07,749 DEBUG HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T17:26:07.749060', 'startedAt': '2023-10-31T17:26:05.854379', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.564025, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.203, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.499, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.203, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.3, 'min': 1200.0, 'max': 3600.0}, {'current': 1.499, 'min': 1200.0, 'max': 3600.0}, {'current': 2.197, 'min': 1200.0, 'max': 3600.0}, {'current': 2.117, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.191, 'min': 1200.0, 'max': 3600.0}, {'current': 1.204, 'min': 1200.0, 'max': 3600.0}, {'current': 1.207, 'min': 1200.0, 'max': 3600.0}, {'current': 2.696, 'min': 1200.0, 'max': 3600.0}, {'current': 1.499, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.693, 'min': 1200.0, 'max': 3600.0}, {'current': 1.3, 'min': 1200.0, 'max': 3600.0}, {'current': 1.498, 'min': 1200.0, 'max': 3600.0}, {'current': 2.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.03, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9234962463379}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 21:26:07,749 INFO HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 21:26:07,749 INFO HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info +2023-10-31 21:26:07,749 DEBUG HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 21:26:07,750 DEBUG HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 21:26:07,750 DEBUG HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 21:26:08,231 INFO Thread-37 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/conda-environment.yaml +2023-10-31 21:26:08,232 INFO Thread-37 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/requirements.txt +2023-10-31 21:26:14,230 INFO Thread-37 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/conda-environment.yaml +2023-10-31 21:26:14,287 DEBUG HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 21:26:14,289 INFO HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 21:26:14,290 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:26:14,290 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 21:26:14,291 DEBUG SenderThread:34227 [sender.py:send():380] send: files +2023-10-31 21:26:14,291 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 21:26:14,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:26:14,306 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:26:14,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:26:14,819 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 21:26:15,230 INFO Thread-37 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/wandb-metadata.json +2023-10-31 21:26:15,538 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmp4s44var1wandb/ijwe4a60-wandb-metadata.json +2023-10-31 21:26:16,820 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:26:21,821 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:26:26,821 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:26:29,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:26:29,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:26:29,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:26:32,676 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:26:37,687 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:26:38,238 INFO Thread-37 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/config.yaml +2023-10-31 21:26:43,065 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:26:44,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:26:44,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:26:44,305 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:26:48,681 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:26:53,682 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:26:58,683 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:26:59,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:26:59,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:26:59,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:27:03,684 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:27:07,688 DEBUG SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 21:27:07,691 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:27:08,692 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:27:13,693 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:27:14,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:27:14,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:27:14,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:27:19,674 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:27:24,675 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:27:29,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:27:29,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:27:29,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:27:30,656 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:27:35,656 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:27:37,694 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:27:40,696 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:27:44,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:27:44,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:27:44,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:27:45,698 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:27:50,699 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:27:55,700 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:27:59,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:27:59,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:27:59,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:28:00,714 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:28:05,715 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:28:07,696 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:28:11,698 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:28:14,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:28:14,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:28:14,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:28:17,668 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:28:22,669 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:28:27,669 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:28:29,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:28:29,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:28:29,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:28:32,698 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:28:37,698 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:28:37,702 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:28:42,701 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:28:44,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:28:44,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:28:44,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:28:48,685 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:28:53,686 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:28:58,687 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:28:59,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:28:59,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:28:59,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:29:03,724 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:29:07,700 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:29:09,702 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:29:14,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:29:14,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:29:14,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:29:14,734 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:29:19,735 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:29:24,736 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:29:29,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:29:29,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:29:29,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:29:30,721 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:29:35,722 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:29:37,703 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:29:41,704 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:29:44,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:29:44,305 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:29:44,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:29:46,734 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:29:51,734 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:29:56,735 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:29:59,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:29:59,305 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:29:59,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:30:02,719 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:30:07,705 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:30:08,706 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:30:13,707 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:30:14,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:30:14,305 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:30:14,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:30:15,315 INFO Thread-37 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/output.log +2023-10-31 21:30:17,316 INFO Thread-37 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/output.log +2023-10-31 21:30:19,266 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:30:24,266 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:30:29,267 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:30:29,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:30:29,305 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:30:29,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:30:34,729 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:30:37,707 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:30:40,709 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:30:44,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:30:44,305 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:30:44,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:30:46,708 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:30:51,708 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:30:56,709 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:30:59,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:30:59,305 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:30:59,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:31:01,735 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:31:06,735 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:31:07,710 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:31:12,712 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:31:14,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:31:14,305 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:31:14,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:31:18,691 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:31:23,692 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:31:28,693 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:31:29,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:31:29,305 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:31:29,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:31:33,753 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:31:37,713 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:31:39,714 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:31:44,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:31:44,305 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:31:44,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:31:45,649 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:31:50,650 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:31:55,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:31:59,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:31:59,305 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:31:59,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:32:00,746 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:32:05,747 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:32:07,715 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:32:11,716 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:32:14,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:32:14,306 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:32:14,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:32:17,685 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:32:22,686 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:32:27,687 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:32:29,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:32:29,306 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:32:29,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:32:32,702 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:32:37,702 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:32:37,716 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:32:42,717 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:32:44,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:32:44,306 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:32:44,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:32:48,668 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:32:53,668 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:32:58,669 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:32:59,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:32:59,306 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:32:59,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:33:03,730 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:33:07,718 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:33:09,719 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:33:14,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:33:14,306 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:33:14,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:33:14,744 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:33:19,745 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:33:24,746 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:33:29,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:33:29,307 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:33:29,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:33:30,739 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:33:35,740 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:33:37,720 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:33:41,723 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:33:44,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:33:44,307 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:33:44,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:33:46,733 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:33:51,734 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:33:56,734 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:33:59,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:33:59,307 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:33:59,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:34:01,750 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:34:06,751 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:34:07,723 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:34:12,725 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:34:14,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:34:14,306 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:34:14,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:34:16,401 INFO Thread-37 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/output.log +2023-10-31 21:34:17,732 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:34:22,732 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:34:27,733 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:34:29,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:34:29,306 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:34:29,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:34:32,741 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:34:37,726 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:34:38,727 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:34:43,727 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:34:44,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:34:44,307 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:34:44,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:34:49,712 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:34:54,713 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:34:59,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:34:59,307 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:34:59,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:34:59,737 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:35:04,738 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:35:07,727 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:35:10,729 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:35:14,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:35:14,307 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:35:14,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:35:16,635 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:35:21,636 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:35:26,636 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:35:29,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:35:29,307 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:35:29,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:35:31,658 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:35:36,659 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:35:37,729 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:35:41,731 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:35:44,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:35:44,307 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:35:44,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:35:47,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:35:52,652 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:35:57,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:35:59,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:35:59,307 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:35:59,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:36:02,685 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:36:07,686 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:36:07,732 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:36:12,735 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:36:14,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:36:14,307 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:36:14,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:36:17,789 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:36:22,790 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:36:27,790 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:36:29,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:36:29,308 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:36:29,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:36:32,809 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:36:37,735 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:36:38,736 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:36:43,737 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:36:44,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:36:44,308 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:36:44,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:36:48,767 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:36:53,768 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:36:58,769 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:36:59,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:36:59,308 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:36:59,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:37:03,795 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:37:07,737 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:37:09,739 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:37:14,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:37:14,308 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:37:14,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:37:15,706 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:37:20,707 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:37:25,708 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:37:29,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:37:29,308 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:37:29,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:37:31,697 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:37:36,698 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:37:37,739 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:37:41,741 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:37:44,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:37:44,308 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:37:44,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:37:46,744 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:37:51,745 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:37:56,745 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:37:59,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:37:59,308 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:37:59,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:38:02,714 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:38:07,714 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:38:07,741 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:38:12,744 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:38:14,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:38:14,308 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:38:14,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:38:17,835 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:38:22,836 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:38:27,836 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:38:29,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:38:29,308 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:38:29,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:38:33,775 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:38:37,743 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:38:39,744 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:38:44,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:38:44,309 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:38:44,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:38:44,776 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:38:49,777 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:38:54,777 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:38:59,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:38:59,309 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:38:59,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:39:00,758 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:39:05,759 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:39:07,746 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:39:11,747 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:39:14,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:39:14,309 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:39:14,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:39:17,713 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:39:22,714 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:39:27,715 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:39:29,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:39:29,309 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:39:29,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:39:32,721 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:39:37,721 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:39:37,749 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:39:42,750 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:39:44,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:39:44,309 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:39:44,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:39:48,703 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:39:53,704 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:39:58,705 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:39:59,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:39:59,309 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:39:59,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:40:03,718 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:40:07,751 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:40:08,752 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:40:13,753 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:40:14,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:40:14,309 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:40:14,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:40:19,669 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:40:24,670 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:40:29,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:40:29,309 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:40:29,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:40:29,731 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:40:34,731 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:40:37,752 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:40:39,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:40:44,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:40:44,309 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:40:44,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:40:45,708 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:40:50,709 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:40:55,710 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:40:59,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:40:59,309 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:40:59,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:41:00,733 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:41:05,734 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:41:07,755 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:41:10,757 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:41:14,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:41:14,310 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:41:14,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:41:16,710 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:41:21,711 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:41:26,711 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:41:29,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:41:29,310 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:41:29,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:41:32,686 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:41:37,687 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:41:37,756 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:41:42,758 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:41:44,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:41:44,310 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:41:44,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:41:48,676 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:41:53,676 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:41:58,677 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:41:59,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:41:59,310 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:41:59,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:42:03,743 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:42:07,758 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:42:08,759 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:42:13,760 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:42:14,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:42:14,310 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:42:14,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:42:19,729 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:42:24,730 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:42:29,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:42:29,310 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:42:29,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:42:30,711 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:42:35,712 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:42:37,760 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:42:40,762 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:42:44,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:42:44,310 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:42:44,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:42:45,785 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:42:50,786 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:42:55,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:42:59,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:42:59,311 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:42:59,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:43:00,795 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:43:05,796 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:43:07,763 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:43:11,765 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:43:14,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:43:14,311 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:43:14,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:43:17,750 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:43:22,751 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:43:27,752 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:43:29,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:43:29,311 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:43:29,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:43:32,770 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:43:37,767 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:43:38,769 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:43:43,770 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:43:44,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:43:44,311 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:43:44,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:43:49,682 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:43:54,683 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:43:59,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:43:59,311 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:43:59,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:43:59,864 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:44:04,864 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:44:07,769 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:44:10,770 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:44:14,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:44:14,311 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:44:14,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:44:15,878 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:44:20,879 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:44:25,880 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:44:29,311 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:44:29,311 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:44:29,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:44:31,829 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:44:36,830 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:44:37,770 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:44:42,772 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:44:44,311 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:44:44,311 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:44:44,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:44:47,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:44:52,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:44:57,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:44:59,311 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:44:59,311 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:44:59,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:45:03,745 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:45:07,772 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:45:08,773 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:45:13,774 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:45:14,311 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:45:14,312 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:45:14,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:45:18,843 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:45:23,844 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:45:28,844 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:45:29,311 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:45:29,312 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:45:29,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:45:34,712 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:45:37,774 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:45:39,776 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:45:44,311 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:45:44,312 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:45:44,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:45:45,714 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:45:50,715 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:45:55,715 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:45:59,311 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:45:59,312 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:45:59,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:46:00,766 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:46:05,767 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:46:07,776 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:46:10,778 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:46:14,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:46:14,312 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:46:14,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:46:16,753 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:46:21,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:46:26,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:46:29,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:46:29,312 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:46:29,363 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:46:31,772 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:46:36,773 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:46:37,778 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:46:41,780 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:46:44,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:46:44,312 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:46:44,363 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:46:46,785 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:46:51,786 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:46:56,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:46:59,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:46:59,312 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:46:59,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:47:02,763 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:47:07,764 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:47:07,780 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:47:12,782 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:47:14,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:47:14,313 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:47:14,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:47:17,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:47:22,912 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:47:27,913 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:47:29,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:47:29,312 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:47:29,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:47:33,814 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:47:37,783 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:47:39,784 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:47:44,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:47:44,313 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:47:44,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:47:44,863 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:47:49,864 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:47:54,865 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:47:59,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:47:59,313 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:47:59,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:47:59,959 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:48:04,960 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:48:07,785 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:48:10,786 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:48:14,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:48:14,313 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:48:14,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:48:15,901 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:48:20,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:48:25,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:48:29,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:48:29,313 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:48:29,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:48:31,826 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:48:36,827 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:48:37,788 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:48:42,790 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:48:44,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:48:44,313 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:48:44,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:48:47,860 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:48:52,861 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:48:57,862 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:48:59,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:48:59,313 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:48:59,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:49:02,862 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:49:07,789 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:49:08,791 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:49:13,791 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:49:14,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:49:14,313 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:49:14,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:49:18,830 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:49:23,831 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:49:28,831 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:49:29,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:49:29,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:49:29,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:49:34,827 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:49:37,791 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:49:40,793 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:49:44,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:49:44,314 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:49:44,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:49:45,844 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:49:50,844 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:49:55,845 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:49:59,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:49:59,314 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:49:59,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:50:00,870 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:50:05,871 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:50:07,794 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:50:11,795 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:50:14,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:50:14,314 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:50:14,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:50:16,991 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:50:21,992 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:50:26,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:50:29,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:50:29,314 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:50:29,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:50:32,857 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:50:37,796 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:50:38,797 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:50:43,798 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:50:44,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:50:44,314 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:50:44,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:50:48,863 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:50:53,864 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:50:58,865 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:50:59,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:50:59,314 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:50:59,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:51:04,827 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:51:07,797 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:51:10,799 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:51:14,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:51:14,314 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:51:14,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:51:16,735 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:51:21,736 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:51:26,737 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:51:29,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:51:29,315 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:51:29,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:51:31,771 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:51:36,772 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:51:37,799 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:51:41,801 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:51:44,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:51:44,314 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:51:44,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:51:47,738 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:51:52,738 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:51:57,739 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:51:59,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:51:59,315 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:51:59,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:52:02,783 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:52:07,784 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:52:07,803 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:52:12,804 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:52:14,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:52:14,315 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:52:14,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:52:18,774 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:52:23,775 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:52:28,775 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:52:29,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:52:29,315 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:52:29,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:52:34,731 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:52:37,804 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:52:39,806 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:52:44,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:52:44,315 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:52:44,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:52:45,736 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:52:50,737 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:52:55,737 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:52:59,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:52:59,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:52:59,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:53:00,777 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:53:05,778 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:53:07,808 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:53:10,809 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:53:14,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:53:14,315 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:53:14,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:53:16,739 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:53:21,740 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:53:26,741 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:53:29,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:53:29,315 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:53:29,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:53:31,749 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:53:36,750 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:53:37,810 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:53:41,812 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:53:44,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:53:44,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:53:44,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:53:47,741 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:53:52,741 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:53:57,742 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:53:59,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:53:59,315 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:53:59,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:54:03,719 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:54:07,813 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:54:08,815 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:54:13,816 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:54:14,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:54:14,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:54:14,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:54:19,704 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:54:24,704 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:54:29,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:54:29,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:54:29,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:54:29,719 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:54:34,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history +2023-10-31 21:54:34,659 DEBUG SenderThread:34227 [sender.py:send():380] send: history +2023-10-31 21:54:34,659 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: summary_record +2023-10-31 21:54:34,659 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 21:54:34,697 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 21:54:34,697 DEBUG SenderThread:34227 [sender.py:send():380] send: exit +2023-10-31 21:54:34,697 INFO SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0 +2023-10-31 21:54:34,697 INFO SenderThread:34227 [sender.py:send_exit():587] handling runtime: 1707 +2023-10-31 21:54:34,698 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 21:54:34,698 INFO SenderThread:34227 [sender.py:send_exit():593] send defer +2023-10-31 21:54:34,699 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:34,699 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0 +2023-10-31 21:54:34,699 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:34,699 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-10-31 21:54:34,699 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 1 +2023-10-31 21:54:34,699 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:34,699 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1 +2023-10-31 21:54:34,700 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:34,700 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-10-31 21:54:34,700 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 2 +2023-10-31 21:54:34,700 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:34,700 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2 +2023-10-31 21:54:34,700 INFO HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor +2023-10-31 21:54:34,700 DEBUG SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-10-31 21:54:34,701 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor +2023-10-31 21:54:34,701 DEBUG SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-10-31 21:54:34,701 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor +2023-10-31 21:54:34,854 INFO Thread-37 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/wandb-summary.json +2023-10-31 21:54:36,456 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor +2023-10-31 21:54:36,457 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor +2023-10-31 21:54:36,457 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor +2023-10-31 21:54:36,457 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 21:54:36,458 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:54:36,459 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:36,459 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-10-31 21:54:36,459 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 3 +2023-10-31 21:54:36,459 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:54:36,459 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:36,461 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 21:54:36,461 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3 +2023-10-31 21:54:36,462 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:36,462 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-10-31 21:54:36,462 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 4 +2023-10-31 21:54:36,462 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:36,463 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4 +2023-10-31 21:54:36,463 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:36,463 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-10-31 21:54:36,463 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 5 +2023-10-31 21:54:36,463 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:36,463 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5 +2023-10-31 21:54:36,464 DEBUG SenderThread:34227 [sender.py:send():380] send: summary +2023-10-31 21:54:36,465 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 21:54:36,465 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:36,466 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-10-31 21:54:36,466 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 6 +2023-10-31 21:54:36,466 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:36,466 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6 +2023-10-31 21:54:36,466 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:36,467 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-10-31 21:54:36,472 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:54:36,659 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 21:54:36,856 INFO Thread-37 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/wandb-summary.json +2023-10-31 21:54:36,914 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 7 +2023-10-31 21:54:36,914 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 21:54:36,914 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:36,915 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7 +2023-10-31 21:54:36,915 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:36,915 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-10-31 21:54:37,660 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 21:54:37,856 INFO Thread-37 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/config.yaml +2023-10-31 21:54:38,806 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 8 +2023-10-31 21:54:38,806 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 21:54:38,806 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:38,807 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8 +2023-10-31 21:54:38,807 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:38,808 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-10-31 21:54:38,808 INFO SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact +2023-10-31 21:54:38,808 INFO SenderThread:34227 [job_builder.py:_get_source_type():404] no source found +2023-10-31 21:54:38,809 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 9 +2023-10-31 21:54:38,809 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:38,809 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9 +2023-10-31 21:54:38,809 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:38,809 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-10-31 21:54:38,810 INFO SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher +2023-10-31 21:54:38,857 INFO Thread-37 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/output.log +2023-10-31 21:54:38,857 INFO SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files +2023-10-31 21:54:38,858 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/output.log output.log +2023-10-31 21:54:38,858 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/requirements.txt requirements.txt +2023-10-31 21:54:38,865 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/conda-environment.yaml conda-environment.yaml +2023-10-31 21:54:38,873 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/wandb-summary.json wandb-summary.json +2023-10-31 21:54:38,874 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/config.yaml config.yaml +2023-10-31 21:54:38,881 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/wandb-metadata.json wandb-metadata.json +2023-10-31 21:54:38,886 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 10 +2023-10-31 21:54:38,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:38,895 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10 +2023-10-31 21:54:38,896 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:38,896 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-10-31 21:54:38,896 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 21:54:39,881 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/output.log +2023-10-31 21:54:40,123 INFO wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/config.yaml +2023-10-31 21:54:40,132 INFO wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/wandb-summary.json +2023-10-31 21:54:40,146 INFO wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/conda-environment.yaml +2023-10-31 21:54:40,181 INFO wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/files/requirements.txt +2023-10-31 21:54:40,381 INFO Thread-36 :34227 [sender.py:transition_state():613] send defer: 11 +2023-10-31 21:54:40,382 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:40,382 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11 +2023-10-31 21:54:40,383 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:40,383 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-10-31 21:54:40,383 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher +2023-10-31 21:54:40,383 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 12 +2023-10-31 21:54:40,384 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:40,384 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12 +2023-10-31 21:54:40,384 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:40,384 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-10-31 21:54:40,384 INFO SenderThread:34227 [file_stream.py:finish():595] file stream finish called +2023-10-31 21:54:40,985 INFO SenderThread:34227 [file_stream.py:finish():599] file stream finish is done +2023-10-31 21:54:40,985 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 13 +2023-10-31 21:54:40,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:40,985 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13 +2023-10-31 21:54:40,986 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:40,986 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-10-31 21:54:40,986 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 14 +2023-10-31 21:54:40,987 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 21:54:40,987 DEBUG SenderThread:34227 [sender.py:send():380] send: final +2023-10-31 21:54:40,987 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14 +2023-10-31 21:54:40,987 DEBUG SenderThread:34227 [sender.py:send():380] send: footer +2023-10-31 21:54:40,988 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 21:54:40,988 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-10-31 21:54:40,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:54:40,990 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 21:54:40,991 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 21:54:40,991 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info +2023-10-31 21:54:40,992 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary +2023-10-31 21:54:40,992 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history +2023-10-31 21:54:40,992 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: server_info +2023-10-31 21:54:40,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info +2023-10-31 21:54:41,321 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: job_info +2023-10-31 21:54:41,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown +2023-10-31 21:54:41,322 INFO HandlerThread:34227 [handler.py:finish():866] shutting down handler +2023-10-31 21:54:41,999 INFO WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/run-2ork0ogz.wandb +2023-10-31 21:54:42,322 INFO SenderThread:34227 [sender.py:finish():1534] shutting down sender +2023-10-31 21:54:42,322 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 21:54:42,322 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231031_212605-2ork0ogz/logs/debug.log b/wandb/run-20231031_212605-2ork0ogz/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9cac9ab2749670a17aca78f6773160461598a875 --- /dev/null +++ b/wandb/run-20231031_212605-2ork0ogz/logs/debug.log @@ -0,0 +1,34 @@ +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070 +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/logs/debug.log +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz/logs/debug-internal.log +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_init.py:init():568] calling init triggers +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'RMSprop', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 43} +2023-10-31 21:26:05,856 INFO MainThread:34070 [wandb_init.py:init():618] starting backend +2023-10-31 21:26:05,857 INFO MainThread:34070 [wandb_init.py:init():622] setting up manager +2023-10-31 21:26:05,858 INFO MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 21:26:05,859 INFO MainThread:34070 [wandb_init.py:init():628] backend started and connected +2023-10-31 21:26:05,862 INFO MainThread:34070 [wandb_init.py:init():720] updated telemetry +2023-10-31 21:26:05,862 INFO MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 21:26:07,248 INFO MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 21:26:07,667 INFO MainThread:34070 [wandb_run.py:_on_init():2229] got version response +2023-10-31 21:26:07,667 INFO MainThread:34070 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 21:26:14,303 INFO MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 21:26:14,304 INFO MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 21:26:14,304 INFO MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 21:26:14,304 INFO MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 21:26:14,306 INFO MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process +2023-10-31 21:54:34,657 INFO MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/2ork0ogz +2023-10-31 21:54:34,657 INFO MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0 +2023-10-31 21:54:34,657 INFO MainThread:34070 [wandb_run.py:_restore():2151] restore +2023-10-31 21:54:34,657 INFO MainThread:34070 [wandb_run.py:_restore():2157] restore done +2023-10-31 21:54:42,325 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-10-31 21:54:42,325 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-10-31 21:54:42,333 INFO MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files diff --git a/wandb/run-20231031_212605-2ork0ogz/run-2ork0ogz.wandb b/wandb/run-20231031_212605-2ork0ogz/run-2ork0ogz.wandb new file mode 100644 index 0000000000000000000000000000000000000000..7b2635a08de0d89132ec18c3fe19621be5668b0d Binary files /dev/null and b/wandb/run-20231031_212605-2ork0ogz/run-2ork0ogz.wandb differ diff --git a/wandb/run-20231031_215442-pegj5p38/files/conda-environment.yaml b/wandb/run-20231031_215442-pegj5p38/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231031_215442-pegj5p38/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231031_215442-pegj5p38/files/config.yaml b/wandb/run-20231031_215442-pegj5p38/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..348ad87e1c28a54ea81bf21a42873a1753a11197 --- /dev/null +++ b/wandb/run-20231031_215442-pegj5p38/files/config.yaml @@ -0,0 +1,53 @@ +wandb_version: 1 + +optimizer: + desc: null + value: SGD +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 33 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698774882.825091 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 2 + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231031_215442-pegj5p38/files/output.log b/wandb/run-20231031_215442-pegj5p38/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3209b44297ae1951fca862d06bd512462925a40e --- /dev/null +++ b/wandb/run-20231031_215442-pegj5p38/files/output.log @@ -0,0 +1,2 @@ + +Saving Model At: save_model/best_model.pth \ No newline at end of file diff --git a/wandb/run-20231031_215442-pegj5p38/files/requirements.txt b/wandb/run-20231031_215442-pegj5p38/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231031_215442-pegj5p38/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_215442-pegj5p38/files/wandb-metadata.json b/wandb/run-20231031_215442-pegj5p38/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bd6f1aa5d54f121bb37d446ad074eab9deec31ca --- /dev/null +++ b/wandb/run-20231031_215442-pegj5p38/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T17:54:44.756956", + "startedAt": "2023-10-31T17:54:42.820012", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.5048750000000006, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.543, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.588, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.216, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.399, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.201, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.593, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.502, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.526, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.719, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.53, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.397, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.469, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.705, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.501, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.9235534667969 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_215442-pegj5p38/files/wandb-summary.json b/wandb/run-20231031_215442-pegj5p38/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..82604caf3eb2cad3b5a2c087a78b00e5b56e005b --- /dev/null +++ b/wandb/run-20231031_215442-pegj5p38/files/wandb-summary.json @@ -0,0 +1 @@ +{"Train Loss": 1.490024209022522, "Train Accuracy": 0.480780690908432, "Validation Loss": 1.304742693901062, "Validation Accuracy": 0.5425158739089966, "_timestamp": 1698776349.9836571, "_runtime": 1467.158566236496, "_step": 0, "_wandb": {"runtime": 1465}} \ No newline at end of file diff --git a/wandb/run-20231031_215442-pegj5p38/logs/debug-internal.log b/wandb/run-20231031_215442-pegj5p38/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ea0a3fb8b55489b3cdae4fd8b161eda296cbde8e --- /dev/null +++ b/wandb/run-20231031_215442-pegj5p38/logs/debug-internal.log @@ -0,0 +1,811 @@ +2023-10-31 21:54:43,327 INFO StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-10-31 21:54:43.326390 +2023-10-31 21:54:43,329 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status +2023-10-31 21:54:43,330 INFO WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/run-pegj5p38.wandb +2023-10-31 21:54:43,334 DEBUG SenderThread:34227 [sender.py:send():380] send: header +2023-10-31 21:54:43,334 DEBUG SenderThread:34227 [sender.py:send():380] send: run +2023-10-31 21:54:44,193 INFO SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files +2023-10-31 21:54:44,193 INFO SenderThread:34227 [sender.py:_start_run_threads():1122] run started: pegj5p38 with start time 1698774882.825091 +2023-10-31 21:54:44,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 21:54:44,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: check_version +2023-10-31 21:54:44,690 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 21:54:44,691 DEBUG HandlerThread:34227 [system_info.py:__init__():32] System info init +2023-10-31 21:54:44,691 DEBUG HandlerThread:34227 [system_info.py:__init__():47] System info init done +2023-10-31 21:54:44,691 INFO HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor +2023-10-31 21:54:44,692 INFO SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 21:54:44,692 INFO HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info +2023-10-31 21:54:44,693 INFO SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 21:54:44,694 INFO SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring +2023-10-31 21:54:44,695 INFO SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 21:54:44,696 INFO SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring +2023-10-31 21:54:44,698 INFO SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring +2023-10-31 21:54:44,756 DEBUG HandlerThread:34227 [system_info.py:probe():196] Probing system +2023-10-31 21:54:44,757 DEBUG HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 21:54:44,757 DEBUG HandlerThread:34227 [system_info.py:probe():244] Probing system done +2023-10-31 21:54:44,757 DEBUG HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T17:54:44.756956', 'startedAt': '2023-10-31T17:54:42.820012', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.5048750000000006, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.543, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.588, 'min': 1200.0, 'max': 3600.0}, {'current': 2.216, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.399, 'min': 1200.0, 'max': 3600.0}, {'current': 1.201, 'min': 1200.0, 'max': 3600.0}, {'current': 1.593, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.502, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.526, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 2.719, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.53, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.397, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.469, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.705, 'min': 1200.0, 'max': 3600.0}, {'current': 1.501, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9235534667969}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 21:54:44,757 INFO HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 21:54:44,757 INFO HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info +2023-10-31 21:54:44,757 DEBUG HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 21:54:44,758 DEBUG HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 21:54:44,758 DEBUG HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 21:54:45,197 INFO Thread-49 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/conda-environment.yaml +2023-10-31 21:54:45,198 INFO Thread-49 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/requirements.txt +2023-10-31 21:54:51,196 INFO Thread-49 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/conda-environment.yaml +2023-10-31 21:54:51,300 DEBUG HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 21:54:51,301 INFO HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 21:54:51,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:54:51,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 21:54:51,302 DEBUG SenderThread:34227 [sender.py:send():380] send: files +2023-10-31 21:54:51,303 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 21:54:51,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:54:51,317 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:54:51,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:54:51,800 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 21:54:52,197 INFO Thread-49 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/wandb-metadata.json +2023-10-31 21:54:52,555 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmp_4rao6rgwandb/2z9ojxfe-wandb-metadata.json +2023-10-31 21:54:53,801 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:54:58,801 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:55:03,802 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:55:06,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:55:06,315 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:55:06,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:55:09,769 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:55:14,775 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:55:15,204 INFO Thread-49 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/config.yaml +2023-10-31 21:55:20,171 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:55:21,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:55:21,315 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:55:21,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:55:25,719 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:55:30,720 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:55:35,721 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:55:36,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:55:36,315 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:55:36,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:55:40,742 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:55:44,699 DEBUG SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 21:55:44,701 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:55:46,703 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:55:51,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:55:51,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:55:51,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:55:51,711 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:55:56,712 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:56:01,712 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:56:06,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:56:06,315 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:56:06,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:56:06,738 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:56:11,739 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:56:14,703 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:56:17,705 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:56:21,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:56:21,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:56:21,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:56:23,685 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:56:28,685 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:56:33,686 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:56:36,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:56:36,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:56:36,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:56:38,748 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:56:43,749 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:56:44,707 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:56:49,709 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:56:51,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:56:51,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:56:51,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:56:54,722 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:56:59,722 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:57:04,723 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:57:06,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:57:06,317 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:57:06,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:57:10,664 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:57:14,709 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:57:15,710 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:57:20,711 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:57:21,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:57:21,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:57:21,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:57:25,743 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:57:30,743 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:57:35,744 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:57:36,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:57:36,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:57:36,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:57:41,737 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:57:44,712 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:57:47,714 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:57:51,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:57:51,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:57:51,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:57:53,712 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:57:58,713 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:58:03,713 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:58:06,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:58:06,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:58:06,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:58:08,756 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:58:13,757 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:58:14,714 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:58:19,715 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:58:21,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:58:21,316 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:58:21,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:58:24,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:58:29,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:58:34,755 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:58:36,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:58:36,317 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:58:36,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:58:40,718 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:58:44,716 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:58:46,717 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:58:51,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:58:51,317 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:58:51,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:58:52,703 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:58:57,281 INFO Thread-49 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/output.log +2023-10-31 21:58:58,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:58:59,282 INFO Thread-49 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/output.log +2023-10-31 21:59:03,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:59:06,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:59:06,317 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:59:06,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:59:08,751 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:59:13,751 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:59:14,719 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:59:19,720 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:59:21,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:59:21,317 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:59:21,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:59:25,708 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:59:30,708 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:59:35,709 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:59:36,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:59:36,317 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:59:36,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:59:41,665 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:59:44,721 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 21:59:46,722 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:59:51,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 21:59:51,317 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 21:59:51,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 21:59:51,747 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 21:59:56,748 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:00:01,748 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:00:06,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:00:06,317 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:00:06,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:00:07,693 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:00:12,693 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:00:14,723 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:00:17,724 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:00:21,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:00:21,317 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:00:21,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:00:23,652 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:00:28,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:00:33,654 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:00:36,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:00:36,317 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:00:36,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:00:38,923 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:00:43,924 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:00:44,724 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:00:49,727 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:00:51,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:00:51,318 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:00:51,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:00:54,741 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:00:59,741 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:01:04,742 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:01:06,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:01:06,318 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:01:06,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:01:10,733 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:01:14,726 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:01:16,727 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:01:21,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:01:21,318 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:01:21,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:01:21,749 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:01:26,750 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:01:31,751 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:01:36,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:01:36,318 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:01:36,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:01:37,695 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:01:42,696 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:01:44,729 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:01:47,731 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:01:51,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:01:51,318 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:01:51,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:01:52,753 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:01:57,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:02:02,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:02:06,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:02:06,318 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:02:06,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:02:08,660 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:02:13,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:02:14,731 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:02:18,733 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:02:21,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:02:21,318 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:02:21,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:02:24,668 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:02:29,668 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:02:34,669 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:02:36,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:02:36,318 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:02:36,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:02:39,679 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:02:44,679 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:02:44,734 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:02:49,735 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:02:51,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:02:51,318 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:02:51,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:02:55,667 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:03:00,668 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:03:05,669 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:03:06,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:03:06,319 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:03:06,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:03:10,717 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:03:14,736 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:03:15,737 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:03:20,738 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:03:21,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:03:21,319 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:03:21,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:03:25,742 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:03:30,743 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:03:35,744 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:03:36,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:03:36,319 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:03:36,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:03:41,677 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:03:44,738 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:03:46,739 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:03:51,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:03:51,319 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:03:51,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:03:52,670 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:03:57,671 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:04:02,671 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:04:06,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:04:06,319 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:04:06,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:04:07,769 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:04:12,770 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:04:14,739 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:04:18,741 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:04:21,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:04:21,319 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:04:21,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:04:24,666 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:04:29,667 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:04:34,668 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:04:36,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:04:36,319 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:04:36,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:04:39,745 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:04:44,742 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:04:45,743 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:04:50,744 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:04:51,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:04:51,319 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:04:51,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:04:56,742 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:05:01,742 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:05:06,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:05:06,319 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:05:06,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:05:07,673 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:05:12,674 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:05:14,743 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:05:17,745 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:05:21,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:05:21,319 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:05:21,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:05:23,716 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:05:28,717 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:05:33,717 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:05:36,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:05:36,320 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:05:36,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:05:38,741 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:05:43,741 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:05:44,746 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:05:48,748 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:05:51,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:05:51,321 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:05:51,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:05:54,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:05:59,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:06:04,662 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:06:06,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:06:06,320 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:06:06,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:06:09,704 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:06:14,705 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:06:14,749 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:06:19,751 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:06:21,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:06:21,320 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:06:21,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:06:25,724 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:06:30,724 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:06:35,725 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:06:36,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:06:36,320 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:06:36,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:06:40,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:06:44,751 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:06:46,753 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:06:51,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:06:51,320 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:06:51,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:06:52,720 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:06:57,721 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:07:02,722 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:07:06,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:07:06,320 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:07:06,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:07:08,652 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:07:13,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:07:14,754 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:07:18,755 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:07:21,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:07:21,321 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:07:21,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:07:24,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:07:29,662 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:07:34,662 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:07:36,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:07:36,321 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:07:36,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:07:40,659 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:07:44,756 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:07:45,757 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:07:50,758 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:07:51,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:07:51,321 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:07:51,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:07:55,772 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:08:00,773 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:08:05,774 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:08:06,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:08:06,321 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:08:06,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:08:11,673 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:08:14,758 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:08:16,760 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:08:21,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:08:21,321 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:08:21,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:08:22,758 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:08:27,759 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:08:32,759 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:08:36,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:08:36,321 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:08:36,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:08:38,726 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:08:43,726 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:08:44,761 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:08:48,763 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:08:51,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:08:51,321 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:08:51,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:08:54,753 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:08:59,753 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:09:04,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:09:06,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:09:06,322 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:09:06,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:09:10,703 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:09:14,763 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:09:15,764 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:09:20,765 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:09:21,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:09:21,322 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:09:21,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:09:26,749 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:09:31,750 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:09:36,323 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:09:36,323 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:09:36,324 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:09:37,733 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:09:42,734 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:09:44,766 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:09:47,768 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:09:51,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:09:51,322 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:09:51,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:09:53,728 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:09:58,729 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:10:03,730 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:10:06,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:10:06,322 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:10:06,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:10:08,765 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:10:13,765 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:10:14,768 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:10:18,770 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:10:21,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:10:21,322 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:10:21,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:10:24,724 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:10:29,725 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:10:34,726 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:10:36,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:10:36,322 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:10:36,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:10:40,675 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:10:44,770 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:10:45,771 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:10:50,772 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:10:51,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:10:51,322 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:10:51,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:10:56,769 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:11:01,770 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:11:06,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:11:06,323 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:11:06,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:11:07,710 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:11:12,711 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:11:14,773 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:11:17,775 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:11:21,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:11:21,323 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:11:21,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:11:23,713 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:11:28,714 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:11:33,714 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:11:36,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:11:36,323 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:11:36,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:11:39,686 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:11:44,686 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:11:44,775 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:11:49,776 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:11:51,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:11:51,323 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:11:51,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:11:55,773 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:12:00,773 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:12:05,774 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:12:06,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:12:06,324 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:12:06,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:12:11,760 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:12:14,776 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:12:16,777 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:12:21,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:12:21,325 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:12:21,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:12:21,798 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:12:26,799 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:12:31,799 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:12:36,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:12:36,324 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:12:36,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:12:37,716 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:12:42,717 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:12:44,778 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:12:47,779 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:12:51,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:12:51,325 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:12:51,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:12:53,755 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:12:58,756 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:13:03,756 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:13:06,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:13:06,324 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:13:06,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:13:09,680 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:13:14,681 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:13:14,781 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:13:19,783 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:13:21,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:13:21,324 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:13:21,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:13:25,691 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:13:30,692 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:13:35,693 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:13:36,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:13:36,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:13:36,326 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:13:40,760 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:13:44,783 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:13:45,785 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:13:50,785 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:13:51,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:13:51,326 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:13:51,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:13:56,695 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:14:01,696 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:14:06,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:14:06,325 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:14:06,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:14:07,671 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:14:12,671 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:14:14,786 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:14:17,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:14:21,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:14:21,325 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:14:21,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:14:23,701 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:14:28,702 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:14:33,703 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:14:36,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:14:36,325 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:14:36,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:14:38,752 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:14:43,753 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:14:44,789 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:14:48,791 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:14:51,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:14:51,325 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:14:51,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:14:54,688 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:14:59,689 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:15:04,689 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:15:06,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:15:06,325 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:15:06,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:15:09,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:15:14,755 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:15:14,791 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:15:19,793 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:15:21,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:15:21,326 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:15:21,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:15:25,721 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:15:30,722 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:15:35,723 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:15:36,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:15:36,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:15:36,327 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:15:40,740 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:15:44,794 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:15:45,795 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:15:50,796 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:15:51,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:15:51,326 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:15:51,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:15:56,749 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:16:01,750 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:16:06,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:16:06,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:16:06,327 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:16:06,761 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:16:11,761 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:16:14,797 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:16:16,798 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:16:21,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:16:21,326 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:16:21,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:16:22,735 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:16:27,735 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:16:32,736 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:16:36,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:16:36,326 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:16:36,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:16:37,761 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:16:42,762 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:16:44,799 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:16:47,801 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:16:51,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:16:51,326 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:16:51,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:16:53,687 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:16:58,687 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:17:03,688 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:17:06,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:17:06,326 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:17:06,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:17:08,728 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:17:13,729 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:17:14,802 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:17:18,803 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:17:21,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:17:21,326 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:17:21,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:17:24,746 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:17:29,746 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:17:34,747 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:17:36,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:17:36,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:17:36,327 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:17:39,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:17:44,755 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:17:44,803 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:17:49,805 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:17:51,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:17:51,327 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:17:51,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:17:55,682 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:18:00,682 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:18:05,683 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:18:06,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:18:06,327 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:18:06,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:18:10,738 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:18:14,805 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:18:15,806 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:18:20,807 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:18:21,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:18:21,327 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:18:21,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:18:26,728 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:18:31,728 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:18:36,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:18:36,327 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:18:36,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:18:36,739 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:18:41,740 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:18:44,807 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:18:46,808 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:18:51,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:18:51,327 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:18:51,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:18:52,752 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:18:57,753 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:19:02,753 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:19:06,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:19:06,327 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:19:06,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:19:08,747 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:19:09,984 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history +2023-10-31 22:19:09,986 DEBUG SenderThread:34227 [sender.py:send():380] send: history +2023-10-31 22:19:09,986 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: summary_record +2023-10-31 22:19:09,987 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 22:19:10,025 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 22:19:10,026 DEBUG SenderThread:34227 [sender.py:send():380] send: exit +2023-10-31 22:19:10,026 INFO SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0 +2023-10-31 22:19:10,026 INFO SenderThread:34227 [sender.py:send_exit():587] handling runtime: 1465 +2023-10-31 22:19:10,027 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 22:19:10,027 INFO SenderThread:34227 [sender.py:send_exit():593] send defer +2023-10-31 22:19:10,027 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:10,028 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0 +2023-10-31 22:19:10,028 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:10,028 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-10-31 22:19:10,028 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 1 +2023-10-31 22:19:10,028 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:10,028 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1 +2023-10-31 22:19:10,029 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:10,029 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-10-31 22:19:10,029 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 2 +2023-10-31 22:19:10,029 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:10,029 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2 +2023-10-31 22:19:10,029 INFO HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor +2023-10-31 22:19:10,029 DEBUG SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-10-31 22:19:10,030 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor +2023-10-31 22:19:10,030 DEBUG SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-10-31 22:19:10,031 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor +2023-10-31 22:19:10,723 INFO Thread-49 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/wandb-summary.json +2023-10-31 22:19:11,798 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor +2023-10-31 22:19:11,799 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor +2023-10-31 22:19:11,799 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor +2023-10-31 22:19:11,799 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:19:11,800 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:11,800 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-10-31 22:19:11,800 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 3 +2023-10-31 22:19:11,801 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:19:11,801 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:11,802 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:19:11,802 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3 +2023-10-31 22:19:11,803 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:11,803 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-10-31 22:19:11,803 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 4 +2023-10-31 22:19:11,804 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:11,804 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4 +2023-10-31 22:19:11,804 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:11,804 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-10-31 22:19:11,804 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 5 +2023-10-31 22:19:11,804 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:11,804 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5 +2023-10-31 22:19:11,805 DEBUG SenderThread:34227 [sender.py:send():380] send: summary +2023-10-31 22:19:11,806 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 22:19:11,806 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:11,806 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-10-31 22:19:11,806 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 6 +2023-10-31 22:19:11,807 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:11,807 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6 +2023-10-31 22:19:11,807 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:11,807 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-10-31 22:19:11,812 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:19:11,986 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:19:12,423 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 7 +2023-10-31 22:19:12,423 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:19:12,424 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:12,424 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7 +2023-10-31 22:19:12,424 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:12,425 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-10-31 22:19:12,724 INFO Thread-49 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/config.yaml +2023-10-31 22:19:12,725 INFO Thread-49 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/wandb-summary.json +2023-10-31 22:19:12,986 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:19:13,614 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 8 +2023-10-31 22:19:13,614 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:19:13,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:13,615 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8 +2023-10-31 22:19:13,616 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:13,616 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-10-31 22:19:13,616 INFO SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact +2023-10-31 22:19:13,617 INFO SenderThread:34227 [job_builder.py:_get_source_type():404] no source found +2023-10-31 22:19:13,617 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 9 +2023-10-31 22:19:13,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:13,617 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9 +2023-10-31 22:19:13,618 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:13,618 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-10-31 22:19:13,618 INFO SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher +2023-10-31 22:19:13,725 INFO Thread-49 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/output.log +2023-10-31 22:19:13,726 INFO SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files +2023-10-31 22:19:13,726 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/output.log output.log +2023-10-31 22:19:13,726 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/requirements.txt requirements.txt +2023-10-31 22:19:13,733 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/conda-environment.yaml conda-environment.yaml +2023-10-31 22:19:13,741 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/wandb-summary.json wandb-summary.json +2023-10-31 22:19:13,747 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/config.yaml config.yaml +2023-10-31 22:19:13,755 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/wandb-metadata.json wandb-metadata.json +2023-10-31 22:19:13,756 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 10 +2023-10-31 22:19:13,757 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:13,763 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10 +2023-10-31 22:19:13,764 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:13,765 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-10-31 22:19:13,765 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 22:19:13,987 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:19:13,987 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:19:14,775 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/output.log +2023-10-31 22:19:14,987 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:19:14,988 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:19:15,303 INFO wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/conda-environment.yaml +2023-10-31 22:19:15,308 INFO wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/wandb-summary.json +2023-10-31 22:19:15,492 INFO wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/requirements.txt +2023-10-31 22:19:15,514 INFO wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/files/config.yaml +2023-10-31 22:19:15,714 INFO Thread-48 :34227 [sender.py:transition_state():613] send defer: 11 +2023-10-31 22:19:15,716 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:15,716 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11 +2023-10-31 22:19:15,717 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:15,717 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-10-31 22:19:15,717 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher +2023-10-31 22:19:15,718 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 12 +2023-10-31 22:19:15,718 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:15,718 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12 +2023-10-31 22:19:15,718 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:15,718 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-10-31 22:19:15,719 INFO SenderThread:34227 [file_stream.py:finish():595] file stream finish called +2023-10-31 22:19:15,988 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:19:16,414 INFO SenderThread:34227 [file_stream.py:finish():599] file stream finish is done +2023-10-31 22:19:16,414 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 13 +2023-10-31 22:19:16,414 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:19:16,415 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:16,415 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13 +2023-10-31 22:19:16,416 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:16,416 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-10-31 22:19:16,416 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 14 +2023-10-31 22:19:16,417 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:19:16,417 DEBUG SenderThread:34227 [sender.py:send():380] send: final +2023-10-31 22:19:16,417 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14 +2023-10-31 22:19:16,417 DEBUG SenderThread:34227 [sender.py:send():380] send: footer +2023-10-31 22:19:16,418 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:19:16,418 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-10-31 22:19:16,419 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:19:16,420 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:19:16,421 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:19:16,421 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info +2023-10-31 22:19:16,421 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary +2023-10-31 22:19:16,422 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history +2023-10-31 22:19:16,422 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: server_info +2023-10-31 22:19:16,423 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info +2023-10-31 22:19:16,787 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: job_info +2023-10-31 22:19:16,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown +2023-10-31 22:19:16,788 INFO HandlerThread:34227 [handler.py:finish():866] shutting down handler +2023-10-31 22:19:17,429 INFO WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/run-pegj5p38.wandb +2023-10-31 22:19:17,787 INFO SenderThread:34227 [sender.py:finish():1534] shutting down sender +2023-10-31 22:19:17,788 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 22:19:17,788 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231031_215442-pegj5p38/logs/debug.log b/wandb/run-20231031_215442-pegj5p38/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e111ec91aa381cd8eedff99c298ec23c3eec1f8f --- /dev/null +++ b/wandb/run-20231031_215442-pegj5p38/logs/debug.log @@ -0,0 +1,34 @@ +2023-10-31 21:54:42,821 INFO MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 21:54:42,821 INFO MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070 +2023-10-31 21:54:42,821 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 21:54:42,822 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 21:54:42,822 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 21:54:42,822 INFO MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 21:54:42,822 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 21:54:42,822 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 21:54:42,822 INFO MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/logs/debug.log +2023-10-31 21:54:42,822 INFO MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38/logs/debug-internal.log +2023-10-31 21:54:42,822 INFO MainThread:34070 [wandb_init.py:init():568] calling init triggers +2023-10-31 21:54:42,822 INFO MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'SGD', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 33} +2023-10-31 21:54:42,822 INFO MainThread:34070 [wandb_init.py:init():618] starting backend +2023-10-31 21:54:42,822 INFO MainThread:34070 [wandb_init.py:init():622] setting up manager +2023-10-31 21:54:42,824 INFO MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 21:54:42,824 INFO MainThread:34070 [wandb_init.py:init():628] backend started and connected +2023-10-31 21:54:42,828 INFO MainThread:34070 [wandb_init.py:init():720] updated telemetry +2023-10-31 21:54:42,828 INFO MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 21:54:44,214 INFO MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 21:54:44,677 INFO MainThread:34070 [wandb_run.py:_on_init():2229] got version response +2023-10-31 21:54:44,677 INFO MainThread:34070 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 21:54:51,315 INFO MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 21:54:51,315 INFO MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 21:54:51,315 INFO MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 21:54:51,316 INFO MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 21:54:51,317 INFO MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process +2023-10-31 22:19:09,984 INFO MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/pegj5p38 +2023-10-31 22:19:09,984 INFO MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0 +2023-10-31 22:19:09,984 INFO MainThread:34070 [wandb_run.py:_restore():2151] restore +2023-10-31 22:19:09,984 INFO MainThread:34070 [wandb_run.py:_restore():2157] restore done +2023-10-31 22:19:17,791 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-10-31 22:19:17,791 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-10-31 22:19:17,798 INFO MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files diff --git a/wandb/run-20231031_215442-pegj5p38/run-pegj5p38.wandb b/wandb/run-20231031_215442-pegj5p38/run-pegj5p38.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a94d4db992ac961641f581f8b09f3e6ef56777ff Binary files /dev/null and b/wandb/run-20231031_215442-pegj5p38/run-pegj5p38.wandb differ diff --git a/wandb/run-20231031_221918-gr6ablo3/files/conda-environment.yaml b/wandb/run-20231031_221918-gr6ablo3/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231031_221918-gr6ablo3/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231031_221918-gr6ablo3/files/config.yaml b/wandb/run-20231031_221918-gr6ablo3/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d9ee1562063836dcf07e9398ea97a7276ff788e --- /dev/null +++ b/wandb/run-20231031_221918-gr6ablo3/files/config.yaml @@ -0,0 +1,53 @@ +wandb_version: 1 + +optimizer: + desc: null + value: Adam +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 32 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698776358.279181 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 2 + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231031_221918-gr6ablo3/files/output.log b/wandb/run-20231031_221918-gr6ablo3/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3209b44297ae1951fca862d06bd512462925a40e --- /dev/null +++ b/wandb/run-20231031_221918-gr6ablo3/files/output.log @@ -0,0 +1,2 @@ + +Saving Model At: save_model/best_model.pth \ No newline at end of file diff --git a/wandb/run-20231031_221918-gr6ablo3/files/requirements.txt b/wandb/run-20231031_221918-gr6ablo3/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231031_221918-gr6ablo3/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_221918-gr6ablo3/files/wandb-metadata.json b/wandb/run-20231031_221918-gr6ablo3/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0b1333cdd1f442201caaea6dc0e6bbccaf24b5ad --- /dev/null +++ b/wandb/run-20231031_221918-gr6ablo3/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T18:19:20.201831", + "startedAt": "2023-10-31T18:19:18.274146", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.5269249999999999, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.519, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.206, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.496, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.501, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.205, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.215, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.689, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.204, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.444, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.348, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.498, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.499, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.499, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.205, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.204, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.221, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.207, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.459, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.69, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.342, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.206, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.9235801696777 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_221918-gr6ablo3/files/wandb-summary.json b/wandb/run-20231031_221918-gr6ablo3/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..98566a94255c73f833ba64187c0bf5133de250b2 --- /dev/null +++ b/wandb/run-20231031_221918-gr6ablo3/files/wandb-summary.json @@ -0,0 +1 @@ +{"Train Loss": 1.490789532661438, "Train Accuracy": 0.48707833886146545, "Validation Loss": 1.2940490245819092, "Validation Accuracy": 0.5406512022018433, "_timestamp": 1698777820.1073372, "_runtime": 1461.8281562328339, "_step": 0, "_wandb": {"runtime": 1460}} \ No newline at end of file diff --git a/wandb/run-20231031_221918-gr6ablo3/logs/debug-internal.log b/wandb/run-20231031_221918-gr6ablo3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c1af6c0ef8a73532e2f2b5da29354e96b2c19e26 --- /dev/null +++ b/wandb/run-20231031_221918-gr6ablo3/logs/debug-internal.log @@ -0,0 +1,810 @@ +2023-10-31 22:19:18,793 INFO StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-10-31 22:19:18.792197 +2023-10-31 22:19:18,795 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status +2023-10-31 22:19:18,796 INFO WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/run-gr6ablo3.wandb +2023-10-31 22:19:18,799 DEBUG SenderThread:34227 [sender.py:send():380] send: header +2023-10-31 22:19:18,800 DEBUG SenderThread:34227 [sender.py:send():380] send: run +2023-10-31 22:19:19,685 INFO SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files +2023-10-31 22:19:19,685 INFO SenderThread:34227 [sender.py:_start_run_threads():1122] run started: gr6ablo3 with start time 1698776358.279181 +2023-10-31 22:19:19,706 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 22:19:19,706 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: check_version +2023-10-31 22:19:20,120 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 22:19:20,121 DEBUG HandlerThread:34227 [system_info.py:__init__():32] System info init +2023-10-31 22:19:20,121 DEBUG HandlerThread:34227 [system_info.py:__init__():47] System info init done +2023-10-31 22:19:20,122 INFO HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor +2023-10-31 22:19:20,122 INFO SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 22:19:20,122 INFO HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info +2023-10-31 22:19:20,123 INFO SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 22:19:20,123 INFO SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring +2023-10-31 22:19:20,124 INFO SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 22:19:20,125 INFO SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring +2023-10-31 22:19:20,126 INFO SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring +2023-10-31 22:19:20,201 DEBUG HandlerThread:34227 [system_info.py:probe():196] Probing system +2023-10-31 22:19:20,202 DEBUG HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 22:19:20,202 DEBUG HandlerThread:34227 [system_info.py:probe():244] Probing system done +2023-10-31 22:19:20,202 DEBUG HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T18:19:20.201831', 'startedAt': '2023-10-31T18:19:18.274146', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.5269249999999999, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.519, 'min': 1200.0, 'max': 3600.0}, {'current': 2.206, 'min': 1200.0, 'max': 3600.0}, {'current': 1.496, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.501, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.215, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.689, 'min': 1200.0, 'max': 3600.0}, {'current': 1.204, 'min': 1200.0, 'max': 3600.0}, {'current': 1.444, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.348, 'min': 1200.0, 'max': 3600.0}, {'current': 1.599, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.498, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.499, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.499, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.204, 'min': 1200.0, 'max': 3600.0}, {'current': 1.221, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.207, 'min': 1200.0, 'max': 3600.0}, {'current': 1.459, 'min': 1200.0, 'max': 3600.0}, {'current': 2.69, 'min': 1200.0, 'max': 3600.0}, {'current': 1.342, 'min': 1200.0, 'max': 3600.0}, {'current': 1.599, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.206, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9235801696777}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 22:19:20,202 INFO HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 22:19:20,202 INFO HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info +2023-10-31 22:19:20,202 DEBUG HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 22:19:20,203 DEBUG HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 22:19:20,203 DEBUG HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 22:19:20,689 INFO Thread-61 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/requirements.txt +2023-10-31 22:19:20,690 INFO Thread-61 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/conda-environment.yaml +2023-10-31 22:19:27,594 DEBUG HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 22:19:27,596 INFO HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 22:19:27,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:19:27,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 22:19:27,598 DEBUG SenderThread:34227 [sender.py:send():380] send: files +2023-10-31 22:19:27,598 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 22:19:27,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:19:27,613 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:19:27,613 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:19:27,689 INFO Thread-61 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/conda-environment.yaml +2023-10-31 22:19:27,689 INFO Thread-61 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/wandb-metadata.json +2023-10-31 22:19:28,113 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 22:19:28,841 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmpgjfk8isdwandb/1zvfa78z-wandb-metadata.json +2023-10-31 22:19:29,113 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:19:34,114 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:19:39,115 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:19:42,609 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:19:42,610 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:19:42,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:19:44,931 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:19:49,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:19:50,696 INFO Thread-61 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/config.yaml +2023-10-31 22:19:55,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:19:57,609 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:19:57,610 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:19:57,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:20:01,037 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:20:06,037 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:20:11,038 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:20:12,609 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:20:12,610 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:20:12,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:20:16,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:20:20,126 DEBUG SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 22:20:20,129 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:20:22,130 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:20:27,131 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:20:27,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:20:27,611 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:20:27,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:20:33,038 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:20:38,039 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:20:42,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:20:42,611 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:20:42,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:20:43,956 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:20:48,957 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:20:50,132 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:20:54,133 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:20:57,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:20:57,611 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:20:57,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:21:00,032 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:21:05,033 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:21:10,034 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:21:12,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:21:12,611 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:21:12,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:21:15,046 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:21:20,047 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:21:20,134 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:21:25,136 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:21:27,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:21:27,611 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:21:27,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:21:31,040 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:21:36,041 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:21:41,041 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:21:42,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:21:42,611 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:21:42,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:21:47,034 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:21:50,136 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:21:52,138 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:21:57,138 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:21:57,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:21:57,611 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:21:57,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:22:03,021 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:22:08,021 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:22:12,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:22:12,611 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:22:12,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:22:13,956 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:22:18,956 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:22:20,138 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:22:24,139 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:22:27,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:22:27,611 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:22:27,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:22:29,996 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:22:34,997 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:22:39,997 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:22:42,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:22:42,612 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:22:42,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:22:45,988 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:22:50,141 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:22:51,143 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:22:56,143 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:22:57,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:22:57,611 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:22:57,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:23:01,970 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:23:06,971 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:23:11,971 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:23:12,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:23:12,611 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:23:12,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:23:17,047 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:23:20,143 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:23:22,145 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:23:27,146 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:23:27,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:23:27,612 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:23:27,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:23:32,971 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:23:34,771 INFO Thread-61 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/output.log +2023-10-31 22:23:36,772 INFO Thread-61 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/output.log +2023-10-31 22:23:38,746 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:23:42,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:23:42,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:23:42,612 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:23:43,997 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:23:48,998 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:23:50,145 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:23:54,147 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:23:57,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:23:57,612 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:23:57,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:24:00,014 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:24:05,015 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:24:10,016 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:24:12,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:24:12,612 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:24:12,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:24:15,056 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:24:20,056 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:24:20,147 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:24:25,148 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:24:27,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:24:27,612 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:24:27,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:24:31,022 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:24:36,023 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:24:41,023 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:24:42,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:24:42,612 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:24:42,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:24:46,978 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:24:50,149 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:24:52,151 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:24:57,151 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:24:57,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:24:57,613 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:24:57,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:25:03,027 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:25:08,027 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:25:12,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:25:12,612 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:25:12,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:25:13,032 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:25:18,032 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:25:20,150 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:25:23,152 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:25:27,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:25:27,613 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:25:27,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:25:28,964 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:25:33,965 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:25:38,966 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:25:42,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:25:42,613 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:25:42,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:25:43,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:25:48,986 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:25:50,153 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:25:54,154 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:25:57,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:25:57,613 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:25:57,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:25:59,992 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:26:04,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:26:09,994 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:26:12,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:26:12,613 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:26:12,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:26:15,033 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:26:20,034 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:26:20,156 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:26:25,157 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:26:27,613 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:26:27,613 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:26:27,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:26:31,056 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:26:36,057 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:26:41,058 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:26:42,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:26:42,613 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:26:42,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:26:46,938 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:26:50,158 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:26:52,159 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:26:57,160 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:26:57,614 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:26:57,614 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:26:57,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:27:02,966 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:27:07,966 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:27:12,613 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:27:12,613 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:27:12,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:27:13,963 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:27:18,964 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:27:20,160 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:27:24,161 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:27:27,613 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:27:27,613 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:27:27,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:27:30,019 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:27:35,020 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:27:40,021 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:27:42,613 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:27:42,613 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:27:42,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:27:46,008 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:27:50,161 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:27:51,163 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:27:56,164 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:27:57,614 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:27:57,614 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:27:57,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:28:02,011 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:28:07,011 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:28:12,012 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:28:12,613 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:28:12,614 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:28:12,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:28:17,995 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:28:20,163 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:28:23,165 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:28:27,614 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:28:27,614 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:28:27,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:28:28,974 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:28:33,975 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:28:38,976 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:28:42,614 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:28:42,615 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:28:42,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:28:44,033 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:28:49,034 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:28:50,167 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:28:54,169 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:28:57,614 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:28:57,615 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:28:57,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:29:00,006 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:29:05,006 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:29:10,007 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:29:12,614 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:29:12,614 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:29:12,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:29:15,016 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:29:20,017 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:29:20,169 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:29:25,171 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:29:27,614 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:29:27,614 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:29:27,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:29:31,057 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:29:36,057 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:29:41,058 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:29:42,614 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:29:42,615 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:29:42,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:29:47,011 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:29:50,171 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:29:52,172 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:29:57,173 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:29:57,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:29:57,615 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:29:57,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:30:02,979 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:30:07,979 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:30:12,614 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:30:12,615 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:30:12,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:30:13,022 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:30:18,023 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:30:20,173 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:30:23,174 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:30:27,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:30:27,615 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:30:27,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:30:28,980 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:30:33,980 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:30:38,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:30:42,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:30:42,615 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:30:42,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:30:44,974 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:30:49,975 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:30:50,175 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:30:55,177 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:30:57,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:30:57,615 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:30:57,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:31:01,005 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:31:06,006 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:31:11,007 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:31:12,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:31:12,616 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:31:12,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:31:16,014 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:31:20,176 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:31:21,178 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:31:26,178 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:31:27,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:31:27,616 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:31:27,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:31:31,972 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:31:36,973 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:31:41,973 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:31:42,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:31:42,616 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:31:42,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:31:47,035 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:31:50,178 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:31:52,180 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:31:57,180 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:31:57,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:31:57,616 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:31:57,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:32:02,979 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:32:07,979 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:32:12,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:32:12,616 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:32:12,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:32:12,986 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:32:17,986 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:32:20,180 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:32:23,182 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:32:27,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:32:27,616 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:32:27,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:32:28,988 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:32:33,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:32:38,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:32:42,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:32:42,616 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:32:42,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:32:44,056 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:32:49,056 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:32:50,182 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:32:54,184 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:32:57,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:32:57,616 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:32:57,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:33:00,061 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:33:05,062 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:33:10,063 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:33:12,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:33:12,616 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:33:12,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:33:16,056 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:33:20,184 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:33:21,186 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:33:26,186 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:33:27,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:33:27,616 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:33:27,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:33:32,037 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:33:37,037 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:33:42,038 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:33:42,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:33:42,616 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:33:42,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:33:47,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:33:50,187 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:33:53,189 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:33:57,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:33:57,617 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:33:57,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:33:59,052 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:34:04,053 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:34:09,054 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:34:12,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:34:12,617 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:34:12,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:34:14,963 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:34:19,964 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:34:20,191 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:34:25,193 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:34:27,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:34:27,617 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:34:27,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:34:31,053 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:34:36,053 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:34:41,054 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:34:42,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:34:42,617 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:34:42,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:34:46,961 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:34:50,193 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:34:52,194 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:34:57,195 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:34:57,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:34:57,617 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:34:57,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:35:03,016 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:35:08,017 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:35:12,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:35:12,617 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:35:12,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:35:13,990 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:35:18,991 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:35:20,195 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:35:24,196 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:35:27,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:35:27,617 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:35:27,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:35:29,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:35:34,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:35:39,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:35:42,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:35:42,617 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:35:42,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:35:45,039 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:35:50,040 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:35:50,196 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:35:55,198 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:35:57,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:35:57,617 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:35:57,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:36:01,021 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:36:06,022 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:36:11,022 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:36:12,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:36:12,618 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:36:12,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:36:16,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:36:20,198 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:36:22,200 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:36:27,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:36:27,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:36:27,617 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:36:27,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:36:32,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:36:37,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:36:42,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:36:42,618 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:36:42,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:36:43,022 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:36:48,022 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:36:50,201 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:36:53,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:36:57,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:36:57,618 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:36:57,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:36:59,005 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:37:04,006 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:37:09,006 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:37:12,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:37:12,618 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:37:12,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:37:14,044 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:37:19,044 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:37:20,202 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:37:24,203 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:37:27,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:37:27,618 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:37:27,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:37:29,968 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:37:34,969 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:37:39,969 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:37:42,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:37:42,618 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:37:42,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:37:45,010 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:37:50,010 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:37:50,204 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:37:55,206 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:37:57,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:37:57,618 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:37:57,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:38:00,990 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:38:05,990 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:38:10,991 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:38:12,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:38:12,618 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:38:12,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:38:16,048 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:38:20,206 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:38:21,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:38:26,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:38:27,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:38:27,619 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:38:27,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:38:31,968 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:38:36,968 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:38:41,969 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:38:42,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:38:42,619 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:38:42,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:38:46,995 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:38:50,209 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:38:52,211 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:38:57,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:38:57,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:38:57,619 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:38:57,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:39:03,022 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:39:08,023 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:39:12,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:39:12,619 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:39:12,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:39:13,957 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:39:18,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:39:20,211 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:39:24,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:39:27,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:39:27,619 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:39:27,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:39:30,036 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:39:35,037 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:39:40,038 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:39:42,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:39:42,620 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:39:42,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:39:45,984 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:39:50,214 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:39:51,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:39:56,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:39:57,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:39:57,619 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:39:57,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:40:01,957 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:40:06,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:40:11,959 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:40:12,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:40:12,619 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:40:12,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:40:17,061 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:40:20,215 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:40:22,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:40:27,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:40:27,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:40:27,619 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:40:27,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:40:33,029 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:40:38,030 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:40:42,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:40:42,620 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:40:42,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:40:43,992 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:40:48,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:40:50,218 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:40:54,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:40:57,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:40:57,620 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:40:57,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:41:00,013 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:41:05,013 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:41:10,015 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:41:12,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:41:12,620 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:41:12,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:41:15,040 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:41:20,041 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:41:20,220 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:41:25,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:41:27,621 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:41:27,621 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:41:27,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:41:31,010 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:41:36,011 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:41:41,011 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:41:42,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:41:42,620 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:41:42,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:41:46,043 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:41:50,223 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:41:51,224 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:41:56,225 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:41:57,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:41:57,620 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:41:57,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:42:02,062 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:42:07,063 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:42:12,064 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:42:12,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:42:12,620 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:42:12,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:42:18,055 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:42:20,224 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:42:23,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:42:27,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:42:27,621 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:42:27,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:42:28,991 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:42:33,991 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:42:38,992 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:42:42,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:42:42,621 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:42:42,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:42:44,030 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:42:49,031 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:42:50,226 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:42:54,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:42:57,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:42:57,621 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:42:57,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:42:59,970 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:43:04,971 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:43:09,971 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:43:12,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:43:12,621 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:43:12,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:43:15,970 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:43:20,228 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:43:21,229 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:43:26,230 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:43:27,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:43:27,621 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:43:27,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:43:32,047 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:43:37,047 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:43:40,107 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history +2023-10-31 22:43:40,110 DEBUG SenderThread:34227 [sender.py:send():380] send: history +2023-10-31 22:43:40,110 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: summary_record +2023-10-31 22:43:40,110 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 22:43:40,149 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 22:43:40,149 DEBUG SenderThread:34227 [sender.py:send():380] send: exit +2023-10-31 22:43:40,149 INFO SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0 +2023-10-31 22:43:40,149 INFO SenderThread:34227 [sender.py:send_exit():587] handling runtime: 1460 +2023-10-31 22:43:40,150 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 22:43:40,150 INFO SenderThread:34227 [sender.py:send_exit():593] send defer +2023-10-31 22:43:40,150 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:40,150 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0 +2023-10-31 22:43:40,151 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:40,151 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-10-31 22:43:40,151 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 1 +2023-10-31 22:43:40,151 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:40,151 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1 +2023-10-31 22:43:40,152 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:40,152 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-10-31 22:43:40,152 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 2 +2023-10-31 22:43:40,152 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:40,152 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2 +2023-10-31 22:43:40,152 INFO HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor +2023-10-31 22:43:40,152 DEBUG SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-10-31 22:43:40,153 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor +2023-10-31 22:43:40,153 DEBUG SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-10-31 22:43:40,154 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor +2023-10-31 22:43:40,200 INFO Thread-61 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/wandb-summary.json +2023-10-31 22:43:41,805 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor +2023-10-31 22:43:41,806 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor +2023-10-31 22:43:41,806 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor +2023-10-31 22:43:41,807 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:43:41,807 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:41,807 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-10-31 22:43:41,807 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 3 +2023-10-31 22:43:41,808 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:43:41,808 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:43:41,809 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:41,809 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3 +2023-10-31 22:43:41,809 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:41,810 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-10-31 22:43:41,810 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 4 +2023-10-31 22:43:41,810 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:41,810 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4 +2023-10-31 22:43:41,810 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:41,810 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-10-31 22:43:41,810 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 5 +2023-10-31 22:43:41,810 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:41,811 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5 +2023-10-31 22:43:41,811 DEBUG SenderThread:34227 [sender.py:send():380] send: summary +2023-10-31 22:43:41,811 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 22:43:41,812 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:41,812 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-10-31 22:43:41,812 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 6 +2023-10-31 22:43:41,812 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:41,812 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6 +2023-10-31 22:43:41,812 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:41,813 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-10-31 22:43:41,818 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:43:42,109 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:43:42,209 INFO Thread-61 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/wandb-summary.json +2023-10-31 22:43:42,209 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 7 +2023-10-31 22:43:42,209 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:43:42,209 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:42,210 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7 +2023-10-31 22:43:42,210 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:42,210 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-10-31 22:43:43,110 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:43:43,209 INFO Thread-61 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/config.yaml +2023-10-31 22:43:44,025 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 8 +2023-10-31 22:43:44,025 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:43:44,026 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:44,026 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8 +2023-10-31 22:43:44,027 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:44,027 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-10-31 22:43:44,027 INFO SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact +2023-10-31 22:43:44,028 INFO SenderThread:34227 [job_builder.py:_get_source_type():404] no source found +2023-10-31 22:43:44,028 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 9 +2023-10-31 22:43:44,028 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:44,028 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9 +2023-10-31 22:43:44,028 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:44,029 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-10-31 22:43:44,029 INFO SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher +2023-10-31 22:43:44,110 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:43:44,209 INFO Thread-61 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/output.log +2023-10-31 22:43:44,210 INFO SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files +2023-10-31 22:43:44,211 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/output.log output.log +2023-10-31 22:43:44,211 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/requirements.txt requirements.txt +2023-10-31 22:43:44,218 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/conda-environment.yaml conda-environment.yaml +2023-10-31 22:43:44,219 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/wandb-summary.json wandb-summary.json +2023-10-31 22:43:44,226 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/config.yaml config.yaml +2023-10-31 22:43:44,232 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/wandb-metadata.json wandb-metadata.json +2023-10-31 22:43:44,233 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 10 +2023-10-31 22:43:44,238 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:43:44,238 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:44,244 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10 +2023-10-31 22:43:44,245 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:44,245 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-10-31 22:43:44,245 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 22:43:45,111 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:43:45,112 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:43:45,124 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/output.log +2023-10-31 22:43:45,373 INFO wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/conda-environment.yaml +2023-10-31 22:43:45,414 INFO wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/wandb-summary.json +2023-10-31 22:43:45,473 INFO wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/config.yaml +2023-10-31 22:43:46,112 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:43:46,112 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:43:46,700 INFO wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/files/requirements.txt +2023-10-31 22:43:46,901 INFO Thread-60 :34227 [sender.py:transition_state():613] send defer: 11 +2023-10-31 22:43:46,901 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:46,901 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11 +2023-10-31 22:43:46,902 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:46,902 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-10-31 22:43:46,902 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher +2023-10-31 22:43:46,902 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 12 +2023-10-31 22:43:46,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:46,903 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12 +2023-10-31 22:43:46,903 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:46,903 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-10-31 22:43:46,903 INFO SenderThread:34227 [file_stream.py:finish():595] file stream finish called +2023-10-31 22:43:47,112 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:43:48,000 INFO SenderThread:34227 [file_stream.py:finish():599] file stream finish is done +2023-10-31 22:43:48,000 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 13 +2023-10-31 22:43:48,000 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:43:48,000 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:48,001 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13 +2023-10-31 22:43:48,001 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:43:48,001 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:48,002 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-10-31 22:43:48,002 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 14 +2023-10-31 22:43:48,003 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 22:43:48,003 DEBUG SenderThread:34227 [sender.py:send():380] send: final +2023-10-31 22:43:48,003 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14 +2023-10-31 22:43:48,003 DEBUG SenderThread:34227 [sender.py:send():380] send: footer +2023-10-31 22:43:48,004 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 22:43:48,004 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-10-31 22:43:48,005 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:43:48,006 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 22:43:48,006 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 22:43:48,007 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info +2023-10-31 22:43:48,007 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary +2023-10-31 22:43:48,008 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history +2023-10-31 22:43:48,008 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: server_info +2023-10-31 22:43:48,009 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info +2023-10-31 22:43:48,339 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: job_info +2023-10-31 22:43:48,341 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown +2023-10-31 22:43:48,341 INFO HandlerThread:34227 [handler.py:finish():866] shutting down handler +2023-10-31 22:43:49,015 INFO WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/run-gr6ablo3.wandb +2023-10-31 22:43:49,340 INFO SenderThread:34227 [sender.py:finish():1534] shutting down sender +2023-10-31 22:43:49,340 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 22:43:49,340 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231031_221918-gr6ablo3/logs/debug.log b/wandb/run-20231031_221918-gr6ablo3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..25f3ba4a045d81ed318c1f5f8e14ac5a47791800 --- /dev/null +++ b/wandb/run-20231031_221918-gr6ablo3/logs/debug.log @@ -0,0 +1,34 @@ +2023-10-31 22:19:18,275 INFO MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 22:19:18,275 INFO MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070 +2023-10-31 22:19:18,276 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 22:19:18,276 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 22:19:18,276 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 22:19:18,276 INFO MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 22:19:18,276 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 22:19:18,276 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 22:19:18,276 INFO MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/logs/debug.log +2023-10-31 22:19:18,276 INFO MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3/logs/debug-internal.log +2023-10-31 22:19:18,276 INFO MainThread:34070 [wandb_init.py:init():568] calling init triggers +2023-10-31 22:19:18,276 INFO MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'Adam', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 32} +2023-10-31 22:19:18,276 INFO MainThread:34070 [wandb_init.py:init():618] starting backend +2023-10-31 22:19:18,276 INFO MainThread:34070 [wandb_init.py:init():622] setting up manager +2023-10-31 22:19:18,278 INFO MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 22:19:18,279 INFO MainThread:34070 [wandb_init.py:init():628] backend started and connected +2023-10-31 22:19:18,282 INFO MainThread:34070 [wandb_init.py:init():720] updated telemetry +2023-10-31 22:19:18,282 INFO MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 22:19:19,705 INFO MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 22:19:20,107 INFO MainThread:34070 [wandb_run.py:_on_init():2229] got version response +2023-10-31 22:19:20,107 INFO MainThread:34070 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 22:19:27,610 INFO MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 22:19:27,610 INFO MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 22:19:27,610 INFO MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 22:19:27,611 INFO MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 22:19:27,612 INFO MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process +2023-10-31 22:43:40,107 INFO MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/gr6ablo3 +2023-10-31 22:43:40,107 INFO MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0 +2023-10-31 22:43:40,108 INFO MainThread:34070 [wandb_run.py:_restore():2151] restore +2023-10-31 22:43:40,108 INFO MainThread:34070 [wandb_run.py:_restore():2157] restore done +2023-10-31 22:43:49,343 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-10-31 22:43:49,343 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-10-31 22:43:49,351 INFO MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files diff --git a/wandb/run-20231031_221918-gr6ablo3/run-gr6ablo3.wandb b/wandb/run-20231031_221918-gr6ablo3/run-gr6ablo3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..69ea5867ae53aed20233e5b1099ca4afc1238eed Binary files /dev/null and b/wandb/run-20231031_221918-gr6ablo3/run-gr6ablo3.wandb differ diff --git a/wandb/run-20231031_224349-6h4rftgh/files/conda-environment.yaml b/wandb/run-20231031_224349-6h4rftgh/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231031_224349-6h4rftgh/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231031_224349-6h4rftgh/files/config.yaml b/wandb/run-20231031_224349-6h4rftgh/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46fe4e890c4e445984b33d2642ee943a83be3d89 --- /dev/null +++ b/wandb/run-20231031_224349-6h4rftgh/files/config.yaml @@ -0,0 +1,53 @@ +wandb_version: 1 + +optimizer: + desc: null + value: RMSprop +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 32 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698777829.831113 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 2 + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231031_224349-6h4rftgh/files/output.log b/wandb/run-20231031_224349-6h4rftgh/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..eb32e21e6438c3c494861db484183acb094e981c --- /dev/null +++ b/wandb/run-20231031_224349-6h4rftgh/files/output.log @@ -0,0 +1,5 @@ + +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth \ No newline at end of file diff --git a/wandb/run-20231031_224349-6h4rftgh/files/requirements.txt b/wandb/run-20231031_224349-6h4rftgh/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231031_224349-6h4rftgh/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_224349-6h4rftgh/files/wandb-metadata.json b/wandb/run-20231031_224349-6h4rftgh/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..373c915ec07743eea9b607f71b4d9cea7fe59ed8 --- /dev/null +++ b/wandb/run-20231031_224349-6h4rftgh/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T18:43:51.676256", + "startedAt": "2023-10-31T18:43:49.826138", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.6893999999999998, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.295, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.668, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.602, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.6, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.399, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.488, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.304, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.211, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.7, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.3, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.189, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.296, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.484, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.399, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.493, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.605, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.21, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.707, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.208, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.919490814209 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_224349-6h4rftgh/files/wandb-summary.json b/wandb/run-20231031_224349-6h4rftgh/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..62703f397ff87ab5b1ac445fccba8d7a5bf12d2c --- /dev/null +++ b/wandb/run-20231031_224349-6h4rftgh/files/wandb-summary.json @@ -0,0 +1 @@ +{"Train Loss": 1.4823065996170044, "Train Accuracy": 0.4849337935447693, "Validation Loss": 1.2942070960998535, "Validation Accuracy": 0.5403411984443665, "_timestamp": 1698781482.182595, "_runtime": 3652.3514819145203, "_step": 0, "_wandb": {"runtime": 3650}} \ No newline at end of file diff --git a/wandb/run-20231031_224349-6h4rftgh/logs/debug-internal.log b/wandb/run-20231031_224349-6h4rftgh/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..bf94475b37b34c9b2ace48247b7c81872bce250c --- /dev/null +++ b/wandb/run-20231031_224349-6h4rftgh/logs/debug-internal.log @@ -0,0 +1,1735 @@ +2023-10-31 22:43:50,345 INFO StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-10-31 22:43:50.344754 +2023-10-31 22:43:50,347 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status +2023-10-31 22:43:50,349 INFO WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/run-6h4rftgh.wandb +2023-10-31 22:43:50,352 DEBUG SenderThread:34227 [sender.py:send():380] send: header +2023-10-31 22:43:50,353 DEBUG SenderThread:34227 [sender.py:send():380] send: run +2023-10-31 22:43:51,188 INFO SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files +2023-10-31 22:43:51,188 INFO SenderThread:34227 [sender.py:_start_run_threads():1122] run started: 6h4rftgh with start time 1698777829.831113 +2023-10-31 22:43:51,209 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 22:43:51,210 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: check_version +2023-10-31 22:43:51,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 22:43:51,609 DEBUG HandlerThread:34227 [system_info.py:__init__():32] System info init +2023-10-31 22:43:51,609 DEBUG HandlerThread:34227 [system_info.py:__init__():47] System info init done +2023-10-31 22:43:51,609 INFO HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor +2023-10-31 22:43:51,610 INFO SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 22:43:51,610 INFO HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info +2023-10-31 22:43:51,611 INFO SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 22:43:51,612 INFO SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring +2023-10-31 22:43:51,613 INFO SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 22:43:51,614 INFO SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring +2023-10-31 22:43:51,616 INFO SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring +2023-10-31 22:43:51,676 DEBUG HandlerThread:34227 [system_info.py:probe():196] Probing system +2023-10-31 22:43:51,676 DEBUG HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 22:43:51,676 DEBUG HandlerThread:34227 [system_info.py:probe():244] Probing system done +2023-10-31 22:43:51,676 DEBUG HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T18:43:51.676256', 'startedAt': '2023-10-31T18:43:49.826138', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.6893999999999998, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.295, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.668, 'min': 1200.0, 'max': 3600.0}, {'current': 2.602, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.6, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.399, 'min': 1200.0, 'max': 3600.0}, {'current': 1.488, 'min': 1200.0, 'max': 3600.0}, {'current': 2.599, 'min': 1200.0, 'max': 3600.0}, {'current': 1.304, 'min': 1200.0, 'max': 3600.0}, {'current': 2.211, 'min': 1200.0, 'max': 3600.0}, {'current': 1.7, 'min': 1200.0, 'max': 3600.0}, {'current': 2.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.3, 'min': 1200.0, 'max': 3600.0}, {'current': 1.189, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.296, 'min': 1200.0, 'max': 3600.0}, {'current': 1.484, 'min': 1200.0, 'max': 3600.0}, {'current': 2.599, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.599, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.399, 'min': 1200.0, 'max': 3600.0}, {'current': 1.493, 'min': 1200.0, 'max': 3600.0}, {'current': 2.605, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 2.21, 'min': 1200.0, 'max': 3600.0}, {'current': 1.707, 'min': 1200.0, 'max': 3600.0}, {'current': 2.208, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.919490814209}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 22:43:51,677 INFO HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 22:43:51,677 INFO HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info +2023-10-31 22:43:51,677 DEBUG HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 22:43:51,677 DEBUG HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 22:43:51,677 DEBUG HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 22:43:52,191 INFO Thread-73 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/requirements.txt +2023-10-31 22:43:52,192 INFO Thread-73 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/conda-environment.yaml +2023-10-31 22:43:58,180 DEBUG HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 22:43:58,183 INFO HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 22:43:58,183 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:43:58,183 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 22:43:58,185 DEBUG SenderThread:34227 [sender.py:send():380] send: files +2023-10-31 22:43:58,185 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 22:43:58,196 INFO Thread-73 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/conda-environment.yaml +2023-10-31 22:43:58,197 INFO Thread-73 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/wandb-metadata.json +2023-10-31 22:43:58,198 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 22:43:58,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:43:58,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:43:58,202 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:43:59,612 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmpy6zlt_ymwandb/oc4dczxs-wandb-metadata.json +2023-10-31 22:44:00,712 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:44:05,712 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:44:10,713 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:44:13,196 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:44:13,196 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:44:13,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:44:16,572 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:44:21,578 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:44:22,204 INFO Thread-73 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/config.yaml +2023-10-31 22:44:26,968 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:44:28,196 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:44:28,196 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:44:28,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:44:32,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:44:37,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:44:42,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:44:43,196 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:44:43,196 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:44:43,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:44:48,536 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:44:51,617 DEBUG SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 22:44:51,619 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:44:53,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:44:58,196 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:44:58,197 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:44:58,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:44:58,626 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:45:03,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:45:08,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:45:13,196 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:45:13,197 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:45:13,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:45:14,581 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:45:19,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:45:21,621 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:45:24,623 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:45:28,196 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:45:28,197 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:45:28,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:45:30,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:45:35,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:45:40,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:45:43,196 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:45:43,197 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:45:43,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:45:45,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:45:50,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:45:51,624 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:45:55,625 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:45:58,196 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:45:58,197 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:45:58,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:46:01,625 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:46:06,625 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:46:11,626 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:46:13,197 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:46:13,197 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:46:13,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:46:17,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:46:21,627 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:46:22,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:46:27,629 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:46:28,197 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:46:28,197 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:46:28,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:46:33,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:46:38,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:46:43,197 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:46:43,197 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:46:43,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:46:44,580 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:46:49,581 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:46:51,629 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:46:54,630 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:46:58,197 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:46:58,197 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:46:58,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:47:00,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:47:05,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:47:10,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:47:13,197 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:47:13,197 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:47:13,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:47:15,650 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:47:20,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:47:21,632 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:47:26,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:47:28,197 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:47:28,198 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:47:28,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:47:32,568 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:47:37,569 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:47:42,570 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:47:43,197 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:47:43,198 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:47:43,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:47:47,637 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:47:51,634 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:47:53,635 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:47:58,197 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:47:58,198 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:47:58,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:47:59,625 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:48:04,626 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:48:06,277 INFO Thread-73 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/output.log +2023-10-31 22:48:08,278 INFO Thread-73 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/output.log +2023-10-31 22:48:09,682 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:48:13,197 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:48:13,198 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:48:13,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:48:15,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:48:20,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:48:21,636 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:48:25,637 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:48:28,197 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:48:28,198 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:48:28,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:48:31,586 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:48:36,587 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:48:41,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:48:43,198 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:48:43,198 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:48:43,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:48:47,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:48:51,638 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:48:52,639 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:48:57,640 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:48:58,198 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:48:58,198 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:48:58,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:49:03,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:49:08,594 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:49:13,198 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:49:13,199 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:49:13,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:49:13,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:49:18,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:49:21,640 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:49:23,642 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:49:28,198 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:49:28,198 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:49:28,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:49:29,544 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:49:34,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:49:39,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:49:43,198 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:49:43,198 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:49:43,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:49:44,578 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:49:49,578 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:49:51,642 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:49:54,644 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:49:58,198 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:49:58,199 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:49:58,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:49:59,646 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:50:04,647 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:50:09,648 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:50:13,198 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:50:13,199 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:50:13,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:50:15,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:50:20,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:50:21,644 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:50:25,645 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:50:28,198 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:50:28,199 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:50:28,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:50:31,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:50:36,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:50:41,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:50:43,198 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:50:43,199 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:50:43,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:50:46,565 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:50:51,566 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:50:51,646 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:50:56,648 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:50:58,198 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:50:58,199 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:50:58,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:51:02,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:51:07,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:51:12,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:51:13,199 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:51:13,199 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:51:13,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:51:17,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:51:21,648 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:51:22,650 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:51:27,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:51:28,199 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:51:28,199 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:51:28,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:51:33,639 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:51:38,640 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:51:43,200 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:51:43,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:51:43,201 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:51:44,623 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:51:49,623 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:51:51,652 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:51:54,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:51:58,199 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:51:58,200 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:51:58,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:52:00,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:52:05,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:52:10,366 INFO Thread-73 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/output.log +2023-10-31 22:52:11,189 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:52:13,199 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:52:13,200 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:52:13,200 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:52:16,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:52:21,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:52:21,653 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:52:26,655 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:52:28,199 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:52:28,200 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:52:28,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:52:32,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:52:37,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:52:42,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:52:43,200 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:52:43,201 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:52:43,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:52:47,626 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:52:51,655 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:52:52,656 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:52:57,657 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:52:58,199 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:52:58,200 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:52:58,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:53:03,594 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:53:08,594 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:53:13,199 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:53:13,200 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:53:13,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:53:13,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:53:18,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:53:21,658 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:53:23,660 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:53:28,200 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:53:28,200 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:53:28,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:53:29,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:53:34,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:53:39,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:53:43,200 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:53:43,201 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:53:43,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:53:45,613 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:53:50,613 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:53:51,660 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:53:55,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:53:58,200 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:53:58,200 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:53:58,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:54:01,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:54:06,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:54:11,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:54:13,200 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:54:13,200 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:54:13,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:54:16,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:54:21,599 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:54:21,665 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:54:26,667 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:54:28,200 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:54:28,200 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:54:28,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:54:32,605 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:54:37,606 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:54:42,607 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:54:43,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:54:43,201 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:54:43,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:54:47,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:54:51,664 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:54:52,665 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:54:57,666 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:54:58,200 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:54:58,201 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:54:58,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:55:03,594 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:55:08,595 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:55:13,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:55:13,201 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:55:13,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:55:13,625 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:55:18,626 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:55:21,671 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:55:23,673 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:55:28,200 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:55:28,201 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:55:28,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:55:29,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:55:34,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:55:39,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:55:43,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:55:43,201 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:55:43,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:55:45,566 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:55:50,567 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:55:51,669 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:55:55,671 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:55:58,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:55:58,201 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:55:58,241 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:56:01,639 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:56:06,640 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:56:11,640 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:56:13,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:56:13,201 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:56:13,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:56:17,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:56:21,671 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:56:22,672 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:56:27,672 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:56:28,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:56:28,202 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:56:28,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:56:33,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:56:38,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:56:43,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:56:43,202 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:56:43,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:56:43,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:56:48,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:56:51,673 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:56:53,674 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:56:58,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:56:58,202 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:56:58,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:56:59,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:57:04,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:57:09,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:57:13,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:57:13,202 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:57:13,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:57:15,576 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:57:20,576 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:57:21,675 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:57:25,677 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:57:28,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:57:28,202 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:57:28,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:57:30,699 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:57:35,700 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:57:40,700 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:57:43,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:57:43,202 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:57:43,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:57:46,691 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:57:51,677 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:57:52,678 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:57:57,679 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:57:58,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:57:58,202 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:57:58,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:58:03,566 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:58:08,567 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:58:13,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:58:13,203 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:58:13,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:58:13,652 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:58:18,652 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:58:21,679 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:58:23,680 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:58:28,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:58:28,202 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:58:28,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:58:29,640 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:58:34,640 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:58:39,641 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:58:43,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:58:43,203 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:58:43,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:58:45,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:58:50,583 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:58:51,683 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:58:55,684 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:58:58,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:58:58,203 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:58:58,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:59:01,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:59:06,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:59:11,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:59:13,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:59:13,203 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:59:13,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:59:16,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:59:21,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:59:21,684 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:59:26,686 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:59:28,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:59:28,203 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:59:28,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:59:32,632 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:59:37,633 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:59:42,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:59:43,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:59:43,203 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:59:43,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 22:59:47,637 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:59:51,688 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 22:59:52,689 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:59:57,689 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 22:59:58,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 22:59:58,203 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 22:59:58,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:00:03,577 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:00:08,577 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:00:13,203 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:00:13,203 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:00:13,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:00:14,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:00:19,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:00:21,690 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:00:24,692 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:00:28,203 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:00:28,203 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:00:28,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:00:30,586 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:00:35,587 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:00:40,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:00:43,203 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:00:43,203 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:00:43,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:00:45,607 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:00:50,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:00:51,692 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:00:55,694 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:00:58,203 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:00:58,204 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:00:58,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:01:01,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:01:06,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:01:11,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:01:13,203 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:01:13,204 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:01:13,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:01:16,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:01:21,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:01:21,695 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:01:26,697 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:01:28,203 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:01:28,204 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:01:28,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:01:32,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:01:37,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:01:42,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:01:43,204 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:01:43,205 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:01:43,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:01:47,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:01:51,698 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:01:52,699 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:01:57,700 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:01:58,203 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:01:58,204 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:01:58,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:02:03,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:02:08,652 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:02:13,204 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:02:13,204 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:02:13,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:02:14,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:02:19,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:02:21,700 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:02:24,702 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:02:28,204 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:02:28,204 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:02:28,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:02:30,594 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:02:35,595 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:02:40,595 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:02:43,204 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:02:43,204 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:02:43,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:02:46,559 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:02:51,559 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:02:51,702 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:02:56,704 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:02:58,204 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:02:58,205 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:02:58,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:03:02,577 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:03:07,577 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:03:12,578 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:03:13,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:03:13,205 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:03:13,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:03:18,553 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:03:21,704 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:03:23,706 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:03:28,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:03:28,206 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:03:28,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:03:29,571 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:03:34,572 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:03:39,572 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:03:43,204 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:03:43,205 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:03:43,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:03:44,606 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:03:49,606 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:03:51,707 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:03:54,708 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:03:58,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:03:58,205 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:03:58,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:04:00,559 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:04:05,559 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:04:10,560 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:04:13,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:04:13,206 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:04:13,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:04:15,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:04:20,586 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:04:21,708 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:04:25,710 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:04:28,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:04:28,205 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:04:28,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:04:31,642 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:04:36,643 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:04:41,644 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:04:43,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:04:43,205 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:04:43,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:04:47,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:04:51,711 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:04:52,712 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:04:57,713 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:04:58,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:04:58,205 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:04:58,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:05:03,580 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:05:08,581 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:05:13,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:05:13,205 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:05:13,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:05:14,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:05:19,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:05:21,713 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:05:24,715 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:05:28,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:05:28,205 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:05:28,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:05:30,632 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:05:35,632 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:05:40,633 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:05:43,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:05:43,206 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:05:43,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:05:46,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:05:51,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:05:51,716 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:05:56,718 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:05:58,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:05:58,206 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:05:58,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:06:02,595 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:06:07,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:06:12,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:06:13,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:06:13,206 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:06:13,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:06:17,640 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:06:21,719 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:06:22,721 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:06:27,721 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:06:28,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:06:28,206 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:06:28,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:06:33,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:06:38,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:06:43,206 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:06:43,207 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:06:43,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:06:43,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:06:48,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:06:51,720 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:06:53,722 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:06:58,206 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:06:58,206 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:06:58,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:06:59,624 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:07:04,625 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:07:09,625 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:07:13,206 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:07:13,206 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:07:13,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:07:15,568 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:07:20,568 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:07:21,723 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:07:25,724 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:07:28,206 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:07:28,206 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:07:28,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:07:31,607 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:07:36,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:07:41,609 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:07:43,206 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:07:43,206 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:07:43,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:07:47,605 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:07:51,724 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:07:52,726 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:07:57,727 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:07:58,207 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:07:58,207 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:07:58,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:08:03,645 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:08:08,646 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:08:13,206 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:08:13,207 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:08:13,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:08:14,635 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:08:19,636 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:08:21,727 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:08:24,728 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:08:28,206 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:08:28,207 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:08:28,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:08:30,595 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:08:35,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:08:40,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:08:43,206 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:08:43,207 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:08:43,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:08:45,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:08:50,662 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:08:51,728 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:08:55,730 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:08:58,207 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:08:58,207 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:08:58,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:09:01,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:09:06,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:09:11,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:09:13,207 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:09:13,207 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:09:13,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:09:17,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:09:21,730 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:09:22,732 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:09:27,732 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:09:28,207 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:09:28,207 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:09:28,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:09:33,606 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:09:38,606 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:09:43,207 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:09:43,207 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:09:43,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:09:44,580 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:09:49,581 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:09:51,732 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:09:54,734 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:09:58,207 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:09:58,207 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:09:58,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:10:00,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:10:05,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:10:10,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:10:13,207 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:10:13,208 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:10:13,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:10:15,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:10:20,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:10:21,735 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:10:25,736 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:10:28,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:10:28,208 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:10:28,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:10:31,664 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:10:36,664 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:10:41,665 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:10:43,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:10:43,208 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:10:43,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:10:47,590 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:10:51,736 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:10:52,738 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:10:57,739 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:10:58,210 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:10:58,210 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:10:58,210 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:11:03,662 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:11:08,663 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:11:13,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:11:13,209 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:11:13,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:11:14,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:11:19,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:11:21,740 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:11:24,741 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:11:28,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:11:28,209 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:11:28,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:11:30,570 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:11:35,571 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:11:40,571 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:11:43,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:11:43,209 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:11:43,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:11:45,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:11:50,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:11:51,743 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:11:55,745 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:11:58,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:11:58,208 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:11:58,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:12:01,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:12:06,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:12:11,797 INFO Thread-73 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/output.log +2023-10-31 23:12:11,869 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:12:13,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:12:13,209 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:12:13,209 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:12:17,636 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:12:21,746 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:12:22,747 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:12:27,748 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:12:28,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:12:28,209 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:12:28,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:12:33,647 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:12:38,647 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:12:43,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:12:43,209 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:12:43,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:12:44,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:12:49,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:12:51,747 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:12:54,749 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:12:58,208 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:12:58,209 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:12:58,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:13:00,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:13:05,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:13:10,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:13:13,209 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:13:13,209 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:13:13,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:13:15,637 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:13:20,638 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:13:21,750 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:13:25,752 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:13:28,209 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:13:28,209 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:13:28,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:13:31,637 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:13:36,637 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:13:41,638 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:13:43,209 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:13:43,210 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:13:43,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:13:47,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:13:51,753 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:13:52,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:13:57,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:13:58,209 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:13:58,209 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:13:58,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:14:03,569 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:14:08,569 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:14:13,209 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:14:13,210 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:14:13,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:14:13,693 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:14:18,693 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:14:21,755 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:14:23,757 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:14:28,209 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:14:28,210 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:14:28,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:14:29,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:14:34,629 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:14:39,630 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:14:43,211 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:14:43,211 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:14:43,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:14:44,642 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:14:49,643 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:14:51,756 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:14:54,758 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:14:58,210 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:14:58,210 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:14:58,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:15:00,587 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:15:05,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:15:10,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:15:13,210 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:15:13,210 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:15:13,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:15:16,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:15:21,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:15:21,758 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:15:26,760 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:15:28,210 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:15:28,210 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:15:28,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:15:32,587 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:15:37,587 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:15:42,588 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:15:43,210 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:15:43,210 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:15:43,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:15:48,580 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:15:51,760 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:15:53,762 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:15:58,210 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:15:58,210 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:15:58,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:15:59,573 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:16:04,573 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:16:09,574 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:16:13,211 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:16:13,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:16:13,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:16:14,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:16:19,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:16:21,763 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:16:24,764 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:16:28,211 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:16:28,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:16:28,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:16:30,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:16:35,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:16:40,559 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:16:43,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:16:43,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:16:43,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:16:45,636 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:16:50,636 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:16:51,765 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:16:55,767 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:16:58,211 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:16:58,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:16:58,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:17:01,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:17:06,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:17:11,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:17:13,211 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:17:13,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:17:13,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:17:16,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:17:21,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:17:21,768 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:17:26,769 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:17:28,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:17:28,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:17:28,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:17:32,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:17:37,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:17:42,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:17:43,211 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:17:43,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:17:43,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:17:47,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:17:51,770 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:17:52,771 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:17:57,772 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:17:58,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:17:58,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:17:58,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:18:03,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:18:08,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:18:13,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:18:13,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:18:13,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:18:13,637 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:18:18,638 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:18:21,773 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:18:23,774 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:18:28,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:18:28,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:18:28,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:18:29,552 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:18:34,553 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:18:39,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:18:43,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:18:43,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:18:43,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:18:44,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:18:49,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:18:51,776 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:18:54,777 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:18:58,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:18:58,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:18:58,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:18:59,872 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:19:04,873 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:19:09,873 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:19:13,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:19:13,212 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:19:13,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:19:15,565 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:19:20,565 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:19:21,778 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:19:25,780 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:19:28,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:19:28,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:19:28,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:19:31,594 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:19:36,595 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:19:41,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:19:43,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:19:43,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:19:43,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:19:46,616 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:19:51,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:19:51,780 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:19:56,782 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:19:58,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:19:58,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:19:58,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:20:02,645 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:20:07,646 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:20:12,647 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:20:13,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:20:13,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:20:13,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:20:18,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:20:21,783 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:20:23,784 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:20:28,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:20:28,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:20:28,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:20:29,581 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:20:34,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:20:39,583 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:20:43,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:20:43,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:20:43,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:20:44,626 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:20:49,626 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:20:51,783 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:20:54,785 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:20:58,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:20:58,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:20:58,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:21:00,569 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:21:05,570 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:21:10,571 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:21:13,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:21:13,213 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:21:13,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:21:15,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:21:20,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:21:21,785 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:21:25,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:21:28,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:21:28,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:21:28,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:21:31,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:21:36,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:21:41,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:21:43,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:21:43,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:21:43,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:21:46,576 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:21:51,577 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:21:51,787 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:21:56,789 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:21:58,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:21:58,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:21:58,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:22:02,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:22:07,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:22:12,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:22:13,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:22:13,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:22:13,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:22:17,576 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:22:21,790 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:22:22,791 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:22:27,792 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:22:28,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:22:28,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:22:28,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:22:33,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:22:38,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:22:43,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:22:43,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:22:43,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:22:44,542 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:22:49,542 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:22:51,792 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:22:54,794 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:22:58,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:22:58,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:22:58,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:23:00,632 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:23:05,633 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:23:10,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:23:13,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:23:13,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:23:13,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:23:16,619 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:23:21,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:23:21,795 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:23:26,797 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:23:28,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:23:28,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:23:28,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:23:32,565 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:23:37,565 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:23:42,566 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:23:43,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:23:43,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:23:43,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:23:48,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:23:51,798 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:23:53,799 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:23:58,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:23:58,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:23:58,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:23:59,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:24:04,594 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:24:09,595 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:24:13,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:24:13,214 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:24:13,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:24:14,620 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:24:18,059 INFO Thread-73 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/output.log +2023-10-31 23:24:20,205 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:24:21,799 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:24:25,801 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:24:28,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:24:28,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:24:28,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:24:31,590 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:24:36,591 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:24:41,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:24:43,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:24:43,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:24:43,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:24:46,644 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:24:51,644 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:24:51,802 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:24:56,804 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:24:58,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:24:58,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:24:58,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:25:02,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:25:07,635 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:25:12,635 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:25:13,214 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:25:13,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:25:13,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:25:18,568 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:25:21,804 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:25:23,806 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:25:28,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:25:28,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:25:28,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:25:29,644 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:25:34,645 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:25:39,646 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:25:43,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:25:43,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:25:43,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:25:45,643 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:25:50,644 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:25:51,807 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:25:55,808 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:25:58,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:25:58,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:25:58,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:26:01,646 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:26:06,647 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:26:11,648 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:26:13,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:26:13,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:26:13,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:26:17,641 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:26:21,809 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:26:22,810 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:26:27,811 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:26:28,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:26:28,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:26:28,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:26:33,631 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:26:38,631 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:26:43,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:26:43,215 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:26:43,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:26:44,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:26:49,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:26:51,811 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:26:54,812 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:26:58,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:26:58,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:26:58,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:27:00,559 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:27:05,560 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:27:10,561 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:27:13,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:27:13,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:27:13,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:27:15,642 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:27:20,642 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:27:21,813 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:27:25,815 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:27:28,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:27:28,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:27:28,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:27:31,607 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:27:36,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:27:41,609 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:27:43,215 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:27:43,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:27:43,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:27:47,560 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:27:51,816 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:27:52,817 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:27:57,818 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:27:58,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:27:58,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:27:58,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:28:03,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:28:08,635 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:28:13,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:28:13,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:28:13,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:28:14,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:28:19,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:28:21,817 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:28:24,819 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:28:28,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:28:28,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:28:28,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:28:30,623 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:28:35,624 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:28:40,625 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:28:43,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:28:43,216 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:28:43,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:28:46,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:28:51,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:28:51,820 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:28:56,821 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:28:58,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:28:58,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:28:58,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:29:02,568 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:29:07,568 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:29:12,569 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:29:13,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:29:13,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:29:13,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:29:17,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:29:21,823 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:29:22,825 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:29:27,826 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:29:28,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:29:28,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:29:28,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:29:33,607 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:29:38,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:29:43,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:29:43,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:29:43,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:29:44,571 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:29:49,571 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:29:51,827 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:29:54,829 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:29:58,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:29:58,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:29:58,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:30:00,584 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:30:05,584 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:30:10,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:30:13,216 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:30:13,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:30:13,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:30:16,569 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:30:21,570 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:30:21,829 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:30:26,831 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:30:28,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:30:28,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:30:28,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:30:32,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:30:37,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:30:42,599 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:30:43,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:30:43,217 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:30:43,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:30:48,592 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:30:51,831 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:30:53,833 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:30:58,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:30:58,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:30:58,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:30:59,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:31:04,635 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:31:09,636 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:31:13,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:31:13,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:31:13,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:31:15,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:31:20,583 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:31:21,833 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:31:25,835 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:31:28,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:31:28,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:31:28,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:31:31,604 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:31:36,605 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:31:41,605 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:31:43,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:31:43,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:31:43,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:31:46,641 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:31:51,642 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:31:51,835 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:31:56,837 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:31:58,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:31:58,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:31:58,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:32:02,650 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:32:07,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:32:12,652 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:32:13,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:32:13,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:32:13,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:32:18,561 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:32:21,837 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:32:23,838 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:32:28,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:32:28,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:32:28,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:32:29,663 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:32:34,664 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:32:39,665 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:32:43,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:32:43,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:32:43,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:32:45,644 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:32:50,645 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:32:51,839 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:32:55,841 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:32:58,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:32:58,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:32:58,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:33:01,603 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:33:06,604 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:33:11,605 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:33:13,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:33:13,218 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:33:13,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:33:17,544 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:33:21,841 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:33:22,843 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:33:27,843 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:33:28,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:33:28,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:33:28,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:33:33,599 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:33:38,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:33:43,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:33:43,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:33:43,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:33:44,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:33:49,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:33:51,843 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:33:54,845 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:33:58,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:33:58,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:33:58,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:34:00,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:34:05,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:34:10,559 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:34:13,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:34:13,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:34:13,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:34:16,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:34:21,550 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:34:21,846 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:34:26,848 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:34:28,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:34:28,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:34:28,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:34:32,553 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:34:37,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:34:42,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:34:43,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:34:43,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:34:43,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:34:47,569 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:34:51,849 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:34:52,850 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:34:57,851 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:34:58,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:34:58,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:34:58,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:35:03,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:35:08,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:35:13,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:35:13,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:35:13,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:35:14,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:35:19,629 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:35:21,852 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:35:24,853 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:35:28,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:35:28,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:35:28,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:35:30,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:35:35,599 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:35:40,599 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:35:43,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:35:43,219 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:35:43,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:35:45,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:35:50,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:35:51,853 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:35:55,855 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:35:58,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:35:58,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:35:58,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:36:01,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:36:06,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:36:11,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:36:13,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:36:13,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:36:13,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:36:16,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:36:21,629 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:36:21,855 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:36:26,857 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:36:28,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:36:28,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:36:28,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:36:32,604 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:36:37,605 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:36:42,606 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:36:43,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:36:43,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:36:43,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:36:47,660 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:36:51,858 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:36:52,859 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:36:57,860 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:36:58,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:36:58,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:36:58,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:37:03,575 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:37:08,576 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:37:13,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:37:13,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:37:13,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:37:13,578 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:37:18,579 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:37:21,861 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:37:23,862 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:37:28,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:37:28,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:37:28,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:37:29,607 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:37:34,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:37:39,609 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:37:43,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:37:43,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:37:43,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:37:45,599 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:37:50,600 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:37:51,863 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:37:55,865 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:37:58,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:37:58,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:37:58,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:38:01,609 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:38:06,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:38:11,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:38:13,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:38:13,220 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:38:13,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:38:16,644 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:38:21,645 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:38:21,865 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:38:26,866 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:38:28,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:38:28,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:38:28,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:38:32,610 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:38:37,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:38:42,612 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:38:43,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:38:43,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:38:43,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:38:47,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:38:51,867 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:38:52,868 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:38:57,869 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:38:58,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:38:58,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:38:58,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:39:03,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:39:08,654 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:39:13,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:39:13,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:39:13,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:39:14,568 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:39:19,569 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:39:21,870 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:39:24,872 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:39:28,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:39:28,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:39:28,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:39:30,623 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:39:35,624 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:39:40,629 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:39:43,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:39:43,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:39:43,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:39:46,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:39:51,598 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:39:51,874 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:39:56,876 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:39:58,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:39:58,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:39:58,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:40:02,627 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:40:07,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:40:12,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:40:13,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:40:13,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:40:13,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:40:18,567 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:40:21,876 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:40:23,878 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:40:28,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:40:28,221 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:40:28,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:40:29,628 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:40:34,629 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:40:39,630 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:40:43,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:40:43,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:40:43,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:40:44,660 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:40:49,661 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:40:51,879 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:40:54,880 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:40:58,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:40:58,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:40:58,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:41:00,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:41:05,602 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:41:10,602 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:41:13,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:41:13,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:41:13,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:41:15,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:41:20,611 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:41:21,883 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:41:25,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:41:28,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:41:28,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:41:28,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:41:31,650 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:41:36,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:41:41,651 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:41:43,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:41:43,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:41:43,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:41:47,615 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:41:51,885 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:41:52,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:41:57,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:41:58,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:41:58,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:41:58,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:42:03,675 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:42:08,676 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:42:13,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:42:13,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:42:13,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:42:14,574 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:42:19,575 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:42:21,888 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:42:24,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:42:28,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:42:28,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:42:28,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:42:30,595 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:42:35,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:42:40,596 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:42:43,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:42:43,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:42:43,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:42:45,671 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:42:50,672 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:42:51,889 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:42:55,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:42:58,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:42:58,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:42:58,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:43:01,633 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:43:06,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:43:11,635 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:43:13,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:43:13,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:43:13,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:43:17,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:43:21,891 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:43:22,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:43:27,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:43:28,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:43:28,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:43:28,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:43:33,562 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:43:38,562 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:43:43,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:43:43,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:43:43,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:43:43,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:43:48,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:43:51,893 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:43:53,895 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:43:58,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:43:58,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:43:58,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:43:59,617 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:44:04,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:44:09,618 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:44:13,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:44:13,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:44:13,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:44:14,744 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:44:19,744 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:44:21,896 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:44:24,897 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:44:28,223 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:44:28,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:44:28,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:44:30,644 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:44:35,645 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:44:40,646 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:44:42,183 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history +2023-10-31 23:44:42,185 DEBUG SenderThread:34227 [sender.py:send():380] send: history +2023-10-31 23:44:42,185 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: summary_record +2023-10-31 23:44:42,185 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 23:44:42,225 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 23:44:42,225 DEBUG SenderThread:34227 [sender.py:send():380] send: exit +2023-10-31 23:44:42,226 INFO SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0 +2023-10-31 23:44:42,226 INFO SenderThread:34227 [sender.py:send_exit():587] handling runtime: 3650 +2023-10-31 23:44:42,226 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 23:44:42,226 INFO SenderThread:34227 [sender.py:send_exit():593] send defer +2023-10-31 23:44:42,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:42,227 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0 +2023-10-31 23:44:42,227 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:42,227 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-10-31 23:44:42,227 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 1 +2023-10-31 23:44:42,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:42,228 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1 +2023-10-31 23:44:42,228 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:42,228 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-10-31 23:44:42,228 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 2 +2023-10-31 23:44:42,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:42,228 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2 +2023-10-31 23:44:42,228 INFO HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor +2023-10-31 23:44:42,229 DEBUG SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-10-31 23:44:42,229 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor +2023-10-31 23:44:42,230 DEBUG SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-10-31 23:44:42,230 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor +2023-10-31 23:44:42,506 INFO Thread-73 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/wandb-summary.json +2023-10-31 23:44:43,935 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor +2023-10-31 23:44:43,936 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor +2023-10-31 23:44:43,936 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor +2023-10-31 23:44:43,936 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 23:44:43,937 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:43,938 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-10-31 23:44:43,938 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 3 +2023-10-31 23:44:43,938 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:44:43,938 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:43,940 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 23:44:43,940 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3 +2023-10-31 23:44:43,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:43,941 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-10-31 23:44:43,941 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 4 +2023-10-31 23:44:43,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:43,941 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4 +2023-10-31 23:44:43,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:43,942 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-10-31 23:44:43,942 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 5 +2023-10-31 23:44:43,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:43,942 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5 +2023-10-31 23:44:43,943 DEBUG SenderThread:34227 [sender.py:send():380] send: summary +2023-10-31 23:44:43,943 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-10-31 23:44:43,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:43,944 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-10-31 23:44:43,944 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 6 +2023-10-31 23:44:43,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:43,944 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6 +2023-10-31 23:44:43,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:43,945 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-10-31 23:44:43,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:44:44,185 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 23:44:44,340 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 7 +2023-10-31 23:44:44,341 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 23:44:44,341 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:44,341 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7 +2023-10-31 23:44:44,342 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:44,342 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-10-31 23:44:44,507 INFO Thread-73 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/wandb-summary.json +2023-10-31 23:44:44,507 INFO Thread-73 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/config.yaml +2023-10-31 23:44:45,186 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 23:44:45,332 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 8 +2023-10-31 23:44:45,332 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 23:44:45,332 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:45,333 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8 +2023-10-31 23:44:45,333 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:45,334 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-10-31 23:44:45,334 INFO SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact +2023-10-31 23:44:45,334 INFO SenderThread:34227 [job_builder.py:_get_source_type():404] no source found +2023-10-31 23:44:45,334 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 9 +2023-10-31 23:44:45,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:45,335 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9 +2023-10-31 23:44:45,335 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:45,335 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-10-31 23:44:45,335 INFO SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher +2023-10-31 23:44:45,507 INFO Thread-73 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/output.log +2023-10-31 23:44:45,508 INFO SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files +2023-10-31 23:44:45,508 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/output.log output.log +2023-10-31 23:44:45,509 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/requirements.txt requirements.txt +2023-10-31 23:44:45,514 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/conda-environment.yaml conda-environment.yaml +2023-10-31 23:44:45,516 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/wandb-summary.json wandb-summary.json +2023-10-31 23:44:45,524 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/config.yaml config.yaml +2023-10-31 23:44:45,525 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/wandb-metadata.json wandb-metadata.json +2023-10-31 23:44:45,530 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 10 +2023-10-31 23:44:45,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:45,546 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10 +2023-10-31 23:44:45,547 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:45,548 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-10-31 23:44:45,548 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 23:44:46,186 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 23:44:46,187 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 23:44:46,514 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/output.log +2023-10-31 23:44:46,675 INFO wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/requirements.txt +2023-10-31 23:44:46,714 INFO wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/wandb-summary.json +2023-10-31 23:44:46,718 INFO wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/config.yaml +2023-10-31 23:44:46,772 INFO wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/files/conda-environment.yaml +2023-10-31 23:44:46,973 INFO Thread-72 :34227 [sender.py:transition_state():613] send defer: 11 +2023-10-31 23:44:46,973 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:46,974 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11 +2023-10-31 23:44:46,974 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:46,974 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-10-31 23:44:46,975 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher +2023-10-31 23:44:46,975 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 12 +2023-10-31 23:44:46,975 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:46,975 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12 +2023-10-31 23:44:46,975 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:46,976 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-10-31 23:44:46,976 INFO SenderThread:34227 [file_stream.py:finish():595] file stream finish called +2023-10-31 23:44:47,187 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 23:44:48,120 INFO SenderThread:34227 [file_stream.py:finish():599] file stream finish is done +2023-10-31 23:44:48,121 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 13 +2023-10-31 23:44:48,121 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 23:44:48,121 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:48,121 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13 +2023-10-31 23:44:48,122 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:48,122 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-10-31 23:44:48,122 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 14 +2023-10-31 23:44:48,123 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-10-31 23:44:48,123 DEBUG SenderThread:34227 [sender.py:send():380] send: final +2023-10-31 23:44:48,123 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14 +2023-10-31 23:44:48,124 DEBUG SenderThread:34227 [sender.py:send():380] send: footer +2023-10-31 23:44:48,124 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-10-31 23:44:48,124 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-10-31 23:44:48,125 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:44:48,126 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-10-31 23:44:48,127 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-10-31 23:44:48,128 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info +2023-10-31 23:44:48,128 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary +2023-10-31 23:44:48,128 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history +2023-10-31 23:44:48,128 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: server_info +2023-10-31 23:44:48,134 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info +2023-10-31 23:44:48,456 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: job_info +2023-10-31 23:44:48,458 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown +2023-10-31 23:44:48,458 INFO HandlerThread:34227 [handler.py:finish():866] shutting down handler +2023-10-31 23:44:49,135 INFO WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/run-6h4rftgh.wandb +2023-10-31 23:44:49,457 INFO SenderThread:34227 [sender.py:finish():1534] shutting down sender +2023-10-31 23:44:49,457 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-10-31 23:44:49,457 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231031_224349-6h4rftgh/logs/debug.log b/wandb/run-20231031_224349-6h4rftgh/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bac58baa7c423eac42ff74824534483ecda1d9f4 --- /dev/null +++ b/wandb/run-20231031_224349-6h4rftgh/logs/debug.log @@ -0,0 +1,34 @@ +2023-10-31 22:43:49,827 INFO MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 22:43:49,827 INFO MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070 +2023-10-31 22:43:49,828 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 22:43:49,828 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 22:43:49,828 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 22:43:49,828 INFO MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 22:43:49,828 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 22:43:49,828 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 22:43:49,828 INFO MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/logs/debug.log +2023-10-31 22:43:49,828 INFO MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh/logs/debug-internal.log +2023-10-31 22:43:49,828 INFO MainThread:34070 [wandb_init.py:init():568] calling init triggers +2023-10-31 22:43:49,828 INFO MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'RMSprop', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 32} +2023-10-31 22:43:49,828 INFO MainThread:34070 [wandb_init.py:init():618] starting backend +2023-10-31 22:43:49,828 INFO MainThread:34070 [wandb_init.py:init():622] setting up manager +2023-10-31 22:43:49,830 INFO MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 22:43:49,830 INFO MainThread:34070 [wandb_init.py:init():628] backend started and connected +2023-10-31 22:43:49,833 INFO MainThread:34070 [wandb_init.py:init():720] updated telemetry +2023-10-31 22:43:49,834 INFO MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 22:43:51,209 INFO MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 22:43:51,595 INFO MainThread:34070 [wandb_run.py:_on_init():2229] got version response +2023-10-31 22:43:51,595 INFO MainThread:34070 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 22:43:58,196 INFO MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 22:43:58,196 INFO MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 22:43:58,196 INFO MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 22:43:58,196 INFO MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 22:43:58,197 INFO MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process +2023-10-31 23:44:42,183 INFO MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/6h4rftgh +2023-10-31 23:44:42,183 INFO MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0 +2023-10-31 23:44:42,183 INFO MainThread:34070 [wandb_run.py:_restore():2151] restore +2023-10-31 23:44:42,183 INFO MainThread:34070 [wandb_run.py:_restore():2157] restore done +2023-10-31 23:44:49,460 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-10-31 23:44:49,460 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-10-31 23:44:49,468 INFO MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files diff --git a/wandb/run-20231031_224349-6h4rftgh/run-6h4rftgh.wandb b/wandb/run-20231031_224349-6h4rftgh/run-6h4rftgh.wandb new file mode 100644 index 0000000000000000000000000000000000000000..eccc623741ce7e5c357ac6d5014b2b37f1c06cde Binary files /dev/null and b/wandb/run-20231031_224349-6h4rftgh/run-6h4rftgh.wandb differ diff --git a/wandb/run-20231031_234449-7nbh63pb/files/conda-environment.yaml b/wandb/run-20231031_234449-7nbh63pb/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231031_234449-7nbh63pb/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231031_234449-7nbh63pb/files/config.yaml b/wandb/run-20231031_234449-7nbh63pb/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa4c111ddaa9579d41dfdbff3139faf1758786c3 --- /dev/null +++ b/wandb/run-20231031_234449-7nbh63pb/files/config.yaml @@ -0,0 +1,53 @@ +wandb_version: 1 + +optimizer: + desc: null + value: RMSprop +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 35 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698781489.96626 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 2 + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231031_234449-7nbh63pb/files/output.log b/wandb/run-20231031_234449-7nbh63pb/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3e3243d813ebee0d393d6b3779843d64e779d465 --- /dev/null +++ b/wandb/run-20231031_234449-7nbh63pb/files/output.log @@ -0,0 +1,3 @@ + +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth \ No newline at end of file diff --git a/wandb/run-20231031_234449-7nbh63pb/files/requirements.txt b/wandb/run-20231031_234449-7nbh63pb/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231031_234449-7nbh63pb/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231031_234449-7nbh63pb/files/wandb-metadata.json b/wandb/run-20231031_234449-7nbh63pb/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6398a422d38de11ecd1eb0ff7e658b8a3ba7cb06 --- /dev/null +++ b/wandb/run-20231031_234449-7nbh63pb/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T19:44:51.850452", + "startedAt": "2023-10-31T19:44:49.961894", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.5589250000000001, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.303, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.598, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.762, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.498, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.295, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.455, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.75, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.201, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.709, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.598, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.819, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.496, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.303, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.457, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.298, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.095, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.206, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.9195327758789 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231031_234449-7nbh63pb/files/wandb-summary.json b/wandb/run-20231031_234449-7nbh63pb/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9c464340cafb2ab3aef71d76d7ea981b608f1d2d --- /dev/null +++ b/wandb/run-20231031_234449-7nbh63pb/files/wandb-summary.json @@ -0,0 +1 @@ +{"Train Loss": 1.4545906782150269, "Train Accuracy": 0.4946317970752716, "Validation Loss": 1.2756197452545166, "Validation Accuracy": 0.5495542883872986, "_timestamp": 1698783223.807893, "_runtime": 1733.8416330814362, "_step": 0, "_wandb": {"runtime": 1732}} \ No newline at end of file diff --git a/wandb/run-20231031_234449-7nbh63pb/logs/debug-internal.log b/wandb/run-20231031_234449-7nbh63pb/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f17bce76ef862812aa2a3d57e04570bf1d0468ef --- /dev/null +++ b/wandb/run-20231031_234449-7nbh63pb/logs/debug-internal.log @@ -0,0 +1,923 @@ +2023-10-31 23:44:50,463 INFO StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-10-31 23:44:50.462083 +2023-10-31 23:44:50,465 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status +2023-10-31 23:44:50,467 INFO WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/run-7nbh63pb.wandb +2023-10-31 23:44:50,470 DEBUG SenderThread:34227 [sender.py:send():380] send: header +2023-10-31 23:44:50,470 DEBUG SenderThread:34227 [sender.py:send():380] send: run +2023-10-31 23:44:51,289 INFO SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files +2023-10-31 23:44:51,289 INFO SenderThread:34227 [sender.py:_start_run_threads():1122] run started: 7nbh63pb with start time 1698781489.96626 +2023-10-31 23:44:51,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version +2023-10-31 23:44:51,304 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: check_version +2023-10-31 23:44:51,780 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start +2023-10-31 23:44:51,782 DEBUG HandlerThread:34227 [system_info.py:__init__():32] System info init +2023-10-31 23:44:51,782 DEBUG HandlerThread:34227 [system_info.py:__init__():47] System info init done +2023-10-31 23:44:51,782 INFO HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor +2023-10-31 23:44:51,782 INFO SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-10-31 23:44:51,782 INFO HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info +2023-10-31 23:44:51,783 INFO SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring +2023-10-31 23:44:51,784 INFO SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring +2023-10-31 23:44:51,785 INFO SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring +2023-10-31 23:44:51,786 INFO SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring +2023-10-31 23:44:51,788 INFO SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring +2023-10-31 23:44:51,850 DEBUG HandlerThread:34227 [system_info.py:probe():196] Probing system +2023-10-31 23:44:51,850 DEBUG HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid +2023-10-31 23:44:51,850 DEBUG HandlerThread:34227 [system_info.py:probe():244] Probing system done +2023-10-31 23:44:51,851 DEBUG HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T19:44:51.850452', 'startedAt': '2023-10-31T19:44:49.961894', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.5589250000000001, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.303, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.598, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.762, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.599, 'min': 1200.0, 'max': 3600.0}, {'current': 1.498, 'min': 1200.0, 'max': 3600.0}, {'current': 1.295, 'min': 1200.0, 'max': 3600.0}, {'current': 1.455, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.75, 'min': 1200.0, 'max': 3600.0}, {'current': 2.201, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.709, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.598, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.819, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.599, 'min': 1200.0, 'max': 3600.0}, {'current': 1.496, 'min': 1200.0, 'max': 3600.0}, {'current': 1.303, 'min': 1200.0, 'max': 3600.0}, {'current': 1.457, 'min': 1200.0, 'max': 3600.0}, {'current': 1.298, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.095, 'min': 1200.0, 'max': 3600.0}, {'current': 2.206, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9195327758789}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-10-31 23:44:51,851 INFO HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info +2023-10-31 23:44:51,851 INFO HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info +2023-10-31 23:44:51,851 DEBUG HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-10-31 23:44:51,851 DEBUG HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done +2023-10-31 23:44:51,851 DEBUG HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-10-31 23:44:52,294 INFO Thread-85 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/requirements.txt +2023-10-31 23:44:52,295 INFO Thread-85 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/conda-environment.yaml +2023-10-31 23:44:58,292 INFO Thread-85 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/conda-environment.yaml +2023-10-31 23:44:58,369 DEBUG HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done +2023-10-31 23:44:58,371 INFO HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info +2023-10-31 23:44:58,372 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:44:58,372 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive +2023-10-31 23:44:58,373 DEBUG SenderThread:34227 [sender.py:send():380] send: files +2023-10-31 23:44:58,373 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-10-31 23:44:58,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:44:58,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:44:58,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:44:58,836 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-10-31 23:44:59,293 INFO Thread-85 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/wandb-metadata.json +2023-10-31 23:44:59,613 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmpwrgz7o5kwandb/999f99fp-wandb-metadata.json +2023-10-31 23:45:00,837 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:45:05,837 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:45:10,838 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:45:13,384 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:45:13,385 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:45:13,425 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:45:16,765 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:45:21,770 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:45:22,300 INFO Thread-85 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/config.yaml +2023-10-31 23:45:27,130 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:45:28,384 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:45:28,385 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:45:28,425 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:45:32,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:45:37,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:45:42,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:45:43,384 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:45:43,385 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:45:43,425 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:45:48,765 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:45:51,788 DEBUG SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-10-31 23:45:51,791 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:45:53,793 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:45:58,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:45:58,386 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:45:58,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:45:58,799 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:46:03,799 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:46:08,800 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:46:13,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:46:13,386 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:46:13,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:46:14,798 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:46:19,799 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:46:21,794 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:46:25,795 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:46:28,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:46:28,386 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:46:28,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:46:30,813 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:46:35,813 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:46:40,814 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:46:43,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:46:43,386 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:46:43,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:46:46,756 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:46:51,757 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:46:51,797 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:46:56,798 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:46:58,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:46:58,386 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:46:58,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:47:02,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:47:07,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:47:12,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:47:13,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:47:13,386 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:47:13,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:47:18,752 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:47:21,799 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:47:23,800 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:47:28,386 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:47:28,387 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:47:28,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:47:29,756 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:47:34,757 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:47:39,758 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:47:43,386 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:47:43,386 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:47:43,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:47:44,779 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:47:49,779 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:47:51,801 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:47:54,802 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:47:58,386 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:47:58,386 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:47:58,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:48:00,771 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:48:05,772 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:48:10,772 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:48:13,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:48:13,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:48:13,387 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:48:16,765 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:48:21,766 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:48:21,803 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:48:26,804 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:48:28,386 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:48:28,386 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:48:28,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:48:31,821 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:48:36,821 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:48:41,822 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:48:43,386 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:48:43,386 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:48:43,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:48:47,819 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:48:51,805 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:48:53,807 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:48:58,386 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:48:58,386 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:48:58,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:48:59,802 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:49:04,803 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:49:09,804 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:49:13,379 INFO Thread-85 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/output.log +2023-10-31 23:49:13,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:49:13,387 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:49:13,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:49:14,812 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:49:15,380 INFO Thread-85 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/output.log +2023-10-31 23:49:19,813 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:49:21,807 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:49:25,809 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:49:28,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:49:28,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:49:28,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:49:31,755 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:49:36,756 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:49:41,757 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:49:43,386 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:49:43,387 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:49:43,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:49:46,804 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:49:51,805 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:49:51,810 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:49:56,811 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:49:58,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:49:58,387 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:49:58,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:50:02,783 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:50:07,784 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:50:12,785 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:50:13,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:50:13,387 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:50:13,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:50:18,766 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:50:21,815 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:50:23,817 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:50:28,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:50:28,387 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:50:28,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:50:28,836 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:50:33,837 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:50:38,838 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:50:43,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:50:43,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:50:43,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:50:44,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:50:49,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:50:51,814 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:50:54,816 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:50:58,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:50:58,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:50:58,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:50:59,818 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:51:04,818 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:51:09,819 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:51:13,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:51:13,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:51:13,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:51:15,774 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:51:20,774 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:51:21,816 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:51:25,818 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:51:28,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:51:28,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:51:28,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:51:31,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:51:36,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:51:41,789 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:51:43,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:51:43,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:51:43,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:51:46,790 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:51:51,791 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:51:51,820 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:51:56,822 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:51:58,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:51:58,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:51:58,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:52:02,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:52:07,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:52:12,789 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:52:13,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:52:13,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:52:13,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:52:18,773 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:52:21,822 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:52:23,823 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:52:28,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:52:28,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:52:28,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:52:29,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:52:34,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:52:39,789 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:52:43,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:52:43,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:52:43,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:52:44,822 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:52:49,823 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:52:51,824 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:52:54,826 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:52:58,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:52:58,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:52:58,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:52:59,828 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:53:04,828 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:53:09,829 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:53:13,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:53:13,388 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:53:13,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:53:15,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:53:17,469 INFO Thread-85 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/output.log +2023-10-31 23:53:20,374 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:53:21,825 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:53:25,827 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:53:28,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:53:28,389 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:53:28,392 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:53:31,773 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:53:36,774 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:53:41,775 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:53:43,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:53:43,389 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:53:43,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:53:47,762 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:53:51,827 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:53:52,829 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:53:57,830 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:53:58,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:53:58,389 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:53:58,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:54:03,795 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:54:08,796 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:54:13,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:54:13,389 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:54:13,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:54:14,724 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:54:19,725 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:54:21,829 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:54:24,831 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:54:28,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:54:28,390 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:54:28,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:54:30,823 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:54:35,824 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:54:40,825 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:54:43,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:54:43,389 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:54:43,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:54:46,822 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:54:51,823 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:54:51,832 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:54:56,834 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:54:58,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:54:58,389 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:54:58,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:55:02,818 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:55:07,819 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:55:12,820 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:55:13,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:55:13,390 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:55:13,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:55:18,786 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:55:21,834 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:55:23,836 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:55:28,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:55:28,389 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:55:28,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:55:29,728 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:55:34,729 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:55:39,730 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:55:43,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:55:43,390 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:55:43,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:55:44,785 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:55:49,786 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:55:51,837 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:55:54,838 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:55:58,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:55:58,390 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:55:58,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:56:00,823 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:56:05,824 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:56:10,824 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:56:13,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:56:13,390 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:56:13,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:56:16,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:56:21,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:56:21,840 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:56:26,842 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:56:28,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:56:28,390 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:56:28,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:56:32,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:56:37,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:56:42,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:56:43,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:56:43,390 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:56:43,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:56:47,820 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:56:51,841 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:56:52,843 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:56:57,844 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:56:58,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:56:58,390 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:56:58,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:57:03,753 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:57:08,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:57:13,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:57:13,390 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:57:13,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:57:13,831 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:57:18,831 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:57:21,844 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:57:23,846 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:57:28,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:57:28,391 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:57:28,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:57:29,758 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:57:34,759 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:57:39,760 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:57:43,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:57:43,390 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:57:43,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:57:45,738 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:57:50,739 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:57:51,846 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:57:55,847 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:57:58,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:57:58,390 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:57:58,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:58:01,783 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:58:06,783 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:58:11,784 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:58:13,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:58:13,391 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:58:13,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:58:16,798 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:58:21,798 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:58:21,849 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:58:26,850 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:58:28,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:58:28,391 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:58:28,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:58:32,839 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:58:37,839 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:58:42,840 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:58:43,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:58:43,391 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:58:43,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:58:48,792 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:58:51,851 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:58:53,852 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:58:58,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:58:58,391 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:58:58,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:58:59,828 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:59:04,828 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:59:09,829 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:59:13,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:59:13,391 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:59:13,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:59:15,810 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:59:20,810 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:59:21,853 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:59:25,854 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:59:28,391 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:59:28,391 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:59:28,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:59:31,824 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:59:36,825 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:59:41,826 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:59:43,391 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:59:43,391 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:59:43,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-10-31 23:59:47,824 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:59:51,855 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-10-31 23:59:52,857 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:59:57,858 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-10-31 23:59:58,391 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-10-31 23:59:58,391 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-10-31 23:59:58,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:00:03,754 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:00:08,755 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:00:13,391 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:00:13,391 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:00:13,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:00:13,813 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:00:18,813 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:00:21,857 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:00:23,858 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:00:28,391 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:00:28,392 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:00:28,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:00:29,771 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:00:34,771 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:00:39,772 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:00:43,391 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:00:43,392 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:00:43,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:00:44,779 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:00:49,779 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:00:51,859 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:00:54,860 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:00:58,391 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:00:58,392 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:00:58,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:01:00,821 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:01:05,822 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:01:10,823 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:01:13,392 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:01:13,393 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:01:13,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:01:16,802 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:01:21,803 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:01:21,861 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:01:26,862 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:01:28,392 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:01:28,392 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:01:28,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:01:31,868 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:01:36,869 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:01:41,869 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:01:43,392 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:01:43,392 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:01:43,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:01:47,786 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:01:51,864 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:01:52,866 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:01:57,866 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:01:58,392 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:01:58,393 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:01:58,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:02:03,763 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:02:08,764 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:02:13,392 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:02:13,393 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:02:13,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:02:13,781 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:02:18,782 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:02:21,866 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:02:23,867 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:02:28,392 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:02:28,393 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:02:28,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:02:29,744 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:02:34,745 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:02:39,746 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:02:43,392 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:02:43,393 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:02:43,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:02:44,759 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:02:49,760 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:02:51,867 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:02:54,869 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:02:58,392 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:02:58,393 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:02:58,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:03:00,741 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:03:05,742 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:03:10,743 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:03:13,393 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:03:13,393 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:03:13,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:03:15,816 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:03:20,816 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:03:21,870 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:03:25,871 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:03:28,393 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:03:28,393 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:03:28,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:03:31,760 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:03:36,760 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:03:41,761 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:03:43,393 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:03:43,393 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:03:43,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:03:47,734 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:03:51,872 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:03:52,873 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:03:57,874 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:03:58,393 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:03:58,393 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:03:58,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:04:03,734 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:04:08,735 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:04:13,393 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:04:13,393 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:04:13,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:04:13,824 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:04:18,825 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:04:21,874 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:04:23,876 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:04:28,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:04:28,394 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:04:28,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:04:29,764 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:04:34,765 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:04:39,765 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:04:43,393 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:04:43,394 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:04:43,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:04:44,799 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:04:49,800 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:04:51,877 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:04:54,879 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:04:58,393 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:04:58,394 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:04:58,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:05:00,845 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:05:05,845 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:05:10,846 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:05:13,393 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:05:13,394 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:05:13,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:05:16,809 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:05:21,810 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:05:21,879 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:05:26,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:05:28,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:05:28,394 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:05:28,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:05:32,758 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:05:37,759 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:05:42,759 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:05:43,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:05:43,394 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:05:43,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:05:47,808 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:05:51,882 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:05:52,883 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:05:57,884 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:05:58,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:05:58,394 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:05:58,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:06:03,845 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:06:08,846 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:06:13,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:06:13,395 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:06:13,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:06:14,835 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:06:19,836 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:06:21,885 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:06:24,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:06:28,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:06:28,394 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:06:28,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:06:30,796 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:06:35,796 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:06:40,797 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:06:43,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:06:43,395 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:06:43,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:06:46,787 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:06:51,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:06:51,888 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:06:56,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:06:58,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:06:58,395 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:06:58,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:07:02,831 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:07:07,832 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:07:12,832 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:07:13,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:07:13,395 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:07:13,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:07:18,799 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:07:21,889 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:07:23,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:07:28,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:07:28,395 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:07:28,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:07:29,827 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:07:34,827 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:07:39,828 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:07:43,395 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:07:43,395 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:07:43,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:07:45,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:07:50,788 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:07:51,891 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:07:55,892 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:07:58,396 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:07:58,396 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:07:58,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:08:01,781 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:08:06,781 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:08:11,782 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:08:13,396 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:08:13,396 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:08:13,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:08:17,747 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:08:21,893 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:08:22,895 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:08:27,895 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:08:28,396 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:08:28,396 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:08:28,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:08:33,834 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:08:38,835 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:08:43,396 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:08:43,397 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:08:43,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:08:44,767 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:08:49,768 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:08:51,895 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:08:54,896 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:08:58,396 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:08:58,396 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:08:58,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:09:00,770 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:09:05,771 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:09:10,772 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:09:13,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:09:13,397 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:09:13,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:09:15,780 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:09:20,780 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:09:21,897 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:09:25,898 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:09:28,396 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:09:28,397 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:09:28,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:09:31,790 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:09:36,791 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:09:41,792 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:09:43,396 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:09:43,397 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:09:43,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:09:46,847 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:09:51,848 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:09:51,899 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:09:56,901 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:09:58,396 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:09:58,397 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:09:58,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:10:02,821 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:10:07,822 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:10:12,823 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:10:13,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:10:13,397 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:10:13,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:10:18,768 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:10:21,901 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:10:23,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:10:28,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:10:28,397 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:10:28,441 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:10:29,780 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:10:34,781 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:10:39,782 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:10:43,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:10:43,397 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:10:43,437 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:10:45,766 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:10:50,767 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:10:51,903 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:10:55,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:10:58,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:10:58,398 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:10:58,441 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:11:01,851 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:11:06,852 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:11:11,852 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:11:13,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:11:13,397 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:11:13,441 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:11:17,718 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:11:21,905 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:11:22,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:11:27,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:11:28,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:11:28,398 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:11:28,441 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:11:33,850 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:11:38,850 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:11:43,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:11:43,398 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:11:43,441 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:11:44,781 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:11:49,782 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:11:51,908 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:11:54,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:11:58,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:11:58,398 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:11:58,441 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:12:00,806 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:12:05,806 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:12:10,807 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:12:13,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:12:13,398 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:12:13,441 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:12:16,796 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:12:21,797 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:12:21,909 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:12:26,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:12:28,398 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:12:28,398 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:12:28,441 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:12:32,762 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:12:37,763 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:12:42,764 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:12:43,398 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:12:43,398 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:12:43,441 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:12:47,771 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:12:51,912 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:12:52,913 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:12:57,914 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:12:58,398 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:12:58,398 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:12:58,447 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:13:03,780 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:13:08,780 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:13:13,398 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:13:13,398 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:13:13,447 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:13:14,723 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:13:19,723 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:13:21,915 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:13:24,916 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:13:28,398 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:13:28,398 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:13:28,447 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:13:30,766 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:13:35,767 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:13:40,768 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:13:43,398 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:13:43,398 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:13:43,446 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:13:43,808 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history +2023-11-01 00:13:43,810 DEBUG SenderThread:34227 [sender.py:send():380] send: history +2023-11-01 00:13:43,810 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: summary_record +2023-11-01 00:13:43,811 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 00:13:43,849 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-11-01 00:13:43,850 DEBUG SenderThread:34227 [sender.py:send():380] send: exit +2023-11-01 00:13:43,850 INFO SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0 +2023-11-01 00:13:43,850 INFO SenderThread:34227 [sender.py:send_exit():587] handling runtime: 1732 +2023-11-01 00:13:43,850 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 00:13:43,851 INFO SenderThread:34227 [sender.py:send_exit():593] send defer +2023-11-01 00:13:43,851 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:43,851 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0 +2023-11-01 00:13:43,851 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:43,852 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-11-01 00:13:43,852 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 1 +2023-11-01 00:13:43,852 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:43,852 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1 +2023-11-01 00:13:43,852 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:43,852 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-11-01 00:13:43,853 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 2 +2023-11-01 00:13:43,853 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:43,853 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2 +2023-11-01 00:13:43,853 INFO HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor +2023-11-01 00:13:43,853 DEBUG SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-11-01 00:13:43,854 DEBUG SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-11-01 00:13:43,855 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor +2023-11-01 00:13:43,856 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor +2023-11-01 00:13:43,917 INFO Thread-85 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/wandb-summary.json +2023-11-01 00:13:45,706 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor +2023-11-01 00:13:45,706 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor +2023-11-01 00:13:45,707 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor +2023-11-01 00:13:45,707 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:13:45,707 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:45,708 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-11-01 00:13:45,708 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 3 +2023-11-01 00:13:45,708 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:13:45,708 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:45,708 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:13:45,709 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3 +2023-11-01 00:13:45,709 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:45,709 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-11-01 00:13:45,709 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 4 +2023-11-01 00:13:45,710 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:45,710 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4 +2023-11-01 00:13:45,710 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:45,710 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-11-01 00:13:45,710 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 5 +2023-11-01 00:13:45,710 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:45,710 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5 +2023-11-01 00:13:45,711 DEBUG SenderThread:34227 [sender.py:send():380] send: summary +2023-11-01 00:13:45,711 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 00:13:45,712 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:45,712 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-11-01 00:13:45,712 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 6 +2023-11-01 00:13:45,712 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:45,712 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6 +2023-11-01 00:13:45,712 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:45,712 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-11-01 00:13:45,717 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:13:45,810 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:13:45,918 INFO Thread-85 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/wandb-summary.json +2023-11-01 00:13:46,100 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 7 +2023-11-01 00:13:46,101 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:13:46,101 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:46,101 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7 +2023-11-01 00:13:46,102 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:46,102 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-11-01 00:13:46,402 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 8 +2023-11-01 00:13:46,402 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:46,403 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8 +2023-11-01 00:13:46,403 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:46,403 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-11-01 00:13:46,403 INFO SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact +2023-11-01 00:13:46,404 INFO SenderThread:34227 [job_builder.py:_get_source_type():404] no source found +2023-11-01 00:13:46,404 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 9 +2023-11-01 00:13:46,404 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:46,405 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9 +2023-11-01 00:13:46,405 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:46,405 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-11-01 00:13:46,405 INFO SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher +2023-11-01 00:13:46,810 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:13:46,918 INFO Thread-85 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/output.log +2023-11-01 00:13:46,919 INFO SenderThread:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/config.yaml +2023-11-01 00:13:46,919 INFO SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files +2023-11-01 00:13:46,920 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/output.log output.log +2023-11-01 00:13:46,920 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/requirements.txt requirements.txt +2023-11-01 00:13:46,927 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/conda-environment.yaml conda-environment.yaml +2023-11-01 00:13:46,928 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/wandb-summary.json wandb-summary.json +2023-11-01 00:13:46,939 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/config.yaml config.yaml +2023-11-01 00:13:46,939 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/wandb-metadata.json wandb-metadata.json +2023-11-01 00:13:46,945 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 10 +2023-11-01 00:13:46,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:13:46,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:46,952 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10 +2023-11-01 00:13:46,954 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:46,954 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-11-01 00:13:46,955 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-11-01 00:13:47,811 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:13:47,812 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:13:47,911 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/output.log +2023-11-01 00:13:48,140 INFO wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/config.yaml +2023-11-01 00:13:48,154 INFO wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/conda-environment.yaml +2023-11-01 00:13:48,154 INFO wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/wandb-summary.json +2023-11-01 00:13:48,160 INFO wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/files/requirements.txt +2023-11-01 00:13:48,360 INFO Thread-84 :34227 [sender.py:transition_state():613] send defer: 11 +2023-11-01 00:13:48,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:48,361 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11 +2023-11-01 00:13:48,361 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:48,362 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-11-01 00:13:48,362 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher +2023-11-01 00:13:48,362 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 12 +2023-11-01 00:13:48,362 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:48,362 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12 +2023-11-01 00:13:48,363 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:48,363 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-11-01 00:13:48,363 INFO SenderThread:34227 [file_stream.py:finish():595] file stream finish called +2023-11-01 00:13:48,812 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:13:49,523 INFO SenderThread:34227 [file_stream.py:finish():599] file stream finish is done +2023-11-01 00:13:49,523 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 13 +2023-11-01 00:13:49,523 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:13:49,524 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:49,524 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13 +2023-11-01 00:13:49,525 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:49,525 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-11-01 00:13:49,525 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 14 +2023-11-01 00:13:49,526 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:13:49,526 DEBUG SenderThread:34227 [sender.py:send():380] send: final +2023-11-01 00:13:49,526 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14 +2023-11-01 00:13:49,526 DEBUG SenderThread:34227 [sender.py:send():380] send: footer +2023-11-01 00:13:49,527 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:13:49,527 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-11-01 00:13:49,528 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:13:49,529 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:13:49,530 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:13:49,530 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info +2023-11-01 00:13:49,531 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary +2023-11-01 00:13:49,531 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history +2023-11-01 00:13:49,531 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: server_info +2023-11-01 00:13:49,532 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info +2023-11-01 00:13:49,877 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: job_info +2023-11-01 00:13:49,878 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown +2023-11-01 00:13:49,878 INFO HandlerThread:34227 [handler.py:finish():866] shutting down handler +2023-11-01 00:13:50,538 INFO WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/run-7nbh63pb.wandb +2023-11-01 00:13:50,877 INFO SenderThread:34227 [sender.py:finish():1534] shutting down sender +2023-11-01 00:13:50,878 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-11-01 00:13:50,878 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231031_234449-7nbh63pb/logs/debug.log b/wandb/run-20231031_234449-7nbh63pb/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..158bd765a6aa70474334dc9f9694f3bb655c7aed --- /dev/null +++ b/wandb/run-20231031_234449-7nbh63pb/logs/debug.log @@ -0,0 +1,34 @@ +2023-10-31 23:44:49,963 INFO MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-10-31 23:44:49,963 INFO MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070 +2023-10-31 23:44:49,963 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-10-31 23:44:49,963 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-10-31 23:44:49,963 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-10-31 23:44:49,963 INFO MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-10-31 23:44:49,963 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 23:44:49,963 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-10-31 23:44:49,963 INFO MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/logs/debug.log +2023-10-31 23:44:49,963 INFO MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb/logs/debug-internal.log +2023-10-31 23:44:49,964 INFO MainThread:34070 [wandb_init.py:init():568] calling init triggers +2023-10-31 23:44:49,964 INFO MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'RMSprop', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 35} +2023-10-31 23:44:49,964 INFO MainThread:34070 [wandb_init.py:init():618] starting backend +2023-10-31 23:44:49,964 INFO MainThread:34070 [wandb_init.py:init():622] setting up manager +2023-10-31 23:44:49,965 INFO MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-10-31 23:44:49,966 INFO MainThread:34070 [wandb_init.py:init():628] backend started and connected +2023-10-31 23:44:49,968 INFO MainThread:34070 [wandb_init.py:init():720] updated telemetry +2023-10-31 23:44:49,969 INFO MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-10-31 23:44:51,303 INFO MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version +2023-10-31 23:44:51,770 INFO MainThread:34070 [wandb_run.py:_on_init():2229] got version response +2023-10-31 23:44:51,770 INFO MainThread:34070 [wandb_init.py:init():804] starting run threads in backend +2023-10-31 23:44:58,385 INFO MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg +2023-10-31 23:44:58,385 INFO MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-10-31 23:44:58,386 INFO MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-10-31 23:44:58,386 INFO MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed. +2023-10-31 23:44:58,387 INFO MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process +2023-11-01 00:13:43,808 INFO MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/7nbh63pb +2023-11-01 00:13:43,808 INFO MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0 +2023-11-01 00:13:43,808 INFO MainThread:34070 [wandb_run.py:_restore():2151] restore +2023-11-01 00:13:43,808 INFO MainThread:34070 [wandb_run.py:_restore():2157] restore done +2023-11-01 00:13:50,880 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-11-01 00:13:50,881 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-11-01 00:13:50,888 INFO MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files diff --git a/wandb/run-20231031_234449-7nbh63pb/run-7nbh63pb.wandb b/wandb/run-20231031_234449-7nbh63pb/run-7nbh63pb.wandb new file mode 100644 index 0000000000000000000000000000000000000000..008dcec95984762a4bc6c7063d7e5d9a2e969bb8 Binary files /dev/null and b/wandb/run-20231031_234449-7nbh63pb/run-7nbh63pb.wandb differ diff --git a/wandb/run-20231101_001351-jyel0gll/files/conda-environment.yaml b/wandb/run-20231101_001351-jyel0gll/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231101_001351-jyel0gll/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231101_001351-jyel0gll/files/config.yaml b/wandb/run-20231101_001351-jyel0gll/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec19038b096c1b6aef4d3d34cf554242634ebdb7 --- /dev/null +++ b/wandb/run-20231101_001351-jyel0gll/files/config.yaml @@ -0,0 +1,53 @@ +wandb_version: 1 + +optimizer: + desc: null + value: Adam +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 29 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698783231.971459 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 2 + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231101_001351-jyel0gll/files/output.log b/wandb/run-20231101_001351-jyel0gll/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3209b44297ae1951fca862d06bd512462925a40e --- /dev/null +++ b/wandb/run-20231101_001351-jyel0gll/files/output.log @@ -0,0 +1,2 @@ + +Saving Model At: save_model/best_model.pth \ No newline at end of file diff --git a/wandb/run-20231101_001351-jyel0gll/files/requirements.txt b/wandb/run-20231101_001351-jyel0gll/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231101_001351-jyel0gll/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231101_001351-jyel0gll/files/wandb-metadata.json b/wandb/run-20231101_001351-jyel0gll/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..895e6ec0cf14d41cc63b12ac01155be229d11308 --- /dev/null +++ b/wandb/run-20231101_001351-jyel0gll/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T20:13:53.348879", + "startedAt": "2023-10-31T20:13:51.966224", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.5512000000000001, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.497, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.205, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.54, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.589, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.5, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.196, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.501, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.691, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.699, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.522, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.592, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.598, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.207, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.196, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.9305191040039 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231101_001351-jyel0gll/files/wandb-summary.json b/wandb/run-20231101_001351-jyel0gll/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8a7e8127c891887d46ddb1702b1b288b3f44e4de --- /dev/null +++ b/wandb/run-20231101_001351-jyel0gll/files/wandb-summary.json @@ -0,0 +1 @@ +{"Train Loss": 1.3920401334762573, "Train Accuracy": 0.5099660754203796, "Validation Loss": 1.2373844385147095, "Validation Accuracy": 0.5550567507743835, "_timestamp": 1698784695.2182117, "_runtime": 1463.2467527389526, "_step": 0, "_wandb": {"runtime": 1461}} \ No newline at end of file diff --git a/wandb/run-20231101_001351-jyel0gll/logs/debug-internal.log b/wandb/run-20231101_001351-jyel0gll/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2582cbae9d8ac0a3f03d363c8b14793c2bb384d4 --- /dev/null +++ b/wandb/run-20231101_001351-jyel0gll/logs/debug-internal.log @@ -0,0 +1,810 @@ +2023-11-01 00:13:51,974 INFO StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-11-01 00:13:51.973185 +2023-11-01 00:13:51,976 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status +2023-11-01 00:13:51,977 INFO WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/run-jyel0gll.wandb +2023-11-01 00:13:51,981 DEBUG SenderThread:34227 [sender.py:send():380] send: header +2023-11-01 00:13:51,981 DEBUG SenderThread:34227 [sender.py:send():380] send: run +2023-11-01 00:13:52,869 INFO SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files +2023-11-01 00:13:52,869 INFO SenderThread:34227 [sender.py:_start_run_threads():1122] run started: jyel0gll with start time 1698783231.971459 +2023-11-01 00:13:52,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version +2023-11-01 00:13:52,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: check_version +2023-11-01 00:13:53,284 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start +2023-11-01 00:13:53,285 DEBUG HandlerThread:34227 [system_info.py:__init__():32] System info init +2023-11-01 00:13:53,286 DEBUG HandlerThread:34227 [system_info.py:__init__():47] System info init done +2023-11-01 00:13:53,286 INFO HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor +2023-11-01 00:13:53,286 INFO SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-11-01 00:13:53,286 INFO HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info +2023-11-01 00:13:53,287 INFO SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring +2023-11-01 00:13:53,288 INFO SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring +2023-11-01 00:13:53,289 INFO SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring +2023-11-01 00:13:53,290 INFO SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring +2023-11-01 00:13:53,291 INFO SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring +2023-11-01 00:13:53,348 DEBUG HandlerThread:34227 [system_info.py:probe():196] Probing system +2023-11-01 00:13:53,349 DEBUG HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid +2023-11-01 00:13:53,349 DEBUG HandlerThread:34227 [system_info.py:probe():244] Probing system done +2023-11-01 00:13:53,349 DEBUG HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T20:13:53.348879', 'startedAt': '2023-10-31T20:13:51.966224', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.5512000000000001, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.497, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.599, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.54, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.589, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.5, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.196, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.501, 'min': 1200.0, 'max': 3600.0}, {'current': 2.691, 'min': 1200.0, 'max': 3600.0}, {'current': 2.699, 'min': 1200.0, 'max': 3600.0}, {'current': 1.522, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.592, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.598, 'min': 1200.0, 'max': 3600.0}, {'current': 2.207, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.196, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9305191040039}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-11-01 00:13:53,349 INFO HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info +2023-11-01 00:13:53,349 INFO HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info +2023-11-01 00:13:53,349 DEBUG HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-11-01 00:13:53,350 DEBUG HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done +2023-11-01 00:13:53,350 DEBUG HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-11-01 00:13:53,873 INFO Thread-97 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/requirements.txt +2023-11-01 00:13:53,874 INFO Thread-97 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/conda-environment.yaml +2023-11-01 00:13:59,865 DEBUG HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done +2023-11-01 00:13:59,867 INFO HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info +2023-11-01 00:13:59,868 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:13:59,868 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive +2023-11-01 00:13:59,869 DEBUG SenderThread:34227 [sender.py:send():380] send: files +2023-11-01 00:13:59,869 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-11-01 00:13:59,880 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:13:59,881 INFO Thread-97 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/conda-environment.yaml +2023-11-01 00:13:59,882 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:13:59,882 INFO Thread-97 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/wandb-metadata.json +2023-11-01 00:13:59,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:14:00,434 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-11-01 00:14:01,099 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmp7iiqaqsxwandb/xfo3m5q5-wandb-metadata.json +2023-11-01 00:14:02,435 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:14:07,435 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:14:12,436 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:14:14,879 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:14:14,880 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:14:14,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:14:18,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:14:23,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:14:23,888 INFO Thread-97 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/config.yaml +2023-11-01 00:14:28,692 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:14:29,879 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:14:29,880 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:14:29,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:14:34,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:14:39,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:14:44,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:14:44,879 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:14:44,880 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:14:44,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:14:50,213 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:14:53,294 DEBUG SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-11-01 00:14:53,296 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:14:55,298 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:14:59,880 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:14:59,880 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:14:59,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:15:00,323 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:15:05,323 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:15:10,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:15:14,880 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:15:14,881 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:15:14,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:15:16,285 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:15:21,285 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:15:23,298 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:15:26,299 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:15:29,880 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:15:29,881 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:15:29,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:15:32,251 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:15:37,252 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:15:42,252 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:15:44,880 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:15:44,881 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:15:44,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:15:47,300 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:15:52,301 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:15:53,300 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:15:57,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:15:59,880 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:15:59,881 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:15:59,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:16:03,282 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:16:08,283 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:16:13,284 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:16:14,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:16:14,882 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:16:14,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:16:18,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:16:23,303 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:16:24,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:16:29,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:16:29,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:16:29,881 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:16:29,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:16:35,278 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:16:40,279 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:16:44,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:16:44,881 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:16:44,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:16:46,258 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:16:51,259 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:16:53,304 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:16:56,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:16:59,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:16:59,881 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:16:59,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:17:01,311 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:17:06,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:17:11,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:17:14,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:17:14,881 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:17:14,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:17:16,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:17:21,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:17:23,307 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:17:27,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:17:29,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:17:29,882 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:17:29,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:17:33,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:17:38,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:17:43,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:17:44,882 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:17:44,882 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:17:44,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:17:49,301 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:17:53,310 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:17:54,311 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:17:59,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:17:59,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:17:59,881 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:17:59,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:18:05,251 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:18:08,964 INFO Thread-97 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/output.log +2023-11-01 00:18:10,673 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:18:10,965 INFO Thread-97 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/output.log +2023-11-01 00:18:14,882 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:18:14,883 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:18:14,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:18:16,254 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:18:21,254 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:18:23,313 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:18:26,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:18:29,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:18:29,882 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:18:29,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:18:32,248 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:18:37,248 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:18:42,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:18:44,883 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:18:44,884 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:18:44,884 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:18:47,258 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:18:52,259 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:18:53,315 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:18:57,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:18:59,883 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:18:59,883 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:18:59,884 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:19:03,230 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:19:08,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:19:13,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:19:14,883 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:19:14,884 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:19:14,884 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:19:18,267 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:19:23,268 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:19:23,316 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:19:28,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:19:29,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:19:29,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:19:29,886 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:19:34,262 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:19:39,263 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:19:44,264 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:19:44,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:19:44,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:19:44,886 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:19:50,246 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:19:53,318 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:19:55,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:19:59,884 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:19:59,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:19:59,886 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:20:01,239 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:20:06,240 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:20:11,240 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:20:14,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:20:14,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:20:14,886 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:20:16,263 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:20:21,264 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:20:23,320 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:20:26,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:20:29,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:20:29,886 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:20:29,929 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:20:32,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:20:37,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:20:42,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:20:44,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:20:44,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:20:44,886 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:20:48,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:20:53,266 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:20:53,323 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:20:58,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:20:59,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:20:59,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:20:59,887 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:21:04,292 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:21:09,292 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:21:14,293 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:21:14,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:21:14,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:21:14,886 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:21:19,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:21:23,325 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:21:24,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:21:29,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:21:29,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:21:29,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:21:29,887 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:21:34,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:21:39,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:21:44,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:21:44,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:21:44,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:21:44,887 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:21:50,253 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:21:53,327 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:21:55,329 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:21:59,885 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:21:59,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:21:59,887 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:22:01,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:22:06,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:22:11,266 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:22:14,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:22:14,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:22:14,888 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:22:16,287 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:22:21,288 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:22:23,328 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:22:26,330 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:22:29,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:22:29,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:22:29,887 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:22:32,272 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:22:37,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:22:42,274 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:22:44,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:22:44,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:22:44,888 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:22:47,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:22:52,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:22:53,330 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:22:57,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:22:59,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:22:59,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:22:59,888 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:23:03,283 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:23:08,284 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:23:13,285 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:23:14,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:23:14,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:23:14,888 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:23:19,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:23:23,333 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:23:24,334 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:23:29,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:23:29,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:23:29,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:23:29,888 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:23:35,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:23:40,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:23:44,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:23:44,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:23:44,888 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:23:46,283 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:23:51,284 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:23:53,335 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:23:56,336 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:23:59,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:23:59,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:23:59,888 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:24:02,294 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:24:07,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:24:12,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:24:14,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:24:14,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:24:14,888 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:24:18,274 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:24:23,275 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:24:23,337 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:24:28,339 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:24:29,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:24:29,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:24:29,888 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:24:34,224 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:24:39,225 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:24:44,225 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:24:44,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:24:44,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:24:44,888 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:24:50,211 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:24:53,339 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:24:55,340 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:24:59,886 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:24:59,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:24:59,889 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:25:01,300 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:25:06,301 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:25:11,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:25:14,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:25:14,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:25:14,889 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:25:17,211 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:25:22,212 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:25:23,341 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:25:27,343 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:25:29,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:25:29,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:25:29,889 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:25:33,235 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:25:38,235 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:25:43,236 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:25:44,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:25:44,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:25:44,889 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:25:48,300 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:25:53,301 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:25:53,343 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:25:58,344 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:25:59,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:25:59,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:25:59,889 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:26:04,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:26:09,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:26:14,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:26:14,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:26:14,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:26:14,890 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:26:19,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:26:23,346 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:26:24,347 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:26:29,348 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:26:29,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:26:29,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:26:29,890 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:26:35,257 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:26:40,258 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:26:44,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:26:44,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:26:44,890 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:26:46,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:26:51,217 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:26:53,348 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:26:56,350 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:26:59,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:26:59,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:26:59,889 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:27:02,336 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:27:07,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:27:12,338 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:27:14,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:27:14,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:27:14,889 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:27:18,244 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:27:23,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:27:23,350 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:27:28,352 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:27:29,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:27:29,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:27:29,890 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:27:34,259 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:27:39,260 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:27:44,261 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:27:44,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:27:44,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:27:44,890 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:27:49,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:27:53,353 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:27:54,354 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:27:59,355 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:27:59,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:27:59,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:27:59,890 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:28:05,255 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:28:10,255 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:28:14,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:28:14,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:28:14,890 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:28:16,229 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:28:21,229 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:28:23,355 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:28:26,356 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:28:29,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:28:29,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:28:29,890 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:28:32,330 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:28:37,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:28:42,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:28:44,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:28:44,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:28:44,890 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:28:48,323 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:28:53,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:28:53,357 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:28:58,359 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:28:59,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:28:59,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:28:59,890 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:29:04,301 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:29:09,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:29:14,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:29:14,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:29:14,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:29:14,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:29:20,275 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:29:23,358 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:29:25,359 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:29:29,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:29:29,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:29:29,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:29:31,270 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:29:36,271 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:29:41,271 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:29:44,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:29:44,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:29:44,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:29:47,219 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:29:52,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:29:53,361 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:29:57,362 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:29:59,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:29:59,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:29:59,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:30:03,301 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:30:08,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:30:13,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:30:14,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:30:14,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:30:14,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:30:19,283 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:30:23,363 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:30:24,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:30:29,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:30:29,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:30:29,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:30:29,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:30:35,247 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:30:40,248 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:30:44,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:30:44,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:30:44,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:30:45,293 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:30:50,294 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:30:53,365 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:30:55,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:30:59,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:30:59,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:30:59,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:31:01,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:31:06,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:31:11,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:31:14,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:31:14,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:31:14,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:31:17,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:31:22,227 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:31:23,368 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:31:27,370 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:31:29,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:31:29,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:31:29,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:31:33,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:31:38,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:31:43,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:31:44,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:31:44,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:31:44,892 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:31:49,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:31:53,369 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:31:54,371 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:31:59,371 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:31:59,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:31:59,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:31:59,892 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:32:05,225 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:32:10,226 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:32:14,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:32:14,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:32:14,892 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:32:15,275 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:32:20,276 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:32:23,372 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:32:25,373 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:32:29,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:32:29,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:32:29,892 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:32:31,301 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:32:36,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:32:41,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:32:44,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:32:44,892 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:32:44,892 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:32:47,234 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:32:52,235 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:32:53,375 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:32:57,376 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:32:59,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:32:59,892 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:32:59,892 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:33:03,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:33:08,223 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:33:13,224 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:33:14,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:33:14,892 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:33:14,892 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:33:18,281 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:33:23,281 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:33:23,377 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:33:28,379 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:33:29,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:33:29,892 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:33:29,893 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:33:34,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:33:39,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:33:44,323 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:33:44,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:33:44,892 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:33:44,892 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:33:50,248 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:33:53,379 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:33:55,381 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:33:59,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:33:59,892 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:33:59,892 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:34:01,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:34:06,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:34:11,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:34:14,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:34:14,892 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:34:14,893 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:34:17,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:34:22,278 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:34:23,382 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:34:27,383 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:34:29,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:34:29,892 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:34:29,893 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:34:33,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:34:38,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:34:43,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:34:44,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:34:44,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:34:44,893 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:34:49,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:34:53,383 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:34:54,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:34:59,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:34:59,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:34:59,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:34:59,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:35:05,285 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:35:10,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:35:14,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:35:14,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:35:14,893 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:35:15,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:35:20,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:35:23,386 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:35:25,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:35:29,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:35:29,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:35:29,893 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:35:31,248 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:35:36,249 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:35:41,250 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:35:44,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:35:44,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:35:44,893 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:35:46,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:35:51,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:35:53,388 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:35:56,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:35:59,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:35:59,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:35:59,893 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:36:02,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:36:07,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:36:12,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:36:14,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:36:14,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:36:14,893 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:36:17,338 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:36:22,338 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:36:23,392 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:36:27,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:36:29,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:36:29,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:36:29,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:36:33,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:36:38,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:36:43,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:36:44,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:36:44,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:36:44,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:36:48,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:36:53,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:36:53,393 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:36:58,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:36:59,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:36:59,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:36:59,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:37:04,229 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:37:09,230 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:37:14,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:37:14,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:37:14,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:37:14,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:37:19,260 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:37:23,395 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:37:24,396 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:37:29,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:37:29,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:37:29,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:37:29,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:37:35,254 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:37:40,255 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:37:44,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:37:44,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:37:44,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:37:46,236 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:37:51,237 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:37:53,396 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:37:56,398 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:37:59,892 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:37:59,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:37:59,894 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:38:02,299 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:38:07,299 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:38:12,300 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:38:14,892 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:38:14,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:38:14,895 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:38:15,218 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history +2023-11-01 00:38:15,270 DEBUG SenderThread:34227 [sender.py:send():380] send: history +2023-11-01 00:38:15,270 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: summary_record +2023-11-01 00:38:15,271 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 00:38:15,271 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-11-01 00:38:15,271 DEBUG SenderThread:34227 [sender.py:send():380] send: exit +2023-11-01 00:38:15,271 INFO SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0 +2023-11-01 00:38:15,272 INFO SenderThread:34227 [sender.py:send_exit():587] handling runtime: 1461 +2023-11-01 00:38:15,272 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 00:38:15,272 INFO SenderThread:34227 [sender.py:send_exit():593] send defer +2023-11-01 00:38:15,272 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:15,272 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0 +2023-11-01 00:38:15,273 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:15,273 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-11-01 00:38:15,273 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 1 +2023-11-01 00:38:15,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:15,273 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1 +2023-11-01 00:38:15,273 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:15,273 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-11-01 00:38:15,273 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 2 +2023-11-01 00:38:15,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:15,274 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2 +2023-11-01 00:38:15,274 INFO HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor +2023-11-01 00:38:15,274 DEBUG SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-11-01 00:38:15,274 DEBUG SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-11-01 00:38:15,278 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor +2023-11-01 00:38:15,278 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor +2023-11-01 00:38:15,407 INFO Thread-97 :34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/wandb-summary.json +2023-11-01 00:38:16,982 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor +2023-11-01 00:38:16,983 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor +2023-11-01 00:38:16,983 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor +2023-11-01 00:38:16,983 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:38:16,984 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:16,984 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-11-01 00:38:16,984 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 3 +2023-11-01 00:38:16,984 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:38:16,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:16,986 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:38:16,986 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3 +2023-11-01 00:38:16,987 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:16,987 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-11-01 00:38:16,987 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 4 +2023-11-01 00:38:16,987 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:16,988 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4 +2023-11-01 00:38:16,988 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:16,988 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-11-01 00:38:16,988 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 5 +2023-11-01 00:38:16,988 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:16,988 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5 +2023-11-01 00:38:16,989 DEBUG SenderThread:34227 [sender.py:send():380] send: summary +2023-11-01 00:38:16,990 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 00:38:16,990 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:16,990 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-11-01 00:38:16,990 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 6 +2023-11-01 00:38:16,991 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:16,991 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6 +2023-11-01 00:38:16,991 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:16,991 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-11-01 00:38:16,996 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:38:17,220 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:38:17,407 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 7 +2023-11-01 00:38:17,407 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:38:17,407 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:17,408 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7 +2023-11-01 00:38:17,408 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:17,408 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-11-01 00:38:17,409 INFO Thread-97 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/wandb-summary.json +2023-11-01 00:38:17,409 INFO Thread-97 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/config.yaml +2023-11-01 00:38:17,930 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 8 +2023-11-01 00:38:17,930 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:17,930 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8 +2023-11-01 00:38:17,930 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:17,930 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-11-01 00:38:17,931 INFO SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact +2023-11-01 00:38:17,931 INFO SenderThread:34227 [job_builder.py:_get_source_type():404] no source found +2023-11-01 00:38:17,931 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 9 +2023-11-01 00:38:17,932 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:17,932 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9 +2023-11-01 00:38:17,932 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:17,932 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-11-01 00:38:17,932 INFO SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher +2023-11-01 00:38:18,221 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:38:18,409 INFO Thread-97 :34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/output.log +2023-11-01 00:38:18,410 INFO SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files +2023-11-01 00:38:18,411 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/output.log output.log +2023-11-01 00:38:18,411 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/requirements.txt requirements.txt +2023-11-01 00:38:18,412 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/conda-environment.yaml conda-environment.yaml +2023-11-01 00:38:18,418 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/wandb-summary.json wandb-summary.json +2023-11-01 00:38:18,430 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/config.yaml config.yaml +2023-11-01 00:38:18,431 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/wandb-metadata.json wandb-metadata.json +2023-11-01 00:38:18,436 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 10 +2023-11-01 00:38:18,438 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:38:18,438 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:18,445 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10 +2023-11-01 00:38:18,445 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:18,446 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-11-01 00:38:18,446 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-11-01 00:38:19,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:38:19,222 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:38:19,631 INFO wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/config.yaml +2023-11-01 00:38:19,663 INFO wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/wandb-summary.json +2023-11-01 00:38:19,884 INFO wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/conda-environment.yaml +2023-11-01 00:38:20,222 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:38:20,223 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:38:20,413 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/output.log +2023-11-01 00:38:20,780 INFO wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/files/requirements.txt +2023-11-01 00:38:20,980 INFO Thread-96 :34227 [sender.py:transition_state():613] send defer: 11 +2023-11-01 00:38:20,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:20,981 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11 +2023-11-01 00:38:20,982 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:20,982 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-11-01 00:38:20,982 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher +2023-11-01 00:38:20,982 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 12 +2023-11-01 00:38:20,982 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:20,982 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12 +2023-11-01 00:38:20,983 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:20,983 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-11-01 00:38:20,983 INFO SenderThread:34227 [file_stream.py:finish():595] file stream finish called +2023-11-01 00:38:21,223 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:38:22,025 INFO SenderThread:34227 [file_stream.py:finish():599] file stream finish is done +2023-11-01 00:38:22,025 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 13 +2023-11-01 00:38:22,026 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:38:22,026 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:22,026 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13 +2023-11-01 00:38:22,027 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:22,027 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-11-01 00:38:22,027 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 14 +2023-11-01 00:38:22,028 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 00:38:22,028 DEBUG SenderThread:34227 [sender.py:send():380] send: final +2023-11-01 00:38:22,028 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14 +2023-11-01 00:38:22,028 DEBUG SenderThread:34227 [sender.py:send():380] send: footer +2023-11-01 00:38:22,029 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 00:38:22,029 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-11-01 00:38:22,030 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:38:22,031 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 00:38:22,031 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 00:38:22,032 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info +2023-11-01 00:38:22,032 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary +2023-11-01 00:38:22,033 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history +2023-11-01 00:38:22,033 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: server_info +2023-11-01 00:38:22,034 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info +2023-11-01 00:38:22,376 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: job_info +2023-11-01 00:38:22,377 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown +2023-11-01 00:38:22,377 INFO HandlerThread:34227 [handler.py:finish():866] shutting down handler +2023-11-01 00:38:23,039 INFO WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/run-jyel0gll.wandb +2023-11-01 00:38:23,376 INFO SenderThread:34227 [sender.py:finish():1534] shutting down sender +2023-11-01 00:38:23,377 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-11-01 00:38:23,377 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231101_001351-jyel0gll/logs/debug.log b/wandb/run-20231101_001351-jyel0gll/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b7c6f7a74a7a75aa8efac673188ee4f1333c30da --- /dev/null +++ b/wandb/run-20231101_001351-jyel0gll/logs/debug.log @@ -0,0 +1,34 @@ +2023-11-01 00:13:51,967 INFO MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070 +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/logs/debug.log +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll/logs/debug-internal.log +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_init.py:init():568] calling init triggers +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'Adam', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 29} +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_init.py:init():618] starting backend +2023-11-01 00:13:51,968 INFO MainThread:34070 [wandb_init.py:init():622] setting up manager +2023-11-01 00:13:51,970 INFO MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-11-01 00:13:51,971 INFO MainThread:34070 [wandb_init.py:init():628] backend started and connected +2023-11-01 00:13:51,976 INFO MainThread:34070 [wandb_init.py:init():720] updated telemetry +2023-11-01 00:13:51,977 INFO MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-11-01 00:13:52,890 INFO MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version +2023-11-01 00:13:53,273 INFO MainThread:34070 [wandb_run.py:_on_init():2229] got version response +2023-11-01 00:13:53,273 INFO MainThread:34070 [wandb_init.py:init():804] starting run threads in backend +2023-11-01 00:13:59,880 INFO MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg +2023-11-01 00:13:59,881 INFO MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-11-01 00:13:59,881 INFO MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-11-01 00:13:59,881 INFO MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed. +2023-11-01 00:13:59,882 INFO MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process +2023-11-01 00:38:15,218 INFO MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/jyel0gll +2023-11-01 00:38:15,218 INFO MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0 +2023-11-01 00:38:15,219 INFO MainThread:34070 [wandb_run.py:_restore():2151] restore +2023-11-01 00:38:15,219 INFO MainThread:34070 [wandb_run.py:_restore():2157] restore done +2023-11-01 00:38:23,380 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-11-01 00:38:23,380 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-11-01 00:38:23,387 INFO MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files diff --git a/wandb/run-20231101_001351-jyel0gll/run-jyel0gll.wandb b/wandb/run-20231101_001351-jyel0gll/run-jyel0gll.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ae78bcac7b868103c3a14f1513f63f194f30717e Binary files /dev/null and b/wandb/run-20231101_001351-jyel0gll/run-jyel0gll.wandb differ diff --git a/wandb/run-20231101_003823-t9dkdynt/files/conda-environment.yaml b/wandb/run-20231101_003823-t9dkdynt/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231101_003823-t9dkdynt/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231101_003823-t9dkdynt/files/config.yaml b/wandb/run-20231101_003823-t9dkdynt/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88e6ecfa908408621b895a21ae01f6d4b6d44dfb --- /dev/null +++ b/wandb/run-20231101_003823-t9dkdynt/files/config.yaml @@ -0,0 +1,53 @@ +wandb_version: 1 + +optimizer: + desc: null + value: SGD +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 35 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698784703.88423 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 2 + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231101_003823-t9dkdynt/files/output.log b/wandb/run-20231101_003823-t9dkdynt/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3e3243d813ebee0d393d6b3779843d64e779d465 --- /dev/null +++ b/wandb/run-20231101_003823-t9dkdynt/files/output.log @@ -0,0 +1,3 @@ + +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth \ No newline at end of file diff --git a/wandb/run-20231101_003823-t9dkdynt/files/requirements.txt b/wandb/run-20231101_003823-t9dkdynt/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231101_003823-t9dkdynt/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231101_003823-t9dkdynt/files/wandb-metadata.json b/wandb/run-20231101_003823-t9dkdynt/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..58fa12eb2be7ce4cffe4524e8ee5ce3865c4de81 --- /dev/null +++ b/wandb/run-20231101_003823-t9dkdynt/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T20:38:26.986824", + "startedAt": "2023-10-31T20:38:23.879960", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.4736500000000001, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.455, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.598, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.201, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.204, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.205, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.207, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.192, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.421, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.599, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.211, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.7, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.208, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.205, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.208, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.6, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.9304962158203 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231101_003823-t9dkdynt/files/wandb-summary.json b/wandb/run-20231101_003823-t9dkdynt/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1653610fc339597522e3db0158eb118a9e8d02b1 --- /dev/null +++ b/wandb/run-20231101_003823-t9dkdynt/files/wandb-summary.json @@ -0,0 +1 @@ +{"Train Loss": 1.3817832469940186, "Train Accuracy": 0.5096835494041443, "Validation Loss": 1.2287707328796387, "Validation Accuracy": 0.5589190125465393, "_timestamp": 1698787119.671722, "_runtime": 2415.7874920368195, "_step": 0, "_wandb": {"runtime": 2412}} \ No newline at end of file diff --git a/wandb/run-20231101_003823-t9dkdynt/logs/debug-internal.log b/wandb/run-20231101_003823-t9dkdynt/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e2916a8042a01375fe51b0311117fb79dee8b5e4 --- /dev/null +++ b/wandb/run-20231101_003823-t9dkdynt/logs/debug-internal.log @@ -0,0 +1,1215 @@ +2023-11-01 00:38:24,382 INFO StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-11-01 00:38:24.381117 +2023-11-01 00:38:24,384 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status +2023-11-01 00:38:24,386 INFO WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/run-t9dkdynt.wandb +2023-11-01 00:38:24,388 DEBUG SenderThread:34227 [sender.py:send():380] send: header +2023-11-01 00:38:24,389 DEBUG SenderThread:34227 [sender.py:send():380] send: run +2023-11-01 00:38:26,205 INFO SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files +2023-11-01 00:38:26,205 INFO SenderThread:34227 [sender.py:_start_run_threads():1122] run started: t9dkdynt with start time 1698784703.88423 +2023-11-01 00:38:26,228 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version +2023-11-01 00:38:26,229 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: check_version +2023-11-01 00:38:26,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start +2023-11-01 00:38:26,912 DEBUG HandlerThread:34227 [system_info.py:__init__():32] System info init +2023-11-01 00:38:26,912 DEBUG HandlerThread:34227 [system_info.py:__init__():47] System info init done +2023-11-01 00:38:26,912 INFO HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor +2023-11-01 00:38:26,912 INFO SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-11-01 00:38:26,912 INFO HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info +2023-11-01 00:38:26,913 INFO SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring +2023-11-01 00:38:26,914 INFO SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring +2023-11-01 00:38:26,915 INFO SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring +2023-11-01 00:38:26,916 INFO SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring +2023-11-01 00:38:26,917 INFO SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring +2023-11-01 00:38:26,986 DEBUG HandlerThread:34227 [system_info.py:probe():196] Probing system +2023-11-01 00:38:26,987 DEBUG HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid +2023-11-01 00:38:26,987 DEBUG HandlerThread:34227 [system_info.py:probe():244] Probing system done +2023-11-01 00:38:26,987 DEBUG HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T20:38:26.986824', 'startedAt': '2023-10-31T20:38:23.879960', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.4736500000000001, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.455, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.598, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.201, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.204, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.207, 'min': 1200.0, 'max': 3600.0}, {'current': 1.192, 'min': 1200.0, 'max': 3600.0}, {'current': 1.599, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.421, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.599, 'min': 1200.0, 'max': 3600.0}, {'current': 2.211, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 2.7, 'min': 1200.0, 'max': 3600.0}, {'current': 1.208, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.208, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.6, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9304962158203}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-11-01 00:38:26,987 INFO HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info +2023-11-01 00:38:26,987 INFO HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info +2023-11-01 00:38:26,987 DEBUG HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-11-01 00:38:26,988 DEBUG HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done +2023-11-01 00:38:26,988 DEBUG HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-11-01 00:38:27,210 INFO Thread-109:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/conda-environment.yaml +2023-11-01 00:38:27,211 INFO Thread-109:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/requirements.txt +2023-11-01 00:38:33,524 DEBUG HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done +2023-11-01 00:38:33,526 INFO HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info +2023-11-01 00:38:33,527 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:38:33,527 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive +2023-11-01 00:38:33,528 DEBUG SenderThread:34227 [sender.py:send():380] send: files +2023-11-01 00:38:33,529 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-11-01 00:38:33,542 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:38:33,543 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:38:33,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:38:34,038 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-11-01 00:38:34,209 INFO Thread-109:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/conda-environment.yaml +2023-11-01 00:38:34,209 INFO Thread-109:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/wandb-metadata.json +2023-11-01 00:38:34,756 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmp6h8e9lf3wandb/gzxt7zno-wandb-metadata.json +2023-11-01 00:38:35,038 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:38:40,039 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:38:45,039 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:38:48,540 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:38:48,541 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:38:48,581 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:38:50,897 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:38:55,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:38:57,216 INFO Thread-109:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/config.yaml +2023-11-01 00:39:01,272 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:39:03,541 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:39:03,542 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:39:03,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:39:06,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:39:11,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:39:16,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:39:18,541 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:39:18,541 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:39:18,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:39:22,022 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:39:26,918 DEBUG SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-11-01 00:39:26,922 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:39:27,924 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:39:32,924 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:39:33,541 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:39:33,541 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:39:33,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:39:37,929 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:39:42,930 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:39:47,930 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:39:48,540 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:39:48,541 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:39:48,581 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:39:52,975 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:39:56,924 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:39:58,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:40:03,540 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:40:03,541 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:40:03,581 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:40:03,931 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:40:08,931 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:40:13,932 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:40:18,540 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:40:18,541 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:40:18,581 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:40:19,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:40:24,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:40:26,926 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:40:29,927 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:40:33,541 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:40:33,541 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:40:33,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:40:35,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:40:40,922 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:40:45,923 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:40:48,541 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:40:48,541 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:40:48,581 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:40:50,926 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:40:55,926 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:40:56,929 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:41:00,931 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:41:03,541 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:41:03,541 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:41:03,581 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:41:06,908 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:41:11,908 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:41:16,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:41:18,541 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:41:18,542 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:41:18,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:41:21,924 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:41:26,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:41:26,931 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:41:31,933 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:41:33,541 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:41:33,542 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:41:33,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:41:36,976 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:41:41,977 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:41:46,977 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:41:48,541 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:41:48,541 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:41:48,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:41:52,893 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:41:56,933 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:41:57,934 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:42:02,935 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:42:03,541 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:42:03,542 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:42:03,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:42:08,916 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:42:13,917 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:42:18,542 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:42:18,542 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:42:18,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:42:18,978 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:42:23,978 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:42:26,937 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:42:29,939 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:42:33,541 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:42:33,542 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:42:33,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:42:34,966 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:42:36,289 INFO Thread-109:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/output.log +2023-11-01 00:42:38,290 INFO Thread-109:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/output.log +2023-11-01 00:42:40,930 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:42:45,930 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:42:48,542 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:42:48,543 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:42:48,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:42:50,959 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:42:55,960 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:42:56,940 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:43:01,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:43:03,542 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:43:03,542 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:43:03,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:43:07,930 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:43:12,930 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:43:17,931 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:43:18,542 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:43:18,542 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:43:18,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:43:23,900 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:43:26,942 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:43:28,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:43:33,542 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:43:33,542 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:43:33,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:43:34,933 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:43:39,934 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:43:44,934 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:43:48,542 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:43:48,543 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:43:48,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:43:50,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:43:55,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:43:56,944 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:44:00,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:44:03,542 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:44:03,543 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:44:03,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:44:06,912 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:44:11,913 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:44:16,914 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:44:18,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:44:18,543 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:44:18,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:44:21,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:44:26,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:44:26,946 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:44:31,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:44:33,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:44:33,543 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:44:33,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:44:36,965 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:44:41,966 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:44:46,966 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:44:48,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:44:48,543 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:44:48,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:44:52,962 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:44:56,948 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:44:58,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:45:03,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:45:03,544 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:45:03,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:45:03,980 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:45:08,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:45:13,982 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:45:18,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:45:18,544 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:45:18,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:45:19,933 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:45:24,933 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:45:26,950 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:45:29,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:45:33,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:45:33,544 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:45:33,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:45:35,871 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:45:40,872 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:45:45,872 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:45:48,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:45:48,544 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:45:48,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:45:50,897 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:45:55,897 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:45:56,953 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:46:00,955 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:46:03,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:46:03,544 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:46:03,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:46:06,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:46:11,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:46:16,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:46:18,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:46:18,544 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:46:18,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:46:21,965 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:46:26,956 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:46:27,957 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:46:32,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:46:33,543 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:46:33,544 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:46:33,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:46:38,938 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:46:43,939 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:46:48,544 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:46:48,544 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:46:48,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:46:48,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:46:53,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:46:56,957 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:46:58,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:47:03,544 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:47:03,544 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:47:03,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:47:03,968 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:47:08,969 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:47:13,970 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:47:18,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:47:18,545 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:47:18,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:47:19,922 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:47:24,923 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:47:26,959 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:47:29,961 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:47:33,544 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:47:33,544 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:47:33,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:47:35,918 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:47:40,919 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:47:45,920 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:47:48,544 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:47:48,545 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:47:48,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:47:51,901 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:47:56,901 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:47:56,961 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:48:01,963 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:48:03,544 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:48:03,545 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:48:03,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:48:07,929 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:48:12,929 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:48:17,930 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:48:18,544 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:48:18,545 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:48:18,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:48:23,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:48:26,964 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:48:28,965 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:48:33,544 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:48:33,545 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:48:33,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:48:34,917 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:48:39,918 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:48:44,919 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:48:48,544 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:48:48,545 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:48:48,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:48:49,977 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:48:54,977 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:48:56,966 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:49:00,967 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:49:03,544 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:49:03,545 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:49:03,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:49:06,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:49:11,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:49:16,904 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:49:18,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:49:18,545 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:49:18,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:49:21,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:49:26,906 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:49:26,967 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:49:31,969 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:49:33,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:49:33,545 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:49:33,585 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:49:37,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:49:42,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:49:47,912 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:49:48,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:49:48,545 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:49:48,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:49:52,977 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:49:56,970 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:49:58,972 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:50:03,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:50:03,545 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:50:03,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:50:04,878 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:50:09,879 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:50:14,880 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:50:18,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:50:18,545 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:50:18,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:50:19,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:50:24,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:50:26,972 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:50:29,973 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:50:33,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:50:33,546 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:50:33,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:50:35,967 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:50:40,967 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:50:45,968 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:50:48,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:50:48,546 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:50:48,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:50:51,871 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:50:56,872 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:50:56,974 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:51:01,975 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:51:03,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:51:03,546 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:51:03,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:51:07,908 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:51:12,909 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:51:17,910 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:51:18,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:51:18,546 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:51:18,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:51:23,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:51:26,976 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:51:28,977 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:51:33,545 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:51:33,546 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:51:33,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:51:34,968 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:51:39,968 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:51:44,969 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:51:48,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:51:48,546 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:51:48,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:51:50,887 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:51:55,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:51:56,978 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:52:00,980 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:52:03,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:52:03,546 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:52:03,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:52:06,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:52:11,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:52:16,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:52:18,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:52:18,546 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:52:18,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:52:21,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:52:26,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:52:26,981 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:52:31,982 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:52:33,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:52:33,547 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:52:33,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:52:37,926 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:52:42,926 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:52:47,927 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:52:48,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:52:48,548 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:52:48,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:52:53,919 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:52:56,983 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:52:58,984 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:53:03,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:53:03,547 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:53:03,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:53:04,926 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:53:09,927 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:53:14,928 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:53:18,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:53:18,547 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:53:18,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:53:19,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:53:24,982 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:53:26,987 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:53:29,988 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:53:33,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:53:33,547 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:53:33,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:53:35,957 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:53:40,957 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:53:45,958 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:53:48,546 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:53:48,547 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:53:48,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:53:50,964 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:53:55,965 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:53:56,988 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:54:00,990 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:54:03,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:54:03,547 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:54:03,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:54:06,936 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:54:11,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:54:16,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:54:18,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:54:18,547 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:54:18,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:54:22,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:54:26,990 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:54:27,992 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:54:32,992 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:54:33,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:54:33,547 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:54:33,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:54:38,109 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:54:43,110 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:54:48,110 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:54:48,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:54:48,547 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:54:48,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:54:53,932 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:54:56,993 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:54:58,994 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:55:03,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:55:03,547 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:55:03,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:55:04,896 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:55:09,897 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:55:14,898 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:55:18,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:55:18,548 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:55:18,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:55:19,974 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:55:24,974 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:55:26,995 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:55:29,997 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:55:33,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:55:33,548 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:55:33,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:55:35,937 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:55:40,938 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:55:45,938 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:55:48,547 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:55:48,548 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:55:48,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:55:50,962 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:55:55,963 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:55:56,997 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:56:00,999 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:56:03,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:56:03,548 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:56:03,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:56:06,929 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:56:11,930 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:56:16,930 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:56:18,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:56:18,548 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:56:18,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:56:21,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:56:26,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:56:27,000 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:56:32,001 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:56:33,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:56:33,548 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:56:33,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:56:37,875 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:56:42,876 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:56:47,877 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:56:48,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:56:48,548 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:56:48,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:56:52,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:56:57,001 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:56:58,002 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:57:03,003 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:57:03,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:57:03,548 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:57:03,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:57:08,888 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:57:13,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:57:18,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:57:18,548 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:57:18,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:57:18,975 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:57:23,976 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:57:27,004 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:57:29,005 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:57:33,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:57:33,549 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:57:33,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:57:34,924 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:57:39,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:57:44,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:57:48,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:57:48,549 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:57:48,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:57:49,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:57:54,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:57:57,006 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:58:00,008 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:58:03,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:58:03,549 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:58:03,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:58:05,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:58:10,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:58:15,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:58:18,548 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:58:18,549 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:58:18,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:58:21,924 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:58:26,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:58:27,008 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:58:32,009 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:58:33,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:58:33,549 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:58:33,589 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:58:37,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:58:43,636 INFO Thread-109:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/output.log +2023-11-01 00:58:43,881 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:58:48,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:58:48,549 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:58:48,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:58:48,934 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:58:53,934 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:58:57,010 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:58:59,011 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:59:03,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:59:03,549 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:59:03,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:59:04,978 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:59:09,978 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:59:14,979 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:59:18,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:59:18,549 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:59:18,605 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:59:20,898 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:59:25,899 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:59:27,012 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 00:59:31,014 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:59:33,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:59:33,550 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:59:33,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:59:36,968 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:59:41,968 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:59:46,969 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:59:48,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 00:59:48,550 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 00:59:48,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 00:59:51,971 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:59:56,972 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 00:59:57,015 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:00:02,016 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:00:03,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:00:03,550 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:00:03,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:00:07,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:00:12,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:00:17,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:00:18,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:00:18,550 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:00:18,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:00:23,889 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:00:27,016 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:00:29,018 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:00:33,549 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:00:33,550 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:00:33,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:00:34,896 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:00:39,896 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:00:44,897 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:00:48,550 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:00:48,550 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:00:48,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:00:49,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:00:54,912 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:00:57,019 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:01:00,021 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:01:03,550 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:01:03,550 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:01:03,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:01:05,932 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:01:10,933 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:01:15,933 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:01:18,550 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:01:18,550 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:01:18,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:01:21,133 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:01:26,133 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:01:27,020 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:01:32,022 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:01:33,550 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:01:33,551 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:01:33,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:01:37,975 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:01:42,976 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:01:47,977 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:01:48,551 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:01:48,551 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:01:48,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:01:53,927 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:01:57,023 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:01:59,025 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:02:03,550 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:02:03,551 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:02:03,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:02:04,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:02:09,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:02:14,982 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:02:18,550 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:02:18,551 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:02:18,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:02:20,928 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:02:25,928 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:02:27,025 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:02:31,027 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:02:33,550 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:02:33,551 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:02:33,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:02:37,015 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:02:42,015 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:02:47,016 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:02:48,550 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:02:48,551 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:02:48,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:02:52,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:02:57,027 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:02:58,029 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:03:03,029 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:03:03,551 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:03:03,551 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:03:03,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:03:08,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:03:13,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:03:18,551 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:03:18,551 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:03:18,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:03:18,972 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:03:23,973 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:03:27,031 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:03:29,032 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:03:33,551 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:03:33,551 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:03:33,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:03:34,933 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:03:39,934 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:03:44,935 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:03:48,551 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:03:48,551 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:03:48,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:03:49,976 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:03:54,977 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:03:57,033 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:04:00,034 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:04:03,551 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:04:03,552 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:04:03,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:04:05,940 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:04:10,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:04:15,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:04:18,551 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:04:18,552 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:04:18,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:04:20,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:04:25,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:04:27,035 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:04:31,036 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:04:33,551 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:04:33,552 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:04:33,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:04:36,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:04:41,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:04:46,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:04:48,552 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:04:48,552 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:04:48,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:04:52,908 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:04:57,037 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:04:58,038 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:05:03,039 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:05:03,552 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:05:03,552 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:05:03,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:05:08,902 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:05:13,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:05:18,552 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:05:18,552 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:05:18,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:05:18,965 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:05:23,965 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:05:27,039 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:05:29,041 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:05:33,552 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:05:33,552 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:05:33,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:05:34,988 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:05:39,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:05:44,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:05:48,552 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:05:48,552 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:05:48,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:05:50,974 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:05:55,974 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:05:57,041 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:06:01,042 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:06:03,552 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:06:03,552 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:06:03,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:06:06,934 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:06:11,935 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:06:16,935 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:06:18,552 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:06:18,553 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:06:18,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:06:21,979 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:06:26,980 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:06:27,042 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:06:32,044 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:06:33,552 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:06:33,553 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:06:33,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:06:37,973 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:06:42,974 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:06:47,974 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:06:48,552 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:06:48,553 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:06:48,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:06:52,983 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:06:57,045 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:06:58,046 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:07:03,047 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:07:03,552 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:07:03,553 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:07:03,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:07:08,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:07:13,926 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:07:18,553 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:07:18,553 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:07:18,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:07:18,980 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:07:23,980 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:07:27,047 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:07:29,048 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:07:33,553 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:07:33,553 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:07:33,593 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:07:34,903 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:07:39,904 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:07:44,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:07:48,553 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:07:48,553 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:07:48,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:07:49,924 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:07:54,924 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:07:57,051 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:08:00,053 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:08:03,553 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:08:03,553 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:08:03,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:08:06,036 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:08:11,042 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:08:16,042 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:08:18,553 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:08:18,554 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:08:18,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:08:21,935 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:08:26,935 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:08:27,052 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:08:32,054 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:08:33,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:08:33,554 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:08:33,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:08:37,959 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:08:42,959 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:08:47,960 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:08:48,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:08:48,555 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:08:48,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:08:52,969 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:08:57,054 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:08:58,056 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:09:03,056 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:09:03,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:09:03,554 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:09:03,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:09:08,875 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:09:13,876 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:09:18,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:09:18,555 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:09:18,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:09:18,924 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:09:23,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:09:27,055 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:09:29,057 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:09:33,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:09:33,554 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:09:33,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:09:34,959 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:09:39,960 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:09:44,960 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:09:48,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:09:48,554 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:09:48,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:09:50,922 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:09:55,922 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:09:57,058 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:10:01,060 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:10:03,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:10:03,555 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:10:03,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:10:06,982 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:10:11,982 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:10:16,983 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:10:18,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:10:18,555 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:10:18,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:10:22,917 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:10:27,060 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:10:28,062 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:10:33,062 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:10:33,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:10:33,555 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:10:33,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:10:38,979 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:10:43,979 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:10:48,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:10:48,556 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:10:48,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:10:49,924 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:10:54,925 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:10:57,063 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:11:00,064 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:11:03,554 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:11:03,555 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:11:03,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:11:05,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:11:10,990 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:11:15,991 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:11:18,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:11:18,555 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:11:18,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:11:21,984 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:11:26,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:11:27,066 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:11:32,068 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:11:33,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:11:33,555 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:11:33,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:11:37,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:11:42,990 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:11:47,991 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:11:48,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:11:48,555 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:11:48,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:11:53,921 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:11:57,068 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:11:59,069 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:12:03,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:12:03,555 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:12:03,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:12:04,983 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:12:09,984 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:12:14,984 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:12:18,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:12:18,556 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:12:18,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:12:20,919 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:12:25,920 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:12:27,071 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:12:31,073 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:12:33,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:12:33,556 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:12:33,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:12:36,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:12:41,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:12:46,994 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:12:48,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:12:48,556 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:12:48,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:12:52,984 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:12:57,073 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:12:58,074 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:13:03,075 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:13:03,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:13:03,556 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:13:03,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:13:08,933 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:13:13,934 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:13:18,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:13:18,556 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:13:18,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:13:19,906 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:13:24,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:13:27,075 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:13:30,077 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:13:33,555 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:13:33,556 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:13:33,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:13:35,917 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:13:40,918 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:13:45,919 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:13:48,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:13:48,556 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:13:48,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:13:50,992 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:13:55,992 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:13:57,077 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:14:01,078 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:14:03,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:14:03,556 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:14:03,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:14:06,914 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:14:11,914 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:14:16,915 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:14:18,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:14:18,557 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:14:18,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:14:21,999 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:14:26,999 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:14:27,079 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:14:32,081 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:14:33,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:14:33,557 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:14:33,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:14:37,926 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:14:42,927 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:14:47,928 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:14:48,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:14:48,557 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:14:48,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:14:53,911 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:14:57,081 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:14:59,082 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:15:03,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:15:03,557 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:15:03,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:15:04,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:15:09,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:15:14,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:15:18,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:15:18,557 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:15:18,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:15:20,898 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:15:25,899 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:15:27,084 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:15:31,085 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:15:33,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:15:33,557 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:15:33,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:15:36,882 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:15:41,883 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:15:46,883 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:15:48,556 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:15:48,557 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:15:48,597 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:15:51,994 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:15:56,995 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:15:57,085 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:16:02,086 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:16:03,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:16:03,557 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:16:03,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:16:07,894 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:16:12,895 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:16:17,895 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:16:18,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:16:18,558 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:16:18,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:16:22,905 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:16:27,088 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:16:28,089 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:16:33,090 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:16:33,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:16:33,558 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:16:33,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:16:38,933 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:16:43,934 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:16:48,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:16:48,559 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:16:48,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:16:49,928 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:16:54,929 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:16:57,089 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:17:00,091 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:17:03,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:17:03,558 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:17:03,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:17:05,960 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:17:10,961 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:17:15,962 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:17:18,557 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:17:18,558 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:17:18,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:17:20,979 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:17:25,979 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:17:27,091 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:17:31,093 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:17:33,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:17:33,558 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:17:33,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:17:36,972 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:17:41,973 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:17:46,974 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:17:48,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:17:48,558 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:17:48,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:17:52,924 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:17:57,094 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:17:58,095 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:18:03,096 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:18:03,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:18:03,558 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:18:03,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:18:08,971 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:18:13,972 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:18:18,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:18:18,558 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:18:18,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:18:18,992 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:18:23,992 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:18:27,097 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:18:29,098 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:18:33,558 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:18:33,559 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:18:33,601 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:18:34,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:18:39,672 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history +2023-11-01 01:18:39,674 DEBUG SenderThread:34227 [sender.py:send():380] send: history +2023-11-01 01:18:39,674 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: summary_record +2023-11-01 01:18:39,674 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 01:18:39,713 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-11-01 01:18:39,713 DEBUG SenderThread:34227 [sender.py:send():380] send: exit +2023-11-01 01:18:39,714 INFO SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0 +2023-11-01 01:18:39,714 INFO SenderThread:34227 [sender.py:send_exit():587] handling runtime: 2412 +2023-11-01 01:18:39,714 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 01:18:39,714 INFO SenderThread:34227 [sender.py:send_exit():593] send defer +2023-11-01 01:18:39,715 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:39,715 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0 +2023-11-01 01:18:39,715 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:39,715 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-11-01 01:18:39,715 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 1 +2023-11-01 01:18:39,716 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:39,716 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1 +2023-11-01 01:18:39,716 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:39,716 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-11-01 01:18:39,716 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 2 +2023-11-01 01:18:39,716 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:39,716 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2 +2023-11-01 01:18:39,716 INFO HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor +2023-11-01 01:18:39,717 DEBUG SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-11-01 01:18:39,717 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor +2023-11-01 01:18:39,717 DEBUG SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-11-01 01:18:39,718 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor +2023-11-01 01:18:40,058 INFO Thread-109:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/wandb-summary.json +2023-11-01 01:18:41,420 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor +2023-11-01 01:18:41,421 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor +2023-11-01 01:18:41,421 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor +2023-11-01 01:18:41,422 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:18:41,422 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:18:41,422 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:41,423 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-11-01 01:18:41,423 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 3 +2023-11-01 01:18:41,423 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:18:41,424 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:18:41,424 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:41,424 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3 +2023-11-01 01:18:41,425 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:41,425 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-11-01 01:18:41,425 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 4 +2023-11-01 01:18:41,425 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:41,425 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4 +2023-11-01 01:18:41,426 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:41,426 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-11-01 01:18:41,426 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 5 +2023-11-01 01:18:41,426 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:41,426 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5 +2023-11-01 01:18:41,427 DEBUG SenderThread:34227 [sender.py:send():380] send: summary +2023-11-01 01:18:41,427 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 01:18:41,427 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:41,428 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-11-01 01:18:41,428 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 6 +2023-11-01 01:18:41,428 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:41,428 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6 +2023-11-01 01:18:41,428 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:41,428 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-11-01 01:18:41,434 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:18:41,674 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:18:41,906 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 7 +2023-11-01 01:18:41,906 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:18:41,907 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:41,907 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7 +2023-11-01 01:18:41,908 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:41,908 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-11-01 01:18:42,059 INFO Thread-109:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/config.yaml +2023-11-01 01:18:42,059 INFO Thread-109:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/wandb-summary.json +2023-11-01 01:18:42,271 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 8 +2023-11-01 01:18:42,271 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:42,271 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8 +2023-11-01 01:18:42,272 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:42,272 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-11-01 01:18:42,272 INFO SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact +2023-11-01 01:18:42,273 INFO SenderThread:34227 [job_builder.py:_get_source_type():404] no source found +2023-11-01 01:18:42,273 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 9 +2023-11-01 01:18:42,273 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:42,273 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9 +2023-11-01 01:18:42,274 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:42,274 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-11-01 01:18:42,274 INFO SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher +2023-11-01 01:18:42,674 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:18:43,059 INFO Thread-109:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/output.log +2023-11-01 01:18:43,060 INFO SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files +2023-11-01 01:18:43,061 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/output.log output.log +2023-11-01 01:18:43,061 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/requirements.txt requirements.txt +2023-11-01 01:18:43,067 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/conda-environment.yaml conda-environment.yaml +2023-11-01 01:18:43,069 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/wandb-summary.json wandb-summary.json +2023-11-01 01:18:43,076 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/config.yaml config.yaml +2023-11-01 01:18:43,084 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/wandb-metadata.json wandb-metadata.json +2023-11-01 01:18:43,090 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 10 +2023-11-01 01:18:43,092 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:18:43,092 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:43,099 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10 +2023-11-01 01:18:43,100 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:43,100 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-11-01 01:18:43,100 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-11-01 01:18:43,677 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:18:43,680 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:18:44,058 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/output.log +2023-11-01 01:18:44,323 INFO wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/config.yaml +2023-11-01 01:18:44,328 INFO wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/requirements.txt +2023-11-01 01:18:44,337 INFO wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/conda-environment.yaml +2023-11-01 01:18:44,676 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:18:44,676 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:18:45,432 INFO wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/files/wandb-summary.json +2023-11-01 01:18:45,632 INFO Thread-108:34227 [sender.py:transition_state():613] send defer: 11 +2023-11-01 01:18:45,633 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:45,633 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11 +2023-11-01 01:18:45,634 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:45,634 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-11-01 01:18:45,634 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher +2023-11-01 01:18:45,634 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 12 +2023-11-01 01:18:45,634 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:45,634 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12 +2023-11-01 01:18:45,635 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:45,635 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-11-01 01:18:45,635 INFO SenderThread:34227 [file_stream.py:finish():595] file stream finish called +2023-11-01 01:18:45,676 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:18:46,745 INFO SenderThread:34227 [file_stream.py:finish():599] file stream finish is done +2023-11-01 01:18:46,745 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 13 +2023-11-01 01:18:46,745 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:18:46,745 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:46,746 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13 +2023-11-01 01:18:46,746 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:18:46,746 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:46,747 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-11-01 01:18:46,747 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 14 +2023-11-01 01:18:46,747 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:18:46,747 DEBUG SenderThread:34227 [sender.py:send():380] send: final +2023-11-01 01:18:46,748 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14 +2023-11-01 01:18:46,748 DEBUG SenderThread:34227 [sender.py:send():380] send: footer +2023-11-01 01:18:46,748 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:18:46,748 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-11-01 01:18:46,749 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:18:46,750 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:18:46,751 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:18:46,752 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info +2023-11-01 01:18:46,752 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary +2023-11-01 01:18:46,752 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history +2023-11-01 01:18:46,752 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: server_info +2023-11-01 01:18:46,753 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info +2023-11-01 01:18:47,084 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: job_info +2023-11-01 01:18:47,085 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown +2023-11-01 01:18:47,085 INFO HandlerThread:34227 [handler.py:finish():866] shutting down handler +2023-11-01 01:18:47,759 INFO WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/run-t9dkdynt.wandb +2023-11-01 01:18:48,084 INFO SenderThread:34227 [sender.py:finish():1534] shutting down sender +2023-11-01 01:18:48,084 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-11-01 01:18:48,085 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231101_003823-t9dkdynt/logs/debug.log b/wandb/run-20231101_003823-t9dkdynt/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3bdbcdaaa9ae5a07590729ad2f9b502dd11c7c46 --- /dev/null +++ b/wandb/run-20231101_003823-t9dkdynt/logs/debug.log @@ -0,0 +1,34 @@ +2023-11-01 00:38:23,881 INFO MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-11-01 00:38:23,881 INFO MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070 +2023-11-01 00:38:23,881 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-11-01 00:38:23,881 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-11-01 00:38:23,881 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-11-01 00:38:23,881 INFO MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-11-01 00:38:23,881 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-11-01 00:38:23,881 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-11-01 00:38:23,881 INFO MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/logs/debug.log +2023-11-01 00:38:23,881 INFO MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt/logs/debug-internal.log +2023-11-01 00:38:23,882 INFO MainThread:34070 [wandb_init.py:init():568] calling init triggers +2023-11-01 00:38:23,882 INFO MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'SGD', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 35} +2023-11-01 00:38:23,882 INFO MainThread:34070 [wandb_init.py:init():618] starting backend +2023-11-01 00:38:23,882 INFO MainThread:34070 [wandb_init.py:init():622] setting up manager +2023-11-01 00:38:23,883 INFO MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-11-01 00:38:23,884 INFO MainThread:34070 [wandb_init.py:init():628] backend started and connected +2023-11-01 00:38:23,886 INFO MainThread:34070 [wandb_init.py:init():720] updated telemetry +2023-11-01 00:38:23,887 INFO MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-11-01 00:38:26,227 INFO MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version +2023-11-01 00:38:26,897 INFO MainThread:34070 [wandb_run.py:_on_init():2229] got version response +2023-11-01 00:38:26,897 INFO MainThread:34070 [wandb_init.py:init():804] starting run threads in backend +2023-11-01 00:38:33,542 INFO MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg +2023-11-01 00:38:33,542 INFO MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-11-01 00:38:33,542 INFO MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-11-01 00:38:33,542 INFO MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed. +2023-11-01 00:38:33,544 INFO MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process +2023-11-01 01:18:39,672 INFO MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/t9dkdynt +2023-11-01 01:18:39,672 INFO MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0 +2023-11-01 01:18:39,672 INFO MainThread:34070 [wandb_run.py:_restore():2151] restore +2023-11-01 01:18:39,672 INFO MainThread:34070 [wandb_run.py:_restore():2157] restore done +2023-11-01 01:18:48,087 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-11-01 01:18:48,088 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-11-01 01:18:48,095 INFO MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files diff --git a/wandb/run-20231101_003823-t9dkdynt/run-t9dkdynt.wandb b/wandb/run-20231101_003823-t9dkdynt/run-t9dkdynt.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ea6e2200e6a9c41baec5fca92e85395548c2b36d Binary files /dev/null and b/wandb/run-20231101_003823-t9dkdynt/run-t9dkdynt.wandb differ diff --git a/wandb/run-20231101_011849-ota77xrr/files/conda-environment.yaml b/wandb/run-20231101_011849-ota77xrr/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231101_011849-ota77xrr/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231101_011849-ota77xrr/files/config.yaml b/wandb/run-20231101_011849-ota77xrr/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..773443fb76139cbd17db3a47587a4082041239ce --- /dev/null +++ b/wandb/run-20231101_011849-ota77xrr/files/config.yaml @@ -0,0 +1,53 @@ +wandb_version: 1 + +optimizer: + desc: null + value: Adam +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 33 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698787129.127706 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 2 + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231101_011849-ota77xrr/files/output.log b/wandb/run-20231101_011849-ota77xrr/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3e3243d813ebee0d393d6b3779843d64e779d465 --- /dev/null +++ b/wandb/run-20231101_011849-ota77xrr/files/output.log @@ -0,0 +1,3 @@ + +Saving Model At: save_model/best_model.pth +Saving Model At: save_model/best_model.pth \ No newline at end of file diff --git a/wandb/run-20231101_011849-ota77xrr/files/requirements.txt b/wandb/run-20231101_011849-ota77xrr/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231101_011849-ota77xrr/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231101_011849-ota77xrr/files/wandb-metadata.json b/wandb/run-20231101_011849-ota77xrr/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8be2d5337fd5271bdc08c8a7c1dac81b1a92527d --- /dev/null +++ b/wandb/run-20231101_011849-ota77xrr/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T21:18:50.441713", + "startedAt": "2023-10-31T21:18:49.123304", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.5325250000000001, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.398, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.415, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.448, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.37, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.499, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.192, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.398, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.183, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.203, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.398, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.514, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.51, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.476, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.499, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.181, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.401, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.206, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.246, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.204, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.192, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 3.043, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.9305229187012 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231101_011849-ota77xrr/files/wandb-summary.json b/wandb/run-20231101_011849-ota77xrr/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..94cb4197664fd9398e1e30b1e5981029bcf5c5f4 --- /dev/null +++ b/wandb/run-20231101_011849-ota77xrr/files/wandb-summary.json @@ -0,0 +1 @@ +{"Train Loss": 1.3497313261032104, "Train Accuracy": 0.5220903158187866, "Validation Loss": 1.2040987014770508, "Validation Accuracy": 0.5616196393966675, "_timestamp": 1698788809.3675807, "_runtime": 1680.2398746013641, "_step": 0, "_wandb": {"runtime": 1679}} \ No newline at end of file diff --git a/wandb/run-20231101_011849-ota77xrr/logs/debug-internal.log b/wandb/run-20231101_011849-ota77xrr/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d0460b6489843f198962383e80ca40a2bef89835 --- /dev/null +++ b/wandb/run-20231101_011849-ota77xrr/logs/debug-internal.log @@ -0,0 +1,902 @@ +2023-11-01 01:18:49,130 INFO StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-11-01 01:18:49.129445 +2023-11-01 01:18:49,132 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status +2023-11-01 01:18:49,134 INFO WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/run-ota77xrr.wandb +2023-11-01 01:18:49,137 DEBUG SenderThread:34227 [sender.py:send():380] send: header +2023-11-01 01:18:49,137 DEBUG SenderThread:34227 [sender.py:send():380] send: run +2023-11-01 01:18:49,957 INFO SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files +2023-11-01 01:18:49,957 INFO SenderThread:34227 [sender.py:_start_run_threads():1122] run started: ota77xrr with start time 1698787129.127706 +2023-11-01 01:18:49,978 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version +2023-11-01 01:18:49,978 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: check_version +2023-11-01 01:18:50,376 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start +2023-11-01 01:18:50,377 DEBUG HandlerThread:34227 [system_info.py:__init__():32] System info init +2023-11-01 01:18:50,377 DEBUG HandlerThread:34227 [system_info.py:__init__():47] System info init done +2023-11-01 01:18:50,377 INFO HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor +2023-11-01 01:18:50,378 INFO SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-11-01 01:18:50,378 INFO HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info +2023-11-01 01:18:50,379 INFO SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring +2023-11-01 01:18:50,379 INFO SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring +2023-11-01 01:18:50,380 INFO SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring +2023-11-01 01:18:50,382 INFO SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring +2023-11-01 01:18:50,382 INFO SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring +2023-11-01 01:18:50,441 DEBUG HandlerThread:34227 [system_info.py:probe():196] Probing system +2023-11-01 01:18:50,442 DEBUG HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid +2023-11-01 01:18:50,442 DEBUG HandlerThread:34227 [system_info.py:probe():244] Probing system done +2023-11-01 01:18:50,442 DEBUG HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T21:18:50.441713', 'startedAt': '2023-10-31T21:18:49.123304', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.5325250000000001, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.398, 'min': 1200.0, 'max': 3600.0}, {'current': 1.415, 'min': 1200.0, 'max': 3600.0}, {'current': 1.448, 'min': 1200.0, 'max': 3600.0}, {'current': 1.37, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.499, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.192, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.398, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.183, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.203, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.398, 'min': 1200.0, 'max': 3600.0}, {'current': 1.514, 'min': 1200.0, 'max': 3600.0}, {'current': 1.51, 'min': 1200.0, 'max': 3600.0}, {'current': 1.476, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.499, 'min': 1200.0, 'max': 3600.0}, {'current': 2.181, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.401, 'min': 1200.0, 'max': 3600.0}, {'current': 1.206, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.246, 'min': 1200.0, 'max': 3600.0}, {'current': 1.204, 'min': 1200.0, 'max': 3600.0}, {'current': 1.192, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 3.043, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9305229187012}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-11-01 01:18:50,442 INFO HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info +2023-11-01 01:18:50,442 INFO HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info +2023-11-01 01:18:50,442 DEBUG HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-11-01 01:18:50,443 DEBUG HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done +2023-11-01 01:18:50,443 DEBUG HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-11-01 01:18:50,962 INFO Thread-121:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/conda-environment.yaml +2023-11-01 01:18:50,963 INFO Thread-121:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/requirements.txt +2023-11-01 01:18:56,925 DEBUG HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done +2023-11-01 01:18:56,927 INFO HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info +2023-11-01 01:18:56,928 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:18:56,928 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive +2023-11-01 01:18:56,929 DEBUG SenderThread:34227 [sender.py:send():380] send: files +2023-11-01 01:18:56,929 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-11-01 01:18:56,939 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:18:56,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:18:56,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:18:56,960 INFO Thread-121:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/conda-environment.yaml +2023-11-01 01:18:56,961 INFO Thread-121:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/wandb-metadata.json +2023-11-01 01:18:57,455 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-11-01 01:18:58,140 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmpu5bper0fwandb/8s07c4zf-wandb-metadata.json +2023-11-01 01:18:59,456 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:19:04,456 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:19:09,457 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:19:11,938 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:19:11,939 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:19:11,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:19:15,280 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:19:20,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:19:20,968 INFO Thread-121:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/config.yaml +2023-11-01 01:19:25,643 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:19:26,938 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:19:26,939 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:19:26,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:19:31,289 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:19:36,289 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:19:41,290 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:19:41,939 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:19:41,939 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:19:41,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:19:46,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:19:50,382 DEBUG SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-11-01 01:19:50,385 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:19:51,386 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:19:56,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:19:56,938 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:19:56,939 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:19:56,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:20:02,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:20:07,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:20:11,938 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:20:11,939 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:20:11,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:20:13,294 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:20:18,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:20:20,387 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:20:23,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:20:26,938 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:20:26,939 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:20:26,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:20:29,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:20:34,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:20:39,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:20:41,939 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:20:41,939 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:20:41,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:20:45,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:20:50,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:20:50,389 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:20:55,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:20:56,939 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:20:56,940 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:20:56,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:21:01,278 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:21:06,279 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:21:11,280 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:21:11,939 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:21:11,940 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:21:11,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:21:16,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:21:20,391 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:21:21,392 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:21:26,393 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:21:26,939 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:21:26,940 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:21:26,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:21:32,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:21:37,358 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:21:41,939 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:21:41,940 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:21:41,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:21:43,329 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:21:48,330 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:21:50,393 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:21:53,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:21:56,940 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:21:56,940 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:21:56,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:21:59,344 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:22:04,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:22:09,346 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:22:11,939 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:22:11,940 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:22:11,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:22:14,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:22:19,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:22:20,395 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:22:24,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:22:26,939 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:22:26,940 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:22:26,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:22:30,276 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:22:35,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:22:40,278 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:22:41,940 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:22:41,940 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:22:41,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:22:45,289 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:22:50,290 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:22:50,398 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:22:55,399 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:22:56,940 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:22:56,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:22:56,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:23:00,039 INFO Thread-121:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/output.log +2023-11-01 01:23:01,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:23:02,039 INFO Thread-121:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/output.log +2023-11-01 01:23:06,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:23:11,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:23:11,940 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:23:11,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:23:11,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:23:16,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:23:20,400 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:23:21,402 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:23:26,402 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:23:26,940 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:23:26,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:23:26,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:23:32,356 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:23:37,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:23:41,940 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:23:41,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:23:41,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:23:43,283 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:23:48,284 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:23:50,402 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:23:53,404 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:23:56,940 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:23:56,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:23:56,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:23:59,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:24:04,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:24:09,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:24:11,940 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:24:11,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:24:11,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:24:14,355 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:24:19,356 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:24:20,404 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:24:24,406 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:24:26,940 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:24:26,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:24:26,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:24:30,334 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:24:35,334 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:24:40,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:24:41,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:24:41,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:24:41,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:24:46,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:24:50,406 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:24:51,407 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:24:56,408 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:24:56,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:24:56,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:24:56,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:25:02,270 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:25:07,271 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:25:11,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:25:11,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:25:11,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:25:12,354 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:25:17,354 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:25:20,407 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:25:22,408 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:25:26,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:25:26,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:25:26,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:25:28,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:25:33,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:25:38,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:25:41,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:25:41,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:25:41,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:25:43,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:25:48,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:25:50,410 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:25:53,412 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:25:56,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:25:56,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:25:56,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:25:59,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:26:04,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:26:09,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:26:11,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:26:11,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:26:11,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:26:15,284 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:26:20,284 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:26:20,413 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:26:25,415 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:26:26,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:26:26,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:26:26,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:26:31,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:26:36,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:26:41,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:26:41,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:26:41,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:26:41,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:26:47,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:26:50,416 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:26:52,417 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:26:56,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:26:56,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:26:56,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:26:58,147 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:27:00,121 INFO Thread-121:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/output.log +2023-11-01 01:27:03,190 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:27:08,190 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:27:11,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:27:11,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:27:11,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:27:13,336 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:27:18,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:27:20,418 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:27:23,419 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:27:26,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:27:26,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:27:26,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:27:29,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:27:34,386 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:27:39,386 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:27:41,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:27:41,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:27:41,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:27:44,394 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:27:49,395 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:27:50,420 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:27:54,422 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:27:56,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:27:56,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:27:56,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:28:00,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:28:05,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:28:10,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:28:11,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:28:11,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:28:11,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:28:15,343 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:28:20,344 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:28:20,423 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:28:25,425 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:28:26,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:28:26,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:28:26,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:28:31,345 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:28:36,346 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:28:41,346 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:28:41,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:28:41,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:28:41,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:28:47,343 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:28:50,425 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:28:52,426 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:28:56,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:28:56,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:28:56,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:28:58,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:29:03,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:29:08,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:29:11,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:29:11,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:29:11,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:29:14,293 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:29:19,294 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:29:20,427 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:29:24,429 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:29:26,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:29:26,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:29:26,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:29:30,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:29:35,297 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:29:40,298 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:29:41,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:29:41,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:29:41,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:29:45,355 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:29:50,356 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:29:50,429 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:29:55,431 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:29:56,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:29:56,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:29:56,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:30:01,279 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:30:06,280 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:30:11,280 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:30:11,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:30:11,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:30:11,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:30:16,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:30:20,431 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:30:21,432 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:30:26,433 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:30:26,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:30:26,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:30:26,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:30:32,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:30:37,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:30:41,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:30:41,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:30:41,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:30:42,342 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:30:47,343 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:30:50,434 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:30:52,435 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:30:56,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:30:56,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:30:56,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:30:58,351 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:31:03,351 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:31:08,352 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:31:11,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:31:11,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:31:11,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:31:14,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:31:19,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:31:20,435 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:31:24,436 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:31:26,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:31:26,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:31:26,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:31:30,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:31:35,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:31:40,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:31:41,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:31:41,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:31:41,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:31:46,283 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:31:50,437 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:31:51,438 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:31:56,439 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:31:56,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:31:56,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:31:56,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:32:02,298 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:32:07,299 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:32:11,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:32:11,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:32:11,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:32:12,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:32:17,323 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:32:20,439 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:32:22,441 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:32:26,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:32:26,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:32:26,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:32:28,348 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:32:33,348 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:32:38,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:32:41,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:32:41,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:32:41,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:32:44,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:32:49,278 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:32:50,442 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:32:54,443 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:32:56,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:32:56,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:32:56,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:33:00,275 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:33:05,276 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:33:10,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:33:11,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:33:11,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:33:11,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:33:15,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:33:20,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:33:20,444 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:33:25,446 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:33:26,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:33:26,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:33:26,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:33:31,271 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:33:36,272 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:33:41,272 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:33:41,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:33:41,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:33:41,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:33:46,341 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:33:50,447 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:33:51,448 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:33:56,448 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:33:56,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:33:56,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:33:56,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:34:02,432 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:34:07,432 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:34:11,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:34:11,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:34:11,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:34:13,300 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:34:18,301 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:34:20,448 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:34:23,450 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:34:26,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:34:26,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:34:26,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:34:29,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:34:34,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:34:39,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:34:41,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:34:41,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:34:41,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:34:44,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:34:49,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:34:50,451 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:34:54,453 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:34:56,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:34:56,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:34:56,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:35:00,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:35:05,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:35:10,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:35:11,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:35:11,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:35:11,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:35:15,329 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:35:20,330 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:35:20,453 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:35:25,455 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:35:26,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:35:26,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:35:26,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:35:31,370 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:35:36,370 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:35:41,371 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:35:41,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:35:41,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:35:41,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:35:46,380 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:35:50,455 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:35:51,456 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:35:56,457 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:35:56,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:35:56,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:35:56,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:36:02,356 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:36:07,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:36:11,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:36:11,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:36:11,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:36:13,311 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:36:18,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:36:20,458 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:36:23,460 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:36:26,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:36:26,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:36:26,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:36:29,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:36:34,354 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:36:39,355 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:36:41,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:36:41,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:36:41,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:36:45,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:36:50,350 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:36:50,461 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:36:55,463 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:36:56,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:36:56,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:36:56,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:37:01,343 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:37:06,343 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:37:11,344 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:37:11,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:37:11,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:37:11,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:37:16,347 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:37:20,462 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:37:21,463 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:37:26,464 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:37:26,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:37:26,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:37:26,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:37:32,358 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:37:37,359 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:37:41,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:37:41,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:37:41,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:37:43,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:37:48,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:37:50,465 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:37:53,466 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:37:56,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:37:56,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:37:56,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:37:59,373 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:38:04,374 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:38:09,374 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:38:11,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:38:11,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:38:11,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:38:15,291 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:38:20,292 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:38:20,468 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:38:25,469 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:38:26,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:38:26,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:38:26,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:38:31,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:38:36,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:38:41,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:38:41,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:38:41,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:38:41,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:38:47,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:38:50,470 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:38:52,471 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:38:56,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:38:56,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:38:56,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:38:58,301 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:39:03,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:39:08,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:39:11,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:39:11,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:39:11,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:39:13,371 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:39:18,372 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:39:20,471 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:39:23,473 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:39:26,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:39:26,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:39:26,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:39:29,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:39:34,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:39:39,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:39:41,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:39:41,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:39:41,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:39:45,275 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:39:50,275 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:39:50,473 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:39:55,475 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:39:56,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:39:56,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:39:56,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:40:01,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:40:06,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:40:11,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:40:11,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:40:11,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:40:11,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:40:17,298 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:40:20,475 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:40:22,477 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:40:26,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:40:26,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:40:26,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:40:28,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:40:33,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:40:38,310 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:40:41,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:40:41,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:40:41,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:40:43,334 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:40:48,334 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:40:50,477 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:40:53,479 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:40:56,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:40:56,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:40:56,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:40:59,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:41:04,332 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:41:09,333 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:41:11,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:41:11,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:41:11,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:41:14,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:41:19,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:41:20,480 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:41:24,481 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:41:26,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:41:26,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:41:26,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:41:30,364 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:41:35,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:41:40,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:41:41,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:41:41,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:41:41,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:41:46,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:41:50,481 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:41:51,483 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:41:56,483 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:41:56,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:41:56,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:41:56,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:42:02,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:42:07,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:42:11,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:42:11,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:42:11,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:42:13,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:42:18,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:42:20,484 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:42:23,485 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:42:26,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:42:26,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:42:26,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:42:29,341 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:42:34,342 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:42:39,342 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:42:41,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:42:41,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:42:41,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:42:45,301 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:42:50,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:42:50,486 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:42:55,488 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:42:56,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:42:56,950 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:42:56,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:43:01,371 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:43:06,372 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:43:11,373 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:43:11,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:43:11,950 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:43:11,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:43:17,358 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:43:20,488 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:43:22,489 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:43:26,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:43:26,950 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:43:26,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:43:28,405 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:43:33,406 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:43:38,406 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:43:41,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:43:41,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:43:41,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:43:44,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:43:49,303 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:43:50,491 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:43:54,492 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:43:56,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:43:56,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:43:56,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:44:00,287 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:44:05,288 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:44:10,289 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:44:11,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:44:11,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:44:11,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:44:15,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:44:20,338 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:44:20,493 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:44:25,495 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:44:26,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:44:26,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:44:26,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:44:31,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:44:36,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:44:41,362 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:44:41,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:44:41,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:44:41,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:44:46,368 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:44:50,496 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:44:51,497 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:44:56,497 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:44:56,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:44:56,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:44:56,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:45:02,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:45:07,390 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:45:11,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:45:11,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:45:11,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:45:13,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:45:18,386 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:45:20,498 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:45:23,499 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:45:26,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:45:26,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:45:26,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:45:29,329 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:45:34,330 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:45:39,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:45:41,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:45:41,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:45:41,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:45:45,289 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:45:50,290 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:45:50,501 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:45:55,502 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:45:56,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:45:56,952 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:45:56,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:46:01,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:46:06,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:46:11,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:46:11,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:46:11,952 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:46:11,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:46:16,376 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:46:20,503 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:46:21,505 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:46:26,505 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:46:26,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:46:26,952 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:46:26,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:46:32,362 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:46:37,363 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:46:41,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:46:41,952 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:46:41,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:46:43,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:46:48,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:46:49,368 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history +2023-11-01 01:46:49,370 DEBUG SenderThread:34227 [sender.py:send():380] send: history +2023-11-01 01:46:49,370 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: summary_record +2023-11-01 01:46:49,370 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 01:46:49,409 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-11-01 01:46:49,409 DEBUG SenderThread:34227 [sender.py:send():380] send: exit +2023-11-01 01:46:49,409 INFO SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0 +2023-11-01 01:46:49,409 INFO SenderThread:34227 [sender.py:send_exit():587] handling runtime: 1679 +2023-11-01 01:46:49,410 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 01:46:49,410 INFO SenderThread:34227 [sender.py:send_exit():593] send defer +2023-11-01 01:46:49,410 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:49,410 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0 +2023-11-01 01:46:49,410 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:49,411 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-11-01 01:46:49,411 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 1 +2023-11-01 01:46:49,411 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:49,411 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1 +2023-11-01 01:46:49,411 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:49,411 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-11-01 01:46:49,411 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 2 +2023-11-01 01:46:49,411 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:49,411 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2 +2023-11-01 01:46:49,412 INFO HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor +2023-11-01 01:46:49,412 DEBUG SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-11-01 01:46:49,412 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor +2023-11-01 01:46:49,413 DEBUG SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-11-01 01:46:49,413 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor +2023-11-01 01:46:49,543 INFO Thread-121:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/wandb-summary.json +2023-11-01 01:46:51,192 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor +2023-11-01 01:46:51,192 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor +2023-11-01 01:46:51,193 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor +2023-11-01 01:46:51,193 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:46:51,194 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:51,194 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-11-01 01:46:51,194 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 3 +2023-11-01 01:46:51,194 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:46:51,195 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:51,196 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:46:51,198 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3 +2023-11-01 01:46:51,200 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:51,200 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-11-01 01:46:51,200 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 4 +2023-11-01 01:46:51,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:51,201 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4 +2023-11-01 01:46:51,201 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:51,201 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-11-01 01:46:51,201 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 5 +2023-11-01 01:46:51,201 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:51,202 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5 +2023-11-01 01:46:51,202 DEBUG SenderThread:34227 [sender.py:send():380] send: summary +2023-11-01 01:46:51,203 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 01:46:51,203 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:51,203 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-11-01 01:46:51,203 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 6 +2023-11-01 01:46:51,204 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:51,204 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6 +2023-11-01 01:46:51,204 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:51,204 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-11-01 01:46:51,209 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:46:51,370 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:46:51,544 INFO Thread-121:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/wandb-summary.json +2023-11-01 01:46:51,608 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 7 +2023-11-01 01:46:51,608 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:46:51,608 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:51,608 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7 +2023-11-01 01:46:51,609 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:51,609 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-11-01 01:46:52,370 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:46:52,544 INFO Thread-121:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/config.yaml +2023-11-01 01:46:52,842 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 8 +2023-11-01 01:46:52,842 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:46:52,842 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:52,843 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8 +2023-11-01 01:46:52,843 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:52,843 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-11-01 01:46:52,843 INFO SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact +2023-11-01 01:46:52,844 INFO SenderThread:34227 [job_builder.py:_get_source_type():404] no source found +2023-11-01 01:46:52,844 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 9 +2023-11-01 01:46:52,845 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:52,845 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9 +2023-11-01 01:46:52,845 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:52,845 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-11-01 01:46:52,845 INFO SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher +2023-11-01 01:46:53,371 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:46:53,545 INFO Thread-121:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/output.log +2023-11-01 01:46:53,546 INFO SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files +2023-11-01 01:46:53,546 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/output.log output.log +2023-11-01 01:46:53,546 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/requirements.txt requirements.txt +2023-11-01 01:46:53,552 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/conda-environment.yaml conda-environment.yaml +2023-11-01 01:46:53,554 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/wandb-summary.json wandb-summary.json +2023-11-01 01:46:53,561 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/config.yaml config.yaml +2023-11-01 01:46:53,574 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/wandb-metadata.json wandb-metadata.json +2023-11-01 01:46:53,576 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 10 +2023-11-01 01:46:53,576 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:46:53,582 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:53,584 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10 +2023-11-01 01:46:53,585 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:53,585 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-11-01 01:46:53,585 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-11-01 01:46:54,372 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:46:54,372 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:46:54,558 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/output.log +2023-11-01 01:46:54,755 INFO wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/config.yaml +2023-11-01 01:46:54,803 INFO wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/wandb-summary.json +2023-11-01 01:46:54,858 INFO wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/requirements.txt +2023-11-01 01:46:55,000 INFO wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/files/conda-environment.yaml +2023-11-01 01:46:55,200 INFO Thread-120:34227 [sender.py:transition_state():613] send defer: 11 +2023-11-01 01:46:55,200 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:55,201 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11 +2023-11-01 01:46:55,201 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:55,201 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-11-01 01:46:55,202 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher +2023-11-01 01:46:55,202 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 12 +2023-11-01 01:46:55,202 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:55,202 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12 +2023-11-01 01:46:55,202 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:55,203 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-11-01 01:46:55,203 INFO SenderThread:34227 [file_stream.py:finish():595] file stream finish called +2023-11-01 01:46:55,372 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:46:56,239 INFO SenderThread:34227 [file_stream.py:finish():599] file stream finish is done +2023-11-01 01:46:56,239 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 13 +2023-11-01 01:46:56,240 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:46:56,240 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:56,240 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13 +2023-11-01 01:46:56,241 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:56,241 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-11-01 01:46:56,241 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 14 +2023-11-01 01:46:56,242 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 01:46:56,242 DEBUG SenderThread:34227 [sender.py:send():380] send: final +2023-11-01 01:46:56,242 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14 +2023-11-01 01:46:56,242 DEBUG SenderThread:34227 [sender.py:send():380] send: footer +2023-11-01 01:46:56,243 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 01:46:56,243 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-11-01 01:46:56,244 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:46:56,245 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 01:46:56,245 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 01:46:56,246 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info +2023-11-01 01:46:56,246 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary +2023-11-01 01:46:56,247 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history +2023-11-01 01:46:56,247 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: server_info +2023-11-01 01:46:56,248 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info +2023-11-01 01:46:56,582 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: job_info +2023-11-01 01:46:56,583 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown +2023-11-01 01:46:56,584 INFO HandlerThread:34227 [handler.py:finish():866] shutting down handler +2023-11-01 01:46:57,254 INFO WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/run-ota77xrr.wandb +2023-11-01 01:46:57,583 INFO SenderThread:34227 [sender.py:finish():1534] shutting down sender +2023-11-01 01:46:57,583 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-11-01 01:46:57,583 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231101_011849-ota77xrr/logs/debug.log b/wandb/run-20231101_011849-ota77xrr/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b74d24494d3a1565dc5a9e85da786b7b480cb6d0 --- /dev/null +++ b/wandb/run-20231101_011849-ota77xrr/logs/debug.log @@ -0,0 +1,34 @@ +2023-11-01 01:18:49,124 INFO MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-11-01 01:18:49,124 INFO MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070 +2023-11-01 01:18:49,124 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-11-01 01:18:49,124 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-11-01 01:18:49,125 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-11-01 01:18:49,125 INFO MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-11-01 01:18:49,125 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-11-01 01:18:49,125 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-11-01 01:18:49,125 INFO MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/logs/debug.log +2023-11-01 01:18:49,125 INFO MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr/logs/debug-internal.log +2023-11-01 01:18:49,125 INFO MainThread:34070 [wandb_init.py:init():568] calling init triggers +2023-11-01 01:18:49,125 INFO MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'Adam', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 33} +2023-11-01 01:18:49,125 INFO MainThread:34070 [wandb_init.py:init():618] starting backend +2023-11-01 01:18:49,125 INFO MainThread:34070 [wandb_init.py:init():622] setting up manager +2023-11-01 01:18:49,127 INFO MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-11-01 01:18:49,127 INFO MainThread:34070 [wandb_init.py:init():628] backend started and connected +2023-11-01 01:18:49,132 INFO MainThread:34070 [wandb_init.py:init():720] updated telemetry +2023-11-01 01:18:49,133 INFO MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-11-01 01:18:49,977 INFO MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version +2023-11-01 01:18:50,363 INFO MainThread:34070 [wandb_run.py:_on_init():2229] got version response +2023-11-01 01:18:50,364 INFO MainThread:34070 [wandb_init.py:init():804] starting run threads in backend +2023-11-01 01:18:56,939 INFO MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg +2023-11-01 01:18:56,939 INFO MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-11-01 01:18:56,939 INFO MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-11-01 01:18:56,939 INFO MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed. +2023-11-01 01:18:56,941 INFO MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process +2023-11-01 01:46:49,368 INFO MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/ota77xrr +2023-11-01 01:46:49,368 INFO MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0 +2023-11-01 01:46:49,368 INFO MainThread:34070 [wandb_run.py:_restore():2151] restore +2023-11-01 01:46:49,368 INFO MainThread:34070 [wandb_run.py:_restore():2157] restore done +2023-11-01 01:46:57,586 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-11-01 01:46:57,586 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-11-01 01:46:57,594 INFO MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files diff --git a/wandb/run-20231101_011849-ota77xrr/run-ota77xrr.wandb b/wandb/run-20231101_011849-ota77xrr/run-ota77xrr.wandb new file mode 100644 index 0000000000000000000000000000000000000000..dc3a794d8032dd31c7f56ea7a8907f0ab4fe05fa Binary files /dev/null and b/wandb/run-20231101_011849-ota77xrr/run-ota77xrr.wandb differ diff --git a/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml b/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a22d722e8862efc455867933dabd8fcf20f09d4 --- /dev/null +++ b/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml @@ -0,0 +1,197 @@ +name: env +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.4.0=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0 + - ca-certificates=2023.7.22=hbcca054_0 + - comm=0.1.4=pyhd8ed1ab_0 + - debugpy=1.6.7=py38h6a678d5_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.25.2=pyh2140261_0 + - ipython=8.12.0=pyh41d4057_0 + - jedi=0.19.1=pyhd8ed1ab_0 + - jupyter_client=7.3.4=pyhd8ed1ab_0 + - jupyter_core=5.4.0=py38h578d9bd_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.8=pyhd8ed1ab_0 + - openssl=3.0.11=h7f8727e_2 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.2.1=py38h06a4308_0 + - platformdirs=3.11.0=pyhd8ed1ab_0 + - prompt-toolkit=3.0.39=pyha770c72_0 + - prompt_toolkit=3.0.39=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.16.1=pyhd8ed1ab_0 + - python=3.8.18=h955ad1f_0 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.8=2_cp38 + - pyzmq=25.1.0=py38h6a678d5_0 + - readline=8.2=h5eee18b_0 + - setuptools=68.0.0=py38h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py38h0a891b7_3 + - traitlets=5.11.2=pyhd8ed1ab_0 + - typing-extensions=4.8.0=hd8ed1ab_0 + - typing_extensions=4.8.0=pyha770c72_0 + - wcwidth=0.2.8=pyhd8ed1ab_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h2531618_0 + - zlib=1.2.13=h5eee18b_0 + - pip: + - accelerate==0.22.0 + - aiofiles==23.2.1 + - aiohttp==3.8.5 + - aiosignal==1.3.1 + - alembic==1.12.0 + - altair==5.1.2 + - annotated-types==0.6.0 + - anyio==3.7.1 + - appdirs==1.4.4 + - async-timeout==4.0.3 + - attrs==23.1.0 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.7 + - cmake==3.25.0 + - colorama==0.4.6 + - coloredlogs==15.0.1 + - colorlog==6.7.0 + - contourpy==1.1.0 + - cycler==0.11.0 + - datasets==2.14.5 + - diffusers==0.21.0 + - dill==0.3.7 + - docker-pycreds==0.4.0 + - einops==0.6.1 + - exceptiongroup==1.1.3 + - fastapi==0.104.1 + - ffmpy==0.3.1 + - filelock==3.9.0 + - flatbuffers==23.5.26 + - fonttools==4.42.1 + - frozenlist==1.4.0 + - fsspec==2023.6.0 + - gitdb==4.0.11 + - gitpython==3.1.40 + - gradio==4.0.0 + - gradio-client==0.7.0 + - greenlet==3.0.0 + - h11==0.14.0 + - httpcore==0.18.0 + - httpx==0.25.0 + - huggingface-hub==0.17.1 + - humanfriendly==10.0 + - idna==3.4 + - importlib-metadata==6.8.0 + - importlib-resources==6.0.1 + - jinja2==3.1.2 + - joblib==1.3.2 + - jsonschema==4.19.2 + - jsonschema-specifications==2023.7.1 + - kiwisolver==1.4.5 + - lightning-utilities==0.9.0 + - lit==15.0.7 + - lpips==0.1.4 + - mako==1.2.4 + - markdown-it-py==3.0.0 + - markupsafe==2.1.2 + - matplotlib==3.7.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.15 + - networkx==3.0 + - numpy==1.24.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nvtx-cu11==11.7.91 + - onnx==1.15.0 + - onnxruntime==1.16.1 + - onnxruntime-gpu==1.16.1 + - optuna==3.4.0 + - orjson==3.9.10 + - packaging==23.1 + - pandas==2.0.3 + - pathtools==0.1.2 + - pillow==9.3.0 + - pkgutil-resolve-name==1.3.10 + - protobuf==4.24.4 + - psutil==5.9.5 + - pyarrow==13.0.0 + - pydantic==2.4.2 + - pydantic-core==2.10.1 + - pydub==0.25.1 + - pyparsing==3.1.1 + - python-multipart==0.0.6 + - pytz==2023.3.post1 + - pyyaml==6.0.1 + - referencing==0.30.2 + - regex==2023.8.8 + - requests==2.28.1 + - rich==13.6.0 + - rpds-py==0.10.6 + - safetensors==0.3.3 + - scikit-learn==1.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentry-sdk==1.32.0 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - smmap==5.0.1 + - sniffio==1.3.0 + - sqlalchemy==2.0.22 + - starlette==0.27.0 + - sympy==1.11.1 + - threadpoolctl==3.2.0 + - tokenizers==0.13.3 + - tomlkit==0.12.0 + - toolz==0.12.0 + - torch==2.0.1 + - torch-fidelity==0.3.0 + - torch-summary==1.4.5 + - torchaudio==2.0.2 + - torchmetrics==1.1.2 + - torchvision==0.15.2 + - tqdm==4.66.1 + - transformers==4.33.1 + - triton==2.0.0 + - typer==0.9.0 + - tzdata==2023.3 + - urllib3==1.26.13 + - uvicorn==0.23.2 + - wandb==0.15.12 + - websockets==11.0.3 + - xxhash==3.3.0 + - yarl==1.9.2 + - zipp==3.16.2 +prefix: /opt/conda/envs/env diff --git a/wandb/run-20231101_014658-mj672tzu/files/config.yaml b/wandb/run-20231101_014658-mj672tzu/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec2197a6e67a241e6f3cc78d36be7e2c9dd01b70 --- /dev/null +++ b/wandb/run-20231101_014658-mj672tzu/files/config.yaml @@ -0,0 +1,53 @@ +wandb_version: 1 + +optimizer: + desc: null + value: Adam +architecture: + desc: null + value: Efficientnet B0 +dataset: + desc: null + value: Food101 +epochs: + desc: null + value: 25 +_wandb: + desc: null + value: + python_version: 3.8.18 + cli_version: 0.15.12 + framework: huggingface + huggingface_version: 4.33.1 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1698788818.109446 + t: + 1: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 35 + - 41 + - 49 + - 55 + - 71 + - 105 + 3: + - 2 + - 16 + - 23 + 4: 3.8.18 + 5: 0.15.12 + 6: 4.33.1 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20231101_014658-mj672tzu/files/output.log b/wandb/run-20231101_014658-mj672tzu/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3209b44297ae1951fca862d06bd512462925a40e --- /dev/null +++ b/wandb/run-20231101_014658-mj672tzu/files/output.log @@ -0,0 +1,2 @@ + +Saving Model At: save_model/best_model.pth \ No newline at end of file diff --git a/wandb/run-20231101_014658-mj672tzu/files/requirements.txt b/wandb/run-20231101_014658-mj672tzu/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab622cc7f6744a9eb4ca8cec5764631762860a00 --- /dev/null +++ b/wandb/run-20231101_014658-mj672tzu/files/requirements.txt @@ -0,0 +1,168 @@ +accelerate==0.22.0 +aiofiles==23.2.1 +aiohttp==3.8.5 +aiosignal==1.3.1 +alembic==1.12.0 +altair==5.1.2 +annotated-types==0.6.0 +anyio==3.7.1 +appdirs==1.4.4 +asttokens==2.4.0 +async-timeout==4.0.3 +attrs==23.1.0 +backcall==0.2.0 +backports.functools-lru-cache==1.6.5 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.7 +cmake==3.25.0 +colorama==0.4.6 +coloredlogs==15.0.1 +colorlog==6.7.0 +comm==0.1.4 +contourpy==1.1.0 +cycler==0.11.0 +datasets==2.14.5 +debugpy==1.6.7 +decorator==5.1.1 +diffusers==0.21.0 +dill==0.3.7 +docker-pycreds==0.4.0 +einops==0.6.1 +entrypoints==0.4 +exceptiongroup==1.1.3 +executing==1.2.0 +fastapi==0.104.1 +ffmpy==0.3.1 +filelock==3.9.0 +flatbuffers==23.5.26 +fonttools==4.42.1 +frozenlist==1.4.0 +fsspec==2023.6.0 +gitdb==4.0.11 +gitpython==3.1.40 +gradio-client==0.7.0 +gradio==4.0.0 +greenlet==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +huggingface-hub==0.17.1 +humanfriendly==10.0 +idna==3.4 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +ipykernel==6.25.2 +ipython==8.12.0 +jedi==0.19.1 +jinja2==3.1.2 +joblib==1.3.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.2 +jupyter-client==7.3.4 +jupyter-core==5.4.0 +kiwisolver==1.4.5 +lightning-utilities==0.9.0 +lit==15.0.7 +lpips==0.1.4 +mako==1.2.4 +markdown-it-py==3.0.0 +markupsafe==2.1.2 +matplotlib-inline==0.1.6 +matplotlib==3.7.3 +mdurl==0.1.2 +mpmath==1.2.1 +multidict==6.0.4 +multiprocess==0.70.15 +nest-asyncio==1.5.8 +networkx==3.0 +numpy==1.24.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-cupti-cu11==11.7.101 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +nvidia-cufft-cu11==10.9.0.58 +nvidia-curand-cu11==10.2.10.91 +nvidia-cusolver-cu11==11.4.0.1 +nvidia-cusparse-cu11==11.7.4.91 +nvidia-nccl-cu11==2.14.3 +nvidia-nvtx-cu11==11.7.91 +onnx==1.15.0 +onnxruntime-gpu==1.16.1 +onnxruntime==1.16.1 +optuna==3.4.0 +orjson==3.9.10 +packaging==23.1 +pandas==2.0.3 +parso==0.8.3 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==9.3.0 +pip==23.2.1 +pkgutil-resolve-name==1.3.10 +platformdirs==3.11.0 +prompt-toolkit==3.0.39 +protobuf==4.24.4 +psutil==5.9.0 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==13.0.0 +pydantic-core==2.10.1 +pydantic==2.4.2 +pydub==0.25.1 +pygments==2.16.1 +pyparsing==3.1.1 +python-dateutil==2.8.2 +python-multipart==0.0.6 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.0 +referencing==0.30.2 +regex==2023.8.8 +requests==2.28.1 +rich==13.6.0 +rpds-py==0.10.6 +safetensors==0.3.3 +scikit-learn==1.3.1 +scipy==1.10.1 +semantic-version==2.10.0 +sentry-sdk==1.32.0 +setproctitle==1.3.3 +setuptools==68.0.0 +shellingham==1.5.4 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +sqlalchemy==2.0.22 +stack-data==0.6.2 +starlette==0.27.0 +sympy==1.11.1 +threadpoolctl==3.2.0 +tokenizers==0.13.3 +tomlkit==0.12.0 +toolz==0.12.0 +torch-fidelity==0.3.0 +torch-summary==1.4.5 +torch==2.0.1 +torchaudio==2.0.2 +torchmetrics==1.1.2 +torchvision==0.15.2 +tornado==6.1 +tqdm==4.66.1 +traitlets==5.11.2 +transformers==4.33.1 +triton==2.0.0 +typer==0.9.0 +typing-extensions==4.8.0 +tzdata==2023.3 +urllib3==1.26.13 +uvicorn==0.23.2 +wandb==0.15.12 +wcwidth==0.2.8 +websockets==11.0.3 +wheel==0.38.4 +xxhash==3.3.0 +yarl==1.9.2 +zipp==3.16.2 \ No newline at end of file diff --git a/wandb/run-20231101_014658-mj672tzu/files/wandb-metadata.json b/wandb/run-20231101_014658-mj672tzu/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..16f77d8cc1723252f8eddba9ca0e1525098f562b --- /dev/null +++ b/wandb/run-20231101_014658-mj672tzu/files/wandb-metadata.json @@ -0,0 +1,254 @@ +{ + "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17", + "python": "3.8.18", + "heartbeatAt": "2023-10-31T21:47:00.334647", + "startedAt": "2023-10-31T21:46:58.105196", + "docker": null, + "cuda": null, + "args": [], + "state": "running", + "program": "hyperpara.py", + "codePathLocal": "hyperpara.py", + "codePath": "hyperpara.py", + "host": "76ecfeea2960", + "username": "root", + "executable": "/opt/conda/envs/env/bin/python", + "cpu_count": 20, + "cpu_count_logical": 40, + "cpu_freq": { + "current": 1.519625, + "min": 1200.0, + "max": 3600.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.197, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.208, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.297, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.499, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.465, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.697, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.476, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.208, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.698, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.196, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.205, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.204, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.299, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.7, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.498, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.268, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.2, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.251, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.697, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.534, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 2.198, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + }, + { + "current": 1.199, + "min": 1200.0, + "max": 3600.0 + } + ], + "disk": { + "/": { + "total": 1758.95463180542, + "used": 866.9305191040039 + } + }, + "gpu": "Tesla V100-DGXS-32GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + }, + { + "name": "Tesla V100-DGXS-32GB", + "memory_total": 34359738368 + } + ], + "memory": { + "total": 251.8238754272461 + } +} diff --git a/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json b/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b8030111ec6c526834cb398154c20cef73646194 --- /dev/null +++ b/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json @@ -0,0 +1 @@ +{"Train Loss": 1.3167718648910522, "Train Accuracy": 0.5293803811073303, "Validation Loss": 1.200211763381958, "Validation Accuracy": 0.5658254027366638, "_timestamp": 1698790257.8465717, "_runtime": 1439.737125635147, "_step": 0, "_wandb": {"runtime": 1437}} \ No newline at end of file diff --git a/wandb/run-20231101_014658-mj672tzu/logs/debug-internal.log b/wandb/run-20231101_014658-mj672tzu/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..833ddecd68314a60007aa0e7b564084d5be84f6d --- /dev/null +++ b/wandb/run-20231101_014658-mj672tzu/logs/debug-internal.log @@ -0,0 +1,797 @@ +2023-11-01 01:46:58,588 INFO StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-11-01 01:46:58.587580 +2023-11-01 01:46:58,590 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status +2023-11-01 01:46:58,592 INFO WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/run-mj672tzu.wandb +2023-11-01 01:46:58,595 DEBUG SenderThread:34227 [sender.py:send():380] send: header +2023-11-01 01:46:58,595 DEBUG SenderThread:34227 [sender.py:send():380] send: run +2023-11-01 01:46:59,448 INFO SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files +2023-11-01 01:46:59,448 INFO SenderThread:34227 [sender.py:_start_run_threads():1122] run started: mj672tzu with start time 1698788818.109446 +2023-11-01 01:46:59,469 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version +2023-11-01 01:46:59,470 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: check_version +2023-11-01 01:47:00,265 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start +2023-11-01 01:47:00,267 DEBUG HandlerThread:34227 [system_info.py:__init__():32] System info init +2023-11-01 01:47:00,267 DEBUG HandlerThread:34227 [system_info.py:__init__():47] System info init done +2023-11-01 01:47:00,267 INFO HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor +2023-11-01 01:47:00,267 INFO SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-11-01 01:47:00,267 INFO HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info +2023-11-01 01:47:00,268 INFO SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring +2023-11-01 01:47:00,269 INFO SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring +2023-11-01 01:47:00,270 INFO SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring +2023-11-01 01:47:00,271 INFO SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring +2023-11-01 01:47:00,272 INFO SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring +2023-11-01 01:47:00,334 DEBUG HandlerThread:34227 [system_info.py:probe():196] Probing system +2023-11-01 01:47:00,335 DEBUG HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid +2023-11-01 01:47:00,335 DEBUG HandlerThread:34227 [system_info.py:probe():244] Probing system done +2023-11-01 01:47:00,335 DEBUG HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T21:47:00.334647', 'startedAt': '2023-10-31T21:46:58.105196', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.519625, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.208, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.297, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.499, 'min': 1200.0, 'max': 3600.0}, {'current': 1.465, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.697, 'min': 1200.0, 'max': 3600.0}, {'current': 1.476, 'min': 1200.0, 'max': 3600.0}, {'current': 2.208, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.196, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.204, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 2.7, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.498, 'min': 1200.0, 'max': 3600.0}, {'current': 1.268, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.251, 'min': 1200.0, 'max': 3600.0}, {'current': 1.697, 'min': 1200.0, 'max': 3600.0}, {'current': 1.534, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9305191040039}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}} +2023-11-01 01:47:00,335 INFO HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info +2023-11-01 01:47:00,335 INFO HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info +2023-11-01 01:47:00,335 DEBUG HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-11-01 01:47:00,335 DEBUG HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done +2023-11-01 01:47:00,336 DEBUG HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-11-01 01:47:00,451 INFO Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/requirements.txt +2023-11-01 01:47:00,452 INFO Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml +2023-11-01 01:47:06,925 DEBUG HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done +2023-11-01 01:47:06,927 INFO HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info +2023-11-01 01:47:06,927 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:47:06,928 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive +2023-11-01 01:47:06,929 DEBUG SenderThread:34227 [sender.py:send():380] send: files +2023-11-01 01:47:06,929 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now +2023-11-01 01:47:06,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:47:06,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:47:06,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:47:07,442 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-11-01 01:47:07,450 INFO Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml +2023-11-01 01:47:07,451 INFO Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-metadata.json +2023-11-01 01:47:09,285 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmpqi5xsr4gwandb/rhtw7848-wandb-metadata.json +2023-11-01 01:47:09,443 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:47:14,444 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:47:19,445 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:47:21,940 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:47:21,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:47:21,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:47:25,370 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:47:30,381 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:47:31,458 INFO Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/config.yaml +2023-11-01 01:47:35,747 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:47:36,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:47:36,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:47:36,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:47:41,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:47:46,297 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:47:51,298 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:47:51,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:47:51,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:47:51,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:47:57,290 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:48:00,272 DEBUG SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-11-01 01:48:00,276 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:48:03,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:48:06,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:48:06,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:48:06,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:48:08,317 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:48:13,318 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:48:18,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:48:21,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:48:21,941 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:48:21,981 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:48:24,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:48:29,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:48:30,278 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:48:35,279 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:48:36,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:48:36,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:48:36,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:48:40,332 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:48:45,332 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:48:50,333 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:48:51,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:48:51,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:48:51,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:48:56,275 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:49:00,280 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:49:01,281 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:49:06,282 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:49:06,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:49:06,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:49:06,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:49:12,279 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:49:17,279 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:49:21,941 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:49:21,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:49:21,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:49:22,346 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:49:27,347 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:49:30,282 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:49:33,283 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:49:36,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:49:36,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:49:36,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:49:38,291 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:49:43,292 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:49:48,293 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:49:51,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:49:51,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:49:51,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:49:53,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:49:58,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:50:00,284 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:50:04,285 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:50:06,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:50:06,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:50:06,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:50:09,358 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:50:14,359 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:50:19,359 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:50:21,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:50:21,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:50:21,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:50:24,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:50:29,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:50:30,286 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:50:35,287 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:50:36,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:50:36,942 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:50:36,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:50:40,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:50:45,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:50:50,362 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:50:51,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:50:51,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:50:51,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:50:56,350 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:51:00,288 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:51:02,290 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:51:06,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:51:06,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:51:06,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:51:07,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:51:09,530 INFO Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log +2023-11-01 01:51:11,530 INFO Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log +2023-11-01 01:51:12,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:51:17,954 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:51:21,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:51:21,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:51:21,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:51:23,377 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:51:28,378 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:51:30,290 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:51:34,292 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:51:36,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:51:36,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:51:36,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:51:39,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:51:44,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:51:49,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:51:51,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:51:51,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:51:51,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:51:55,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:52:00,293 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:52:01,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:52:06,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:52:06,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:52:06,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:52:06,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:52:11,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:52:16,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:52:21,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:52:21,942 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:52:21,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:52:21,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:52:26,377 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:52:30,295 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:52:32,297 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:52:36,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:52:36,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:52:36,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:52:38,285 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:52:43,285 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:52:48,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:52:51,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:52:51,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:52:51,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:52:53,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:52:58,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:53:00,297 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:53:04,299 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:53:06,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:53:06,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:53:06,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:53:09,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:53:14,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:53:19,370 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:53:21,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:53:21,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:53:21,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:53:25,282 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:53:30,283 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:53:30,299 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:53:35,300 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:53:36,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:53:36,943 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:53:36,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:53:41,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:53:46,287 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:53:51,287 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:53:51,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:53:51,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:53:51,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:53:56,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:54:00,301 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:54:02,302 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:54:06,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:54:06,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:54:06,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:54:07,360 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:54:12,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:54:17,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:54:21,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:54:21,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:54:21,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:54:23,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:54:28,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:54:30,302 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:54:34,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:54:36,943 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:54:36,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:54:36,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:54:39,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:54:44,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:54:49,368 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:54:51,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:54:51,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:54:51,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:54:55,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:55:00,306 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:55:01,307 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:55:06,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:55:06,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:55:06,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:55:06,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:55:12,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:55:17,308 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:55:21,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:55:21,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:55:21,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:55:22,370 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:55:27,371 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:55:30,308 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:55:33,309 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:55:36,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:55:36,944 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:55:36,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:55:38,354 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:55:43,355 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:55:48,356 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:55:51,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:55:51,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:55:51,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:55:54,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:55:59,306 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:56:00,309 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:56:04,311 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:56:06,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:56:06,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:56:06,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:56:09,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:56:14,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:56:19,329 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:56:21,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:56:21,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:56:21,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:56:24,356 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:56:29,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:56:30,311 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:56:35,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:56:36,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:56:36,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:56:36,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:56:40,339 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:56:45,340 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:56:50,340 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:56:51,944 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:56:51,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:56:51,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:56:55,379 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:57:00,313 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:57:01,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:57:06,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:57:06,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:57:06,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:57:06,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:57:12,281 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:57:17,282 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:57:21,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:57:21,945 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:57:21,985 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:57:22,295 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:57:27,296 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:57:30,315 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:57:32,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:57:36,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:57:36,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:57:36,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:57:37,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:57:42,366 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:57:47,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:57:51,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:57:51,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:57:51,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:57:53,336 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:57:58,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:58:00,318 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:58:04,319 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:58:06,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:58:06,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:58:06,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:58:09,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:58:14,338 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:58:19,339 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:58:21,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:58:21,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:58:21,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:58:24,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:58:29,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:58:30,320 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:58:35,322 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:58:36,945 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:58:36,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:58:36,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:58:40,450 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:58:45,451 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:58:50,451 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:58:51,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:58:51,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:58:51,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:58:56,294 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:59:00,322 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:59:01,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:59:06,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:59:06,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:59:06,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:59:06,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:59:11,351 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:59:16,351 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:59:21,352 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:59:21,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:59:21,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:59:21,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:59:27,314 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:59:30,324 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 01:59:32,325 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:59:36,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:59:36,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:59:36,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:59:38,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:59:43,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:59:48,313 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:59:51,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 01:59:51,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 01:59:51,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 01:59:54,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 01:59:59,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:00:00,326 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:00:04,328 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:00:06,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:00:06,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:00:06,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:00:09,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:00:14,338 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:00:19,338 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:00:21,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:00:21,946 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:00:21,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:00:25,276 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:00:30,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:00:30,329 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:00:35,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:00:36,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:00:36,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:00:36,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:00:40,373 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:00:45,374 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:00:50,375 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:00:51,946 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:00:51,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:00:51,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:00:56,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:01:00,330 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:01:01,332 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:01:06,332 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:01:06,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:01:06,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:01:06,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:01:12,304 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:01:17,305 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:01:21,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:01:21,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:01:21,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:01:22,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:01:27,316 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:01:30,332 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:01:32,334 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:01:36,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:01:36,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:01:36,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:01:37,377 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:01:42,377 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:01:47,378 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:01:51,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:01:51,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:01:51,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:01:52,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:01:57,385 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:02:00,335 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:02:03,337 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:02:06,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:02:06,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:02:06,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:02:09,283 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:02:14,284 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:02:19,284 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:02:21,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:02:21,947 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:02:21,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:02:24,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:02:29,327 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:02:30,338 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:02:34,340 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:02:36,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:02:36,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:02:36,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:02:39,351 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:02:44,352 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:02:49,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:02:51,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:02:51,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:02:51,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:02:55,275 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:03:00,276 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:03:00,341 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:03:05,344 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:03:06,947 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:03:06,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:03:06,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:03:10,381 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:03:15,381 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:03:20,382 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:03:21,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:03:21,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:03:21,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:03:25,389 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:03:30,344 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:03:31,346 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:03:36,347 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:03:36,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:03:36,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:03:36,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:03:41,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:03:46,357 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:03:51,358 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:03:51,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:03:51,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:03:51,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:03:56,622 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:04:00,347 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:04:02,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:04:06,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:04:06,948 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:04:06,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:04:08,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:04:13,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:04:18,336 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:04:21,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:04:21,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:04:21,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:04:24,298 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:04:29,298 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:04:30,351 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:04:34,353 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:04:36,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:04:36,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:04:36,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:04:39,360 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:04:44,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:04:49,362 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:04:51,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:04:51,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:04:51,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:04:54,372 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:04:59,373 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:05:00,352 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:05:05,354 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:05:06,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:05:06,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:05:06,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:05:11,299 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:05:16,299 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:05:21,300 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:05:21,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:05:21,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:05:21,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:05:27,283 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:05:30,354 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:05:32,355 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:05:36,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:05:36,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:05:36,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:05:38,348 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:05:43,348 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:05:48,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:05:51,948 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:05:51,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:05:51,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:05:53,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:05:58,367 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:06:00,357 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:06:04,359 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:06:06,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:06:06,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:06:06,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:06:10,333 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:06:15,334 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:06:20,335 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:06:21,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:06:21,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:06:21,989 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:06:26,315 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:06:30,359 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:06:31,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:06:36,361 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:06:36,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:06:36,949 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:06:36,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:06:42,355 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:06:47,355 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:06:51,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:06:51,950 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:06:51,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:06:53,293 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:06:58,293 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:07:00,361 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:07:03,362 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:07:06,949 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:07:06,950 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:07:06,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:07:09,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:07:14,320 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:07:19,321 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:07:21,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:07:21,950 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:07:21,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:07:24,378 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:07:29,379 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:07:30,363 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:07:35,365 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:07:36,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:07:36,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:07:36,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:07:41,323 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:07:46,323 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:07:51,324 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:07:51,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:07:51,950 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:07:51,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:07:56,393 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:08:00,366 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:08:02,368 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:08:06,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:08:06,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:08:06,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:08:07,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:08:12,387 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:08:17,388 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:08:21,950 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:08:21,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:08:21,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:08:23,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:08:28,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:08:30,368 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:08:34,369 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:08:36,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:08:36,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:08:36,953 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:08:39,446 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:08:44,447 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:08:49,447 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:08:51,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:08:51,952 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:08:51,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:08:55,285 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:09:00,286 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:09:00,372 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:09:05,374 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:09:06,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:09:06,951 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:09:06,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:09:11,276 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:09:16,277 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:09:21,278 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:09:21,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:09:21,952 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:09:21,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:09:26,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:09:30,376 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:09:31,377 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:09:36,378 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:09:36,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:09:36,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:09:36,953 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:09:41,398 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:09:46,398 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:09:51,399 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:09:51,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:09:51,952 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:09:51,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:09:57,396 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:00,379 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:10:03,381 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:06,951 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:10:06,952 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:10:06,993 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:10:09,330 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:14,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:19,331 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:21,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:10:21,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:10:21,953 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:10:25,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:30,326 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:30,382 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:10:35,383 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:36,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:10:36,952 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:10:36,953 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:10:41,349 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:46,350 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:51,351 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:51,953 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status +2023-11-01 02:10:51,953 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:10:51,953 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: stop_status +2023-11-01 02:10:57,312 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:57,847 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history +2023-11-01 02:10:57,849 DEBUG SenderThread:34227 [sender.py:send():380] send: history +2023-11-01 02:10:57,849 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: summary_record +2023-11-01 02:10:57,849 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 02:10:57,889 DEBUG SenderThread:34227 [sender.py:send():380] send: telemetry +2023-11-01 02:10:57,889 DEBUG SenderThread:34227 [sender.py:send():380] send: exit +2023-11-01 02:10:57,889 INFO SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0 +2023-11-01 02:10:57,889 INFO SenderThread:34227 [sender.py:send_exit():587] handling runtime: 1437 +2023-11-01 02:10:57,890 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 02:10:57,890 INFO SenderThread:34227 [sender.py:send_exit():593] send defer +2023-11-01 02:10:57,890 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:10:57,890 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0 +2023-11-01 02:10:57,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:10:57,891 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-11-01 02:10:57,891 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 1 +2023-11-01 02:10:57,891 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:10:57,891 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1 +2023-11-01 02:10:57,891 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:10:57,892 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-11-01 02:10:57,892 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 2 +2023-11-01 02:10:57,892 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:10:57,892 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2 +2023-11-01 02:10:57,892 INFO HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor +2023-11-01 02:10:57,892 DEBUG SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-11-01 02:10:57,893 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor +2023-11-01 02:10:57,893 DEBUG SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-11-01 02:10:57,893 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor +2023-11-01 02:10:57,953 INFO Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json +2023-11-01 02:10:59,649 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor +2023-11-01 02:10:59,650 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor +2023-11-01 02:10:59,650 INFO HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor +2023-11-01 02:10:59,650 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 02:10:59,651 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:10:59,651 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-11-01 02:10:59,652 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 3 +2023-11-01 02:10:59,652 DEBUG SenderThread:34227 [sender.py:send():380] send: stats +2023-11-01 02:10:59,652 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 02:10:59,653 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:10:59,653 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3 +2023-11-01 02:10:59,653 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:10:59,654 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-11-01 02:10:59,654 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 4 +2023-11-01 02:10:59,654 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:10:59,654 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4 +2023-11-01 02:10:59,654 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:10:59,654 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-11-01 02:10:59,654 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 5 +2023-11-01 02:10:59,655 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:10:59,655 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5 +2023-11-01 02:10:59,655 DEBUG SenderThread:34227 [sender.py:send():380] send: summary +2023-11-01 02:10:59,656 INFO SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end +2023-11-01 02:10:59,656 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:10:59,656 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-11-01 02:10:59,656 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 6 +2023-11-01 02:10:59,656 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:10:59,657 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6 +2023-11-01 02:10:59,657 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:10:59,657 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-11-01 02:10:59,662 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report +2023-11-01 02:10:59,849 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 02:10:59,954 INFO Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json +2023-11-01 02:11:00,056 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 7 +2023-11-01 02:11:00,056 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 02:11:00,056 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:11:00,057 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7 +2023-11-01 02:11:00,057 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:11:00,057 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-11-01 02:11:00,230 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 8 +2023-11-01 02:11:00,231 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:11:00,231 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8 +2023-11-01 02:11:00,231 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:11:00,231 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-11-01 02:11:00,232 INFO SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact +2023-11-01 02:11:00,232 INFO SenderThread:34227 [job_builder.py:_get_source_type():404] no source found +2023-11-01 02:11:00,232 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 9 +2023-11-01 02:11:00,233 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:11:00,233 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9 +2023-11-01 02:11:00,233 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:11:00,233 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-11-01 02:11:00,233 INFO SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher +2023-11-01 02:11:00,849 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 02:11:00,955 INFO Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/config.yaml +2023-11-01 02:11:00,955 INFO SenderThread:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log +2023-11-01 02:11:00,956 INFO SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files +2023-11-01 02:11:00,956 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log output.log +2023-11-01 02:11:00,957 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/requirements.txt requirements.txt +2023-11-01 02:11:00,962 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml conda-environment.yaml +2023-11-01 02:11:00,963 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json wandb-summary.json +2023-11-01 02:11:00,967 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/config.yaml config.yaml +2023-11-01 02:11:00,980 INFO SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-metadata.json wandb-metadata.json +2023-11-01 02:11:00,981 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 10 +2023-11-01 02:11:00,981 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 02:11:00,982 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:11:00,989 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10 +2023-11-01 02:11:00,990 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:11:00,990 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-11-01 02:11:00,990 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-11-01 02:11:01,845 INFO wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log +2023-11-01 02:11:01,850 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 02:11:01,851 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 02:11:02,100 INFO wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/requirements.txt +2023-11-01 02:11:02,147 INFO wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json +2023-11-01 02:11:02,154 INFO wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/config.yaml +2023-11-01 02:11:02,851 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 02:11:02,852 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 02:11:03,196 INFO wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml +2023-11-01 02:11:03,396 INFO Thread-132:34227 [sender.py:transition_state():613] send defer: 11 +2023-11-01 02:11:03,397 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:11:03,397 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11 +2023-11-01 02:11:03,398 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:11:03,398 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-11-01 02:11:03,398 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher +2023-11-01 02:11:03,398 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 12 +2023-11-01 02:11:03,398 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:11:03,399 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12 +2023-11-01 02:11:03,399 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:11:03,399 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-11-01 02:11:03,399 INFO SenderThread:34227 [file_stream.py:finish():595] file stream finish called +2023-11-01 02:11:03,757 INFO SenderThread:34227 [file_stream.py:finish():599] file stream finish is done +2023-11-01 02:11:03,757 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 13 +2023-11-01 02:11:03,758 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:11:03,758 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13 +2023-11-01 02:11:03,758 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:11:03,758 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-11-01 02:11:03,759 INFO SenderThread:34227 [sender.py:transition_state():613] send defer: 14 +2023-11-01 02:11:03,759 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer +2023-11-01 02:11:03,759 DEBUG SenderThread:34227 [sender.py:send():380] send: final +2023-11-01 02:11:03,760 INFO HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14 +2023-11-01 02:11:03,760 DEBUG SenderThread:34227 [sender.py:send():380] send: footer +2023-11-01 02:11:03,760 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: defer +2023-11-01 02:11:03,760 INFO SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-11-01 02:11:03,762 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-01 02:11:03,763 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-01 02:11:03,763 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit +2023-11-01 02:11:03,764 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info +2023-11-01 02:11:03,764 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary +2023-11-01 02:11:03,765 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history +2023-11-01 02:11:03,765 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: server_info +2023-11-01 02:11:03,771 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info +2023-11-01 02:11:04,116 DEBUG SenderThread:34227 [sender.py:send_request():407] send_request: job_info +2023-11-01 02:11:04,117 DEBUG HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown +2023-11-01 02:11:04,117 INFO HandlerThread:34227 [handler.py:finish():866] shutting down handler +2023-11-01 02:11:04,772 INFO WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/run-mj672tzu.wandb +2023-11-01 02:11:05,116 INFO SenderThread:34227 [sender.py:finish():1534] shutting down sender +2023-11-01 02:11:05,116 INFO SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher +2023-11-01 02:11:05,116 INFO SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher diff --git a/wandb/run-20231101_014658-mj672tzu/logs/debug.log b/wandb/run-20231101_014658-mj672tzu/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f649bb1d9bd585caf1ff01dae2df031789bed728 --- /dev/null +++ b/wandb/run-20231101_014658-mj672tzu/logs/debug.log @@ -0,0 +1,34 @@ +2023-11-01 01:46:58,106 INFO MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12 +2023-11-01 01:46:58,106 INFO MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070 +2023-11-01 01:46:58,106 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings +2023-11-01 01:46:58,106 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings +2023-11-01 01:46:58,106 INFO MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2023-11-01 01:46:58,106 INFO MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'} +2023-11-01 01:46:58,107 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-11-01 01:46:58,107 INFO MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'} +2023-11-01 01:46:58,107 INFO MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/logs/debug.log +2023-11-01 01:46:58,107 INFO MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/logs/debug-internal.log +2023-11-01 01:46:58,107 INFO MainThread:34070 [wandb_init.py:init():568] calling init triggers +2023-11-01 01:46:58,107 INFO MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {} +config: {'optimizer': 'Adam', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 25} +2023-11-01 01:46:58,107 INFO MainThread:34070 [wandb_init.py:init():618] starting backend +2023-11-01 01:46:58,107 INFO MainThread:34070 [wandb_init.py:init():622] setting up manager +2023-11-01 01:46:58,109 INFO MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-11-01 01:46:58,109 INFO MainThread:34070 [wandb_init.py:init():628] backend started and connected +2023-11-01 01:46:58,111 INFO MainThread:34070 [wandb_init.py:init():720] updated telemetry +2023-11-01 01:46:58,112 INFO MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2023-11-01 01:46:59,468 INFO MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version +2023-11-01 01:47:00,159 INFO MainThread:34070 [wandb_run.py:_on_init():2229] got version response +2023-11-01 01:47:00,159 INFO MainThread:34070 [wandb_init.py:init():804] starting run threads in backend +2023-11-01 01:47:06,941 INFO MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg +2023-11-01 01:47:06,942 INFO MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw +2023-11-01 01:47:06,942 INFO MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams. +2023-11-01 01:47:06,942 INFO MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed. +2023-11-01 01:47:06,943 INFO MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process +2023-11-01 02:10:57,847 INFO MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/mj672tzu +2023-11-01 02:10:57,847 INFO MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0 +2023-11-01 02:10:57,847 INFO MainThread:34070 [wandb_run.py:_restore():2151] restore +2023-11-01 02:10:57,847 INFO MainThread:34070 [wandb_run.py:_restore():2157] restore done +2023-11-01 02:11:05,119 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history +2023-11-01 02:11:05,120 INFO MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary +2023-11-01 02:11:05,127 INFO MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files diff --git a/wandb/run-20231101_014658-mj672tzu/run-mj672tzu.wandb b/wandb/run-20231101_014658-mj672tzu/run-mj672tzu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8921c0837033fd49730590ef9d73fe9164e5e2a2 Binary files /dev/null and b/wandb/run-20231101_014658-mj672tzu/run-mj672tzu.wandb differ