Training in progress, step 40

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb filter=lfs diff=lfs merge=lfs -text

.hydra/config.yaml

@@ -0,0 +1,80 @@
+time_start: null
+DEBUG: false
+debug_model: unsloth/Qwen2.5-7B-bnb-4bit
+fold: 0
+random_seed: true
+train_on_all_folds: false
+eval_only: false
+merge_adapters: false
+wandb_id: null
+val_split_name: val
+pad_token: <pad>
+response_template_ids:
+- 4
+num_proc: 20
+hub_repo_tags:
+- odesia
+script_args:
+  dataset_name: nbroad/odesia-combined-dipromats-v1
+  config: null
+  gradient_checkpointing_use_reentrant: true
+  ignore_bias_buffers: false
+model_config:
+  model_name_or_path: mistralai/Ministral-8B-Instruct-2410
+  torch_dtype: bfloat16
+  attn_implementation: flash_attention_2
+  use_peft: true
+  lora_r: 16
+  lora_alpha: 32
+  lora_dropout: 0.05
+  lora_target_modules:
+  - q_proj
+  - v_proj
+  - k_proj
+  - o_proj
+  - up_proj
+  - down_proj
+  - gate_proj
+  lora_modules_to_save: null
+  lora_task_type: CAUSAL_LM
+  use_rslora: true
+  load_in_8bit: false
+  load_in_4bit: false
+  bnb_4bit_quant_type: nf4
+  use_bnb_nested_quant: true
+training_args:
+  resume_from_checkpoint: null
+  output_dir: ./
+  num_train_epochs: 2
+  per_device_train_batch_size: 4
+  per_device_eval_batch_size: 4
+  warmup_ratio: 0.1
+  fp16: false
+  bf16: true
+  eval_strategy: 'no'
+  save_strategy: steps
+  eval_steps: 40
+  save_steps: 40
+  save_total_limit: 2
+  logging_steps: 2
+  run_name: null
+  weight_decay: 0.01
+  report_to: wandb
+  learning_rate: 5.0e-05
+  metric_for_best_model: loss
+  greater_is_better: false
+  gradient_checkpointing: true
+  gradient_accumulation_steps: 8
+  gradient_checkpointing_kwargs:
+    use_reentrant: true
+  optim: adamw_torch
+  dataloader_num_workers: 1
+  seed: 18
+  max_grad_norm: 2.0
+  load_best_model_at_end: false
+  push_to_hub: true
+  hub_private_repo: true
+  lr_scheduler_type: cosine
+  remove_unused_columns: false
+  ddp_find_unused_parameters: false
+  use_liger_kernel: true

.hydra/hydra.yaml

@@ -0,0 +1,157 @@
+hydra:
+  run:
+    dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: train
+    chdir: null
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: d
+    env_set:
+      WANDB_RUN_GROUP: dipromats
+      TOKENIZERS_PARALLELISM: 'False'
+      HF_HUB_ENABLE_HF_TRANSFER: '1'
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /workspace/odesia-2025/train
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/odesia-2025/train/conf
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

adapter_config.json

@@ -0,0 +1,37 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "mistralai/Ministral-8B-Instruct-2410",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "o_proj",
+    "down_proj",
+    "v_proj",
+    "gate_proj",
+    "up_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": true
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d28cc1409287c41567696df3eb3a18d1835089f2c0add705b803d2a406537b95
+size 174655536

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7edbeaf20dd7f571b5dd1c54d9ace4f9b6299127cc7ba2afb14a6d51a4a79a4
+size 17078136

train.log

@@ -0,0 +1 @@
+[2025-02-03 02:28:22,500][accelerate.utils.other][WARNING] - Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.

train.py

@@ -0,0 +1,259 @@
+from transformers import (
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    Trainer,
+    TrainingArguments,
+)
+from datasets import load_dataset, concatenate_datasets
+from omegaconf import DictConfig, OmegaConf
+import hydra
+import wandb
+import shutil
+import os
+from functools import partial
+from pathlib import Path
+from trl import (
+    SFTTrainer,
+    ModelConfig,
+    get_quantization_config,
+    get_kbit_device_map,
+    get_peft_config,
+    DataCollatorForCompletionOnlyLM,
+)
+from dotenv import load_dotenv
+from peft import (
+    get_peft_model,
+    prepare_model_for_kbit_training,
+    AutoPeftModelForSequenceClassification,
+)
+
+# from utils import add_metric_to_card
+
+loaded = load_dotenv("../.env", override=True)
+
+if not loaded:
+    raise ValueError("Failed to load .env file")
+
+
+def tokenize(example, tokenizer):
+    ids = tokenizer.apply_chat_template([
+        {"role": "user", "content": example["text"]},
+        {"role": "assistant", "content": example["response"]},
+    ])
+
+    return {
+        "input_ids": ids,
+    }
+
+
+@hydra.main(config_path="conf", config_name="q7b-4bit")
+def main(cfg: DictConfig):
+
+    cfg.time_start = "_".join(str(Path.cwd()).rsplit("/", 2)[-2:])
+
+    if cfg.DEBUG:
+        cfg.model_config.model_name_or_path = cfg.debug_model
+
+    script_args = cfg.script_args
+    training_args = TrainingArguments(**OmegaConf.to_container(cfg.training_args))
+    model_config = ModelConfig(**OmegaConf.to_container(cfg.model_config))
+
+    if training_args.process_index == 0:
+
+        if cfg.eval_only or training_args.resume_from_checkpoint is not None:
+            wandb_id = cfg.wandb_id
+            resume = "must"
+            config = None
+        else:
+            wandb_id = None
+            resume = None
+            config = OmegaConf.to_container(cfg)
+
+        wandb.init(config=config, id=wandb_id, resume=resume)
+        # copy current file to output, so it gets saved to hub
+        shutil.copy(
+            Path(__file__).resolve(),
+            Path(training_args.output_dir) / Path(__file__).name,
+        )
+
+        shutil.copy(
+            Path(__file__).resolve().parent / "utils.py",
+            Path(training_args.output_dir) / "utils.py",
+        )
+
+    quantization_config = get_quantization_config(model_config)
+    model_kwargs = dict(
+        revision=model_config.model_revision,
+        trust_remote_code=model_config.trust_remote_code,
+        attn_implementation=model_config.attn_implementation,
+        torch_dtype=model_config.torch_dtype,
+        use_cache=False if training_args.gradient_checkpointing else True,
+        device_map=get_kbit_device_map() if quantization_config is not None else None,
+        quantization_config=quantization_config,
+        cache_dir=os.environ["HF_HUB_CACHE"],
+    )
+
+    peft_config = get_peft_config(model_config)
+
+    if training_args.use_liger_kernel:
+        from liger_kernel.transformers import (
+            apply_liger_kernel_to_qwen2,
+            apply_liger_kernel_to_llama,
+            apply_liger_kernel_to_mistral,
+        )
+
+        apply_liger_kernel_to_qwen2()
+        apply_liger_kernel_to_llama()
+        apply_liger_kernel_to_mistral()
+    if cfg.eval_only:
+
+        model = AutoPeftModelForSequenceClassification.from_pretrained(
+            model_config.model_name_or_path,
+            **model_kwargs,
+            token=os.environ["HF_WRITE_PERSONAL"],
+        )
+
+        if cfg.merge_adapters:
+            model = model.merge_and_unload()
+
+    else:
+
+        model = AutoModelForCausalLM.from_pretrained(
+            model_config.model_name_or_path,
+            **model_kwargs,
+            token=os.environ["HF_GATED"],
+        )
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_config.model_name_or_path,
+        use_fast=True,
+        token=os.environ["HF_GATED"],
+    )
+
+    tokenizer.padding_side = "left"
+    tokenizer.pad_token = cfg.pad_token
+
+
+    if not cfg.eval_only and model_config.load_in_4bit:
+        model = prepare_model_for_kbit_training(
+            model,
+            use_gradient_checkpointing=training_args.gradient_checkpointing,
+            gradient_checkpointing_kwargs=training_args.gradient_checkpointing_kwargs,
+        )
+
+    elif not cfg.eval_only and training_args.gradient_checkpointing:
+        model.enable_input_require_grads()
+
+    if not cfg.eval_only:
+        model = get_peft_model(model, peft_config)
+
+    with training_args.main_process_first():
+        ds = load_dataset(
+            script_args.dataset_name,
+            script_args.config,
+            token=os.environ["HF_WRITE_PERSONAL"],
+        )
+
+# hack to downsample english squad
+        # ds["train"] = concatenate_datasets(
+        #     [
+        #         ds["train"].select(range(0, 45000)),
+        #         ds["train"].select(range(98596, len(ds["train"]))),
+        #                                  ])
+
+        if cfg.DEBUG:
+            ds[cfg.train_split_name] = (
+                ds[cfg.train_split_name].shuffle().select(range(100))
+            )
+            # ds[cfg.val_split_name] = ds[cfg.val_split_name].shuffle().select(range(100))
+
+        # if not cfg.eval_only:
+            # ds[cfg.val_split_name] = ds[cfg.val_split_name].shuffle().select(range(500))
+
+        ds = ds.map(tokenize, fn_kwargs={"tokenizer": tokenizer}, num_proc=cfg.num_proc, remove_columns=ds["train"].column_names)
+
+    collator = DataCollatorForCompletionOnlyLM(
+        tokenizer=tokenizer,
+        mlm=False,
+        pad_to_multiple_of=16,
+        response_template=cfg.response_template_ids
+    )
+
+    if training_args.process_index == 0:
+        group = os.environ["WANDB_RUN_GROUP"]
+        training_args.hub_model_id = f"nbroad/nbroad-odesia-{group}-{wandb.run.id}"
+        training_args.hub_token = os.environ["HF_WRITE_PERSONAL"]
+
+    prefix = ""
+
+    if cfg.eval_only:
+        if "awq" in model_config.model_name_or_path.lower():
+            prefix = "awq_"
+        if model_config.load_in_4bit:
+            prefix += "int4_"
+        elif model_config.torch_dtype == "bfloat16":
+            prefix += "bf16_"
+        elif model_config.torch_dtype == "float16":
+            prefix += "fp16_"
+
+    trainer = SFTTrainer(
+        model=model,
+        args=training_args,
+        train_dataset=ds["train"],
+        eval_dataset=(
+            ds[cfg.val_split_name] if training_args.eval_strategy != "no" else None
+        ),
+        processing_class=tokenizer,
+        data_collator=collator,
+        # compute_metrics=partial(compute_metrics, prefix=prefix),
+    )
+
+    if training_args.process_index == 0:
+
+        trainer.model.config.update(
+            {
+                "wandb_id": wandb.run.id,
+                "fold": cfg.fold,
+                "group": group,
+                "dataset": script_args.dataset_name,
+            }
+        )
+
+    if not cfg.eval_only:
+        if training_args.resume_from_checkpoint is not None:
+            os.chdir(Path(training_args.resume_from_checkpoint).parent)
+        trainer.train(resume_from_checkpoint=training_args.resume_from_checkpoint)
+    else:
+        metrics = trainer.evaluate()
+
+        # if training_args.process_index == 0:
+            # met = [x for x in metrics if "accuracy" in x][0]
+
+            # result = add_metric_to_card(
+            #     repo=training_args.hub_model_id,
+            #     metrics_pretty_name=met,
+            #     metrics_value=metrics[met],
+            #     dataset_id=script_args.dataset_name,
+            #     dataset_split=cfg.val_split_name,
+            #     model_path=model_config.model_name_or_path,
+            #     model_dtype=model_config.torch_dtype,
+            #     token=os.environ["HF_WRITE_PERSONAL"],
+            # )
+            # print(result)
+
+    if not cfg.eval_only:
+        # Save and push to hub
+        trainer.save_model(training_args.output_dir)
+        if training_args.push_to_hub:
+            trainer.push_to_hub(
+                dataset_name=script_args.dataset_name,
+                model_name=model_config.model_name_or_path,
+                tags=cfg.hub_repo_tags,
+            )
+
+    if training_args.process_index == 0:
+        wandb.finish()
+
+
+if __name__ == "__main__":
+    main()

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c21a51faff8af5ba380bad147d18caa290391847ceb6f265e3fa75693d83ca3
+size 5688

wandb/debug-internal.log

@@ -0,0 +1,173 @@
+{"time":"2025-02-03T02:28:03.696807615Z","level":"INFO","msg":"stream: starting","core version":"0.19.5","symlink path":"/workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/logs/debug-core.log"}
+{"time":"2025-02-03T02:28:03.817789182Z","level":"INFO","msg":"created new stream","id":"dcc7zgc1"}
+{"time":"2025-02-03T02:28:03.818586274Z","level":"INFO","msg":"stream: started","id":"dcc7zgc1"}
+{"time":"2025-02-03T02:28:03.81870717Z","level":"INFO","msg":"writer: Do: started","stream_id":"dcc7zgc1"}
+{"time":"2025-02-03T02:28:03.818792306Z","level":"INFO","msg":"sender: started","stream_id":"dcc7zgc1"}
+{"time":"2025-02-03T02:28:03.818796388Z","level":"INFO","msg":"handler: started","stream_id":"dcc7zgc1"}
+{"time":"2025-02-03T02:28:04.000719483Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-02-03T02:45:04.031766924Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't write proto: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:04.041356457Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.544802874Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.548768943Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.550274538Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.551655437Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.551922309Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.552153808Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.552375982Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.552603006Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.552820323Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.553020589Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.553360124Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:14.013112727Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:14.013750089Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:14.014179393Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:14.030367758Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:14.037857511Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:24.012724337Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:24.013386919Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:24.013814547Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:24.028549166Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:24.039097369Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:34.013659859Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:34.014050319Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:34.014573857Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:34.031114563Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:34.043242364Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:39.930984996Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:44.012726623Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:44.013205654Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:44.013516Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:44.027916488Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:44.03958783Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:54.013656841Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:54.01418423Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:54.014628659Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:54.033195197Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:54.042937443Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:04.013295732Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:04.013996367Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:04.014422305Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:04.021063311Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:04.030108754Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.379806513Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.383901626Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.384158258Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.384354654Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.385165228Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.385772976Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.386593259Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.386864512Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.387800361Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.38804Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.388250873Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:14.013322405Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:14.013932183Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:14.014478501Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:14.028373706Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:14.036914608Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:24.012931102Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:24.013456745Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:24.013874477Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:24.027705797Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:24.03920141Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:34.013235382Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:34.01375004Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:34.014050249Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:34.030673501Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:34.038844824Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:38.692460479Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:44.013478684Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:44.01471351Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:44.015200968Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:44.030826703Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:44.037007727Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:54.013636339Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:54.014294244Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:54.014572993Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:54.031638342Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:54.037479037Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:04.012801038Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:04.013468243Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:04.013830614Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:04.028870153Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:04.036957318Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.892257226Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.897506757Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.897737471Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.897912333Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.898303742Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.898509465Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.898683867Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.898889889Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.899150903Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.899334621Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.899582611Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:14.013409825Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:14.013867043Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:14.014431258Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:14.031729633Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:14.03618142Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:24.013464527Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:24.014071598Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:24.01456527Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:24.029978559Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:24.04195135Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:34.012987977Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:34.01347962Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:34.013848799Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:34.02958934Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:34.039466785Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:37.147661109Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:44.013538142Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:44.014060275Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:44.014476232Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:44.027713469Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:44.041885403Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:54.013491729Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:54.013973676Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:54.014421506Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:54.031116443Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:54.041253571Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:04.012871554Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:04.013442393Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:04.013901381Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:04.030888657Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:04.037548756Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.483136965Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.484408366Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.484845819Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.485095297Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.485919398Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.48619516Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.486391436Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.486757885Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.487157311Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.487486656Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.487792684Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:10.397152439Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:14.01385952Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:14.014396382Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:14.014735798Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:14.031036142Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:14.043080386Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:24.012675862Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:24.013316809Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:24.013751261Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:24.029782943Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:24.036969142Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:34.013419275Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:34.014061831Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:34.014495496Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:34.03131134Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:34.041725038Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:38.254340511Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:44.013224404Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:44.013909678Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:44.014265637Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:44.031204147Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:44.036940611Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:54.013044626Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:54.013519756Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:54.013948123Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:54.029572076Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:54.037191525Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}

wandb/debug.log

@@ -0,0 +1,25 @@
+2025-02-03 02:28:03,449 INFO    MainThread:1721 [wandb_setup.py:_flush():68] Current SDK version is 0.19.5
+2025-02-03 02:28:03,450 INFO    MainThread:1721 [wandb_setup.py:_flush():68] Configure stats pid to 1721
+2025-02-03 02:28:03,450 INFO    MainThread:1721 [wandb_setup.py:_flush():68] Loading settings from /root/.config/wandb/settings
+2025-02-03 02:28:03,451 INFO    MainThread:1721 [wandb_setup.py:_flush():68] Loading settings from /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/settings
+2025-02-03 02:28:03,451 INFO    MainThread:1721 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2025-02-03 02:28:03,453 INFO    MainThread:1721 [wandb_init.py:setup_run_log_directory():637] Logging user logs to /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/logs/debug.log
+2025-02-03 02:28:03,454 INFO    MainThread:1721 [wandb_init.py:setup_run_log_directory():638] Logging internal logs to /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/logs/debug-internal.log
+2025-02-03 02:28:03,455 INFO    MainThread:1721 [wandb_init.py:init():756] calling init triggers
+2025-02-03 02:28:03,456 INFO    MainThread:1721 [wandb_init.py:init():761] wandb.init called with sweep_config: {}
+config: {'time_start': '2025-02-03_02-28-02', 'DEBUG': False, 'debug_model': 'unsloth/Qwen2.5-7B-bnb-4bit', 'fold': 0, 'random_seed': True, 'train_on_all_folds': False, 'eval_only': False, 'merge_adapters': False, 'wandb_id': None, 'val_split_name': 'val', 'pad_token': '<pad>', 'response_template_ids': [4], 'num_proc': 20, 'hub_repo_tags': ['odesia'], 'script_args': {'dataset_name': 'nbroad/odesia-combined-dipromats-v1', 'config': None, 'gradient_checkpointing_use_reentrant': True, 'ignore_bias_buffers': False}, 'model_config': {'model_name_or_path': 'mistralai/Ministral-8B-Instruct-2410', 'torch_dtype': 'bfloat16', 'attn_implementation': 'flash_attention_2', 'use_peft': True, 'lora_r': 16, 'lora_alpha': 32, 'lora_dropout': 0.05, 'lora_target_modules': ['q_proj', 'v_proj', 'k_proj', 'o_proj', 'up_proj', 'down_proj', 'gate_proj'], 'lora_modules_to_save': None, 'lora_task_type': 'CAUSAL_LM', 'use_rslora': True, 'load_in_8bit': False, 'load_in_4bit': False, 'bnb_4bit_quant_type': 'nf4', 'use_bnb_nested_quant': True}, 'training_args': {'resume_from_checkpoint': None, 'output_dir': './', 'num_train_epochs': 2, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'warmup_ratio': 0.1, 'fp16': False, 'bf16': True, 'eval_strategy': 'no', 'save_strategy': 'steps', 'eval_steps': 40, 'save_steps': 40, 'save_total_limit': 2, 'logging_steps': 2, 'run_name': None, 'weight_decay': 0.01, 'report_to': 'wandb', 'learning_rate': 5e-05, 'metric_for_best_model': 'loss', 'greater_is_better': False, 'gradient_checkpointing': True, 'gradient_accumulation_steps': 8, 'gradient_checkpointing_kwargs': {'use_reentrant': True}, 'optim': 'adamw_torch', 'dataloader_num_workers': 1, 'seed': 18, 'max_grad_norm': 2.0, 'load_best_model_at_end': False, 'push_to_hub': True, 'hub_private_repo': True, 'lr_scheduler_type': 'cosine', 'remove_unused_columns': False, 'ddp_find_unused_parameters': False, 'use_liger_kernel': True}, '_wandb': {}}
+2025-02-03 02:28:03,456 INFO    MainThread:1721 [wandb_init.py:init():789] starting backend
+2025-02-03 02:28:03,671 INFO    MainThread:1721 [wandb_init.py:init():793] sending inform_init request
+2025-02-03 02:28:03,689 INFO    MainThread:1721 [backend.py:_multiprocessing_setup():97] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-02-03 02:28:03,690 INFO    MainThread:1721 [wandb_init.py:init():808] backend started and connected
+2025-02-03 02:28:03,696 INFO    MainThread:1721 [wandb_init.py:init():901] updated telemetry
+2025-02-03 02:28:03,817 INFO    MainThread:1721 [wandb_init.py:init():926] communicating run to backend with 90.0 second timeout
+2025-02-03 02:28:03,981 INFO    MainThread:1721 [wandb_init.py:init():984] starting run threads in backend
+2025-02-03 02:28:04,176 INFO    MainThread:1721 [wandb_run.py:_console_start():2385] atexit reg
+2025-02-03 02:28:04,177 INFO    MainThread:1721 [wandb_run.py:_redirect():2235] redirect: wrap_raw
+2025-02-03 02:28:04,178 INFO    MainThread:1721 [wandb_run.py:_redirect():2300] Wrapping output streams.
+2025-02-03 02:28:04,178 INFO    MainThread:1721 [wandb_run.py:_redirect():2325] Redirects installed.
+2025-02-03 02:28:04,181 INFO    MainThread:1721 [wandb_init.py:init():1026] run started, returning control to user process
+2025-02-03 02:28:28,617 INFO    MainThread:1721 [wandb_run.py:_config_callback():1253] config_cb None None {'peft_config': {'default': {'task_type': 'CAUSAL_LM', 'peft_type': <PeftType.LORA: 'LORA'>, 'auto_mapping': None, 'base_model_name_or_path': 'mistralai/Ministral-8B-Instruct-2410', 'revision': None, 'inference_mode': False, 'r': 16, 'target_modules': {'q_proj', 'k_proj', 'o_proj', 'down_proj', 'v_proj', 'gate_proj', 'up_proj'}, 'exclude_modules': None, 'lora_alpha': 32, 'lora_dropout': 0.05, 'fan_in_fan_out': False, 'bias': 'none', 'use_rslora': True, 'modules_to_save': None, 'init_lora_weights': True, 'layers_to_transform': None, 'layers_pattern': None, 'rank_pattern': {}, 'alpha_pattern': {}, 'megatron_config': None, 'megatron_core': 'megatron.core', 'loftq_config': {}, 'eva_config': None, 'use_dora': False, 'layer_replication': None, 'runtime_config': {'ephemeral_gpu_offload': False}, 'lora_bias': False}}, 'vocab_size': 131072, 'max_position_embeddings': 32768, 'hidden_size': 4096, 'intermediate_size': 12288, 'num_hidden_layers': 36, 'num_attention_heads': 32, 'sliding_window': 32768, 'head_dim': 128, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'use_cache': False, 'rope_theta': 100000000.0, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['MistralForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': None, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'mistralai/Ministral-8B-Instruct-2410', '_attn_implementation_autoset': True, 'transformers_version': '4.48.2', 'model_type': 'mistral', 'wandb_id': 'dcc7zgc1', 'fold': 0, 'group': 'dipromats', 'dataset': 'nbroad/odesia-combined-dipromats-v1', 'output_dir': './', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 8, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 2.0, 'num_train_epochs': 2, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.1, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './runs/Feb03_02-28-02_465247ce1ab7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 2, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 40, 'save_total_limit': 2, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 18, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 40, 'dataloader_num_workers': 1, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': 'loss', 'greater_is_better': False, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 'nbroad/nbroad-odesia-dipromats-dcc7zgc1', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': True, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': {'use_reentrant': True}, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': True, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'dataset_text_field': 'text', 'packing': False, 'max_seq_length': 1024, 'dataset_num_proc': None, 'dataset_batch_size': 1000, 'model_init_kwargs': None, 'dataset_kwargs': {}, 'eval_packing': None, 'num_of_sequences': 1024, 'chars_per_token': '<CHARS_PER_TOKEN>', 'use_liger': False}
+2025-02-03 02:28:28,625 INFO    MainThread:1721 [wandb_config.py:__setitem__():154] config set model/num_parameters = 8063455232 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7f6ecf4e54d0>>
+2025-02-03 02:28:28,625 INFO    MainThread:1721 [wandb_run.py:_config_callback():1253] config_cb model/num_parameters 8063455232 None

wandb/run-20250203_022803-dcc7zgc1/files/output.log

@@ -0,0 +1,47 @@
+Applied Liger kernels to Qwen2
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00,  6.50it/s]
+You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama_fast.LlamaTokenizerFast'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message.
+README.md: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 220/220 [00:00<00:00, 489kB/s]
+train_both.parquet: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7.34M/7.34M [00:00<00:00, 41.0MB/s]
+val_both.parquet: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2.15M/2.15M [00:00<00:00, 46.2MB/s]
+Generating train split: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9792/9792 [00:00<00:00, 123735.03 examples/s]
+Generating val split: 100%|████████████████████████████████████████████████████████████████████���███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2906/2906 [00:00<00:00, 130539.97 examples/s]
+Map (num_proc=20): 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9792/9792 [00:06<00:00, 1539.40 examples/s]
+Map (num_proc=20): 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2906/2906 [00:05<00:00, 546.87 examples/s]
+/usr/local/lib/python3.11/dist-packages/trl/trainer/sft_trainer.py:300: UserWarning: You passed a processing_class with `padding_side` not equal to `right` to the SFTTrainer. This might lead to some unexpected behaviour due to overflow issues when training a model in half-precision. You might consider adding `processing_class.padding_side = 'right'` to your code.
+  warnings.warn(
+[2025-02-03 02:28:22,500][accelerate.utils.other][WARNING] - Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.
+The model is not an instance of PreTrainedModel. No liger kernels will be applied.
+wandb: WARNING The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.
+  0%|                                                                                                                                                                                                                                                                                             | 0/152 [00:00<?, ?it/s]/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
+  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]
+/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
+  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]
+/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
+  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]
+/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
+  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]
+ 26%|████████████████████████████████████████████████████████████████████████▋                                                                                                                                                                                                           | 40/152 [19:37<54:44, 29.32s/it]/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
+{'loss': 3.4803, 'grad_norm': 70.98416137695312, 'learning_rate': 6.25e-06, 'epoch': 0.03}
+{'loss': 3.2992, 'grad_norm': 54.05496597290039, 'learning_rate': 1.25e-05, 'epoch': 0.05}
+{'loss': 1.5778, 'grad_norm': 59.53104782104492, 'learning_rate': 1.8750000000000002e-05, 'epoch': 0.08}
+{'loss': 0.275, 'grad_norm': 10.362302780151367, 'learning_rate': 2.5e-05, 'epoch': 0.1}
+{'loss': 0.1436, 'grad_norm': 5.163769721984863, 'learning_rate': 3.125e-05, 'epoch': 0.13}
+{'loss': 0.1139, 'grad_norm': 1.2144449949264526, 'learning_rate': 3.7500000000000003e-05, 'epoch': 0.16}
+{'loss': 0.1296, 'grad_norm': 3.4494385719299316, 'learning_rate': 4.375e-05, 'epoch': 0.18}
+{'loss': 0.1316, 'grad_norm': 5.409619331359863, 'learning_rate': 5e-05, 'epoch': 0.21}
+{'loss': 0.0927, 'grad_norm': 3.423051118850708, 'learning_rate': 4.997332437005931e-05, 'epoch': 0.24}
+{'loss': 0.1238, 'grad_norm': 1.5564866065979004, 'learning_rate': 4.989335440737586e-05, 'epoch': 0.26}
+{'loss': 0.092, 'grad_norm': 2.6399052143096924, 'learning_rate': 4.976026077188013e-05, 'epoch': 0.29}
+{'loss': 0.096, 'grad_norm': 1.9473035335540771, 'learning_rate': 4.957432749209755e-05, 'epoch': 0.31}
+{'loss': 0.0695, 'grad_norm': 1.3038264513015747, 'learning_rate': 4.933595135901732e-05, 'epoch': 0.34}
+{'loss': 0.0716, 'grad_norm': 2.1920876502990723, 'learning_rate': 4.9045641079320484e-05, 'epoch': 0.37}
+{'loss': 0.0889, 'grad_norm': 1.7892296314239502, 'learning_rate': 4.870401618977415e-05, 'epoch': 0.39}
+{'loss': 0.0866, 'grad_norm': 1.5832806825637817, 'learning_rate': 4.8311805735108894e-05, 'epoch': 0.42}
+{'loss': 0.0828, 'grad_norm': 1.4342199563980103, 'learning_rate': 4.786984671220053e-05, 'epoch': 0.44}
+{'loss': 0.0833, 'grad_norm': 1.1282280683517456, 'learning_rate': 4.7379082283876566e-05, 'epoch': 0.47}
+{'loss': 0.0844, 'grad_norm': 2.0329487323760986, 'learning_rate': 4.684055976615924e-05, 'epoch': 0.5}
+{'loss': 0.0606, 'grad_norm': 0.9544902443885803, 'learning_rate': 4.625542839324036e-05, 'epoch': 0.52}
+  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]
+ 27%|██████████████████████████████████████████████████████████████████████████▍                                                                                                                                                                                                         | 41/152 [20:09<55:36, 30.06s/it]

wandb/run-20250203_022803-dcc7zgc1/files/requirements.txt

@@ -0,0 +1,213 @@
+pytz==2025.1
+antlr4-python3-runtime==4.9.3
+xxhash==3.5.0
+tzdata==2025.1
+einops==0.8.0
+typing_extensions==4.12.2
+tqdm==4.67.1
+smmap==5.0.2
+setproctitle==1.3.4
+sentry-sdk==2.20.0
+safetensors==0.5.2
+regex==2024.11.6
+python-dotenv==1.0.1
+pyarrow==19.0.0
+protobuf==5.29.3
+propcache==0.2.1
+omegaconf==2.3.0
+multidict==6.1.0
+mdurl==0.1.2
+hf_transfer==0.1.9
+frozenlist==1.5.0
+docker-pycreds==0.4.0
+dill==0.3.8
+click==8.1.8
+annotated-types==0.7.0
+aiohappyeyeballs==2.4.4
+yarl==1.18.3
+pydantic_core==2.27.2
+pandas==2.2.3
+multiprocess==0.70.16
+markdown-it-py==3.0.0
+hydra-core==1.3.2
+huggingface-hub==0.28.1
+gitdb==4.0.12
+aiosignal==1.3.2
+tokenizers==0.21.0
+rich==13.9.4
+pydantic==2.10.6
+GitPython==3.1.44
+aiohttp==3.11.11
+wandb==0.19.5
+transformers==4.48.2
+accelerate==1.3.0
+peft==0.14.0
+datasets==3.2.0
+trl==0.14.0
+flash_attn==2.7.4.post1
+scipy==1.15.1
+liger_kernel==0.5.2
+entrypoints==0.4
+jupyter_client==7.4.9
+nbclassic==1.1.0
+notebook==6.5.5
+pyzmq==24.0.1
+PyYAML==6.0.2
+Send2Trash==1.8.3
+anyio==4.6.0
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==2.4.1
+async-lru==2.0.4
+attrs==24.2.0
+babel==2.16.0
+beautifulsoup4==4.12.3
+bleach==6.1.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.3.2
+comm==0.2.2
+debugpy==1.8.5
+decorator==5.1.1
+defusedxml==0.7.1
+executing==2.1.0
+fastjsonschema==2.20.0
+fqdn==1.5.1
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.2
+idna==3.10
+ipykernel==6.29.5
+ipython==8.27.0
+ipython-genutils==0.2.0
+ipywidgets==8.1.5
+isoduration==20.11.0
+jedi==0.19.1
+json5==0.9.25
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
+jupyter-archive==3.4.0
+jupyter_contrib_core==0.4.2
+jupyter_contrib_nbextensions==0.7.0
+jupyter_core==5.7.2
+jupyter-events==0.10.0
+jupyter-highlight-selected-word==0.2.0
+jupyter-lsp==2.2.5
+jupyter_nbextensions_configurator==0.6.4
+jupyter_server==2.14.2
+jupyter_server_terminals==0.5.3
+jupyterlab==4.2.5
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+jupyterlab_widgets==3.0.13
+lxml==5.3.0
+matplotlib-inline==0.1.7
+mistune==3.0.2
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+notebook_shim==0.2.4
+overrides==7.7.0
+packaging==24.1
+pandocfilters==1.5.1
+parso==0.8.4
+pexpect==4.9.0
+platformdirs==4.3.6
+prometheus_client==0.21.0
+prompt_toolkit==3.0.47
+psutil==6.0.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pycparser==2.22
+Pygments==2.18.0
+python-dateutil==2.9.0.post0
+python-json-logger==2.0.7
+referencing==0.35.1
+requests==2.32.3
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rpds-py==0.20.0
+sniffio==1.3.1
+soupsieve==2.6
+stack-data==0.6.3
+terminado==0.18.1
+tinycss2==1.3.0
+tornado==6.4.1
+traitlets==5.14.3
+types-python-dateutil==2.9.0.20240906
+uri-template==1.3.0
+urllib3==2.2.3
+wcwidth==0.2.13
+webcolors==24.8.0
+webencodings==0.5.1
+websocket-client==1.8.0
+widgetsnbextension==4.0.13
+Jinja2==3.1.3
+MarkupSafe==2.1.5
+filelock==3.13.1
+fsspec==2024.2.0
+mpmath==1.3.0
+networkx==3.2.1
+numpy==1.26.3
+nvidia-cublas-cu12==12.4.2.65
+nvidia-cuda-cupti-cu12==12.4.99
+nvidia-cuda-nvrtc-cu12==12.4.99
+nvidia-cuda-runtime-cu12==12.4.99
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.2.0.44
+nvidia-curand-cu12==10.3.5.119
+nvidia-cusolver-cu12==11.6.0.99
+nvidia-cusparse-cu12==12.3.0.142
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.4.99
+nvidia-nvtx-cu12==12.4.99
+pillow==10.2.0
+sympy==1.12
+torch==2.4.1+cu124
+torchaudio==2.4.1+cu124
+torchvision==0.19.1+cu124
+triton==3.0.0
+pip==24.2
+setuptools==75.1.0
+wheel==0.44.0
+PyGObject==3.42.1
+PyJWT==2.3.0
+SecretStorage==3.3.1
+blinker==1.4
+cryptography==3.4.8
+dbus-python==1.2.18
+distro==1.7.0
+httplib2==0.20.2
+importlib-metadata==4.6.4
+jeepney==0.7.1
+keyring==23.5.0
+launchpadlib==1.10.16
+lazr.restfulclient==0.14.4
+lazr.uri==1.0.6
+more-itertools==8.10.0
+oauthlib==3.2.0
+pyparsing==2.4.7
+python-apt==2.4.0+ubuntu4
+six==1.16.0
+wadllib==1.3.6
+zipp==1.0.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+importlib_resources==6.4.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.1
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.43.0
+zipp==3.19.2

wandb/run-20250203_022803-dcc7zgc1/files/wandb-metadata.json

@@ -0,0 +1,63 @@
+{
+  "os": "Linux-5.4.0-196-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.10",
+  "startedAt": "2025-02-03T02:28:03.692281Z",
+  "args": [
+    "-cn",
+    "d"
+  ],
+  "program": "/workspace/odesia-2025/train/train.py",
+  "codePath": "train/train.py",
+  "git": {
+    "remote": "https://github.com/nbroad1881/odesia-2025.git",
+    "commit": "f2adeec8159f2367bf3180060491af1f4eef3ac7"
+  },
+  "email": "[email protected]",
+  "root": "/workspace/odesia-2025/train/outputs/2025-02-03/02-28-02",
+  "host": "465247ce1ab7",
+  "executable": "/usr/bin/python",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA A40",
+  "gpu_count": 4,
+  "disk": {
+    "/": {
+      "total": "214748364800",
+      "used": "1626206208"
+    }
+  },
+  "memory": {
+    "total": "540662591488"
+  },
+  "cpu": {
+    "count": 48,
+    "countLogical": 96
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A40",
+      "memoryTotal": "48305799168",
+      "cudaCores": 10752,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A40",
+      "memoryTotal": "48305799168",
+      "cudaCores": 10752,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A40",
+      "memoryTotal": "48305799168",
+      "cudaCores": 10752,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A40",
+      "memoryTotal": "48305799168",
+      "cudaCores": 10752,
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.7"
+}

wandb/run-20250203_022803-dcc7zgc1/logs/debug-core.log

@@ -0,0 +1,6 @@
+{"time":"2025-02-03T02:28:03.489836232Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpfzdb6ya0/port-1721.txt","pid":1721,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-02-03T02:28:03.490826132Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":1721}
+{"time":"2025-02-03T02:28:03.490839232Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35973,"Zone":""}}
+{"time":"2025-02-03T02:28:03.67160485Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:47978"}
+{"time":"2025-02-03T02:28:03.694550806Z","level":"INFO","msg":"handleInformInit: received","streamId":"dcc7zgc1","id":"127.0.0.1:47978"}
+{"time":"2025-02-03T02:28:03.819359144Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"dcc7zgc1","id":"127.0.0.1:47978"}

wandb/run-20250203_022803-dcc7zgc1/logs/debug-internal.log

@@ -0,0 +1,173 @@
+{"time":"2025-02-03T02:28:03.696807615Z","level":"INFO","msg":"stream: starting","core version":"0.19.5","symlink path":"/workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/logs/debug-core.log"}
+{"time":"2025-02-03T02:28:03.817789182Z","level":"INFO","msg":"created new stream","id":"dcc7zgc1"}
+{"time":"2025-02-03T02:28:03.818586274Z","level":"INFO","msg":"stream: started","id":"dcc7zgc1"}
+{"time":"2025-02-03T02:28:03.81870717Z","level":"INFO","msg":"writer: Do: started","stream_id":"dcc7zgc1"}
+{"time":"2025-02-03T02:28:03.818792306Z","level":"INFO","msg":"sender: started","stream_id":"dcc7zgc1"}
+{"time":"2025-02-03T02:28:03.818796388Z","level":"INFO","msg":"handler: started","stream_id":"dcc7zgc1"}
+{"time":"2025-02-03T02:28:04.000719483Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-02-03T02:45:04.031766924Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't write proto: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:04.041356457Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.544802874Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.548768943Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.550274538Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.551655437Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.551922309Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.552153808Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.552375982Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.552603006Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.552820323Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.553020589Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:10.553360124Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:14.013112727Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:14.013750089Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:14.014179393Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:14.030367758Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:14.037857511Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:24.012724337Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:24.013386919Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:24.013814547Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:24.028549166Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:24.039097369Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:34.013659859Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:34.014050319Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:34.014573857Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:34.031114563Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:34.043242364Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:39.930984996Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:44.012726623Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:44.013205654Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:44.013516Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:44.027916488Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:44.03958783Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:54.013656841Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:54.01418423Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:54.014628659Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:54.033195197Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:45:54.042937443Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:04.013295732Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:04.013996367Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:04.014422305Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:04.021063311Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:04.030108754Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.379806513Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.383901626Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.384158258Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.384354654Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.385165228Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.385772976Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.386593259Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.386864512Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.387800361Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.38804Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:09.388250873Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:14.013322405Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:14.013932183Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:14.014478501Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:14.028373706Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:14.036914608Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:24.012931102Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:24.013456745Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:24.013874477Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:24.027705797Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:24.03920141Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:34.013235382Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:34.01375004Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:34.014050249Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:34.030673501Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:34.038844824Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:38.692460479Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:44.013478684Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:44.01471351Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:44.015200968Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:44.030826703Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:44.037007727Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:54.013636339Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:54.014294244Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:54.014572993Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:54.031638342Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:46:54.037479037Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:04.012801038Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:04.013468243Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:04.013830614Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:04.028870153Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:04.036957318Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.892257226Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.897506757Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.897737471Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.897912333Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.898303742Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.898509465Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.898683867Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.898889889Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.899150903Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.899334621Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:07.899582611Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:14.013409825Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:14.013867043Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:14.014431258Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:14.031729633Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:14.03618142Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:24.013464527Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:24.014071598Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:24.01456527Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:24.029978559Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:24.04195135Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:34.012987977Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:34.01347962Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:34.013848799Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:34.02958934Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:34.039466785Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:37.147661109Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:44.013538142Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:44.014060275Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:44.014476232Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:44.027713469Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:44.041885403Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:54.013491729Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:54.013973676Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:54.014421506Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:54.031116443Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:47:54.041253571Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:04.012871554Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:04.013442393Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:04.013901381Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:04.030888657Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:04.037548756Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.483136965Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.484408366Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.484845819Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.485095297Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.485919398Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.48619516Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.486391436Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.486757885Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.487157311Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.487486656Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:06.487792684Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:10.397152439Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:14.01385952Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:14.014396382Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:14.014735798Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:14.031036142Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:14.043080386Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:24.012675862Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:24.013316809Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:24.013751261Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:24.029782943Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:24.036969142Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:34.013419275Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:34.014061831Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:34.014495496Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:34.03131134Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:34.041725038Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:38.254340511Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:44.013224404Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:44.013909678Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:44.014265637Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:44.031204147Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:44.036940611Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:54.013044626Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:54.013519756Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:54.013948123Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:54.029572076Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}
+{"time":"2025-02-03T02:48:54.037191525Z","level":"ERROR","msg":"writer: startStore: error storing record: store: can't get next record: write /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb: input/output error"}

wandb/run-20250203_022803-dcc7zgc1/logs/debug.log

@@ -0,0 +1,25 @@
+2025-02-03 02:28:03,449 INFO    MainThread:1721 [wandb_setup.py:_flush():68] Current SDK version is 0.19.5
+2025-02-03 02:28:03,450 INFO    MainThread:1721 [wandb_setup.py:_flush():68] Configure stats pid to 1721
+2025-02-03 02:28:03,450 INFO    MainThread:1721 [wandb_setup.py:_flush():68] Loading settings from /root/.config/wandb/settings
+2025-02-03 02:28:03,451 INFO    MainThread:1721 [wandb_setup.py:_flush():68] Loading settings from /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/settings
+2025-02-03 02:28:03,451 INFO    MainThread:1721 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2025-02-03 02:28:03,453 INFO    MainThread:1721 [wandb_init.py:setup_run_log_directory():637] Logging user logs to /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/logs/debug.log
+2025-02-03 02:28:03,454 INFO    MainThread:1721 [wandb_init.py:setup_run_log_directory():638] Logging internal logs to /workspace/odesia-2025/train/outputs/2025-02-03/02-28-02/wandb/run-20250203_022803-dcc7zgc1/logs/debug-internal.log
+2025-02-03 02:28:03,455 INFO    MainThread:1721 [wandb_init.py:init():756] calling init triggers
+2025-02-03 02:28:03,456 INFO    MainThread:1721 [wandb_init.py:init():761] wandb.init called with sweep_config: {}
+config: {'time_start': '2025-02-03_02-28-02', 'DEBUG': False, 'debug_model': 'unsloth/Qwen2.5-7B-bnb-4bit', 'fold': 0, 'random_seed': True, 'train_on_all_folds': False, 'eval_only': False, 'merge_adapters': False, 'wandb_id': None, 'val_split_name': 'val', 'pad_token': '<pad>', 'response_template_ids': [4], 'num_proc': 20, 'hub_repo_tags': ['odesia'], 'script_args': {'dataset_name': 'nbroad/odesia-combined-dipromats-v1', 'config': None, 'gradient_checkpointing_use_reentrant': True, 'ignore_bias_buffers': False}, 'model_config': {'model_name_or_path': 'mistralai/Ministral-8B-Instruct-2410', 'torch_dtype': 'bfloat16', 'attn_implementation': 'flash_attention_2', 'use_peft': True, 'lora_r': 16, 'lora_alpha': 32, 'lora_dropout': 0.05, 'lora_target_modules': ['q_proj', 'v_proj', 'k_proj', 'o_proj', 'up_proj', 'down_proj', 'gate_proj'], 'lora_modules_to_save': None, 'lora_task_type': 'CAUSAL_LM', 'use_rslora': True, 'load_in_8bit': False, 'load_in_4bit': False, 'bnb_4bit_quant_type': 'nf4', 'use_bnb_nested_quant': True}, 'training_args': {'resume_from_checkpoint': None, 'output_dir': './', 'num_train_epochs': 2, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'warmup_ratio': 0.1, 'fp16': False, 'bf16': True, 'eval_strategy': 'no', 'save_strategy': 'steps', 'eval_steps': 40, 'save_steps': 40, 'save_total_limit': 2, 'logging_steps': 2, 'run_name': None, 'weight_decay': 0.01, 'report_to': 'wandb', 'learning_rate': 5e-05, 'metric_for_best_model': 'loss', 'greater_is_better': False, 'gradient_checkpointing': True, 'gradient_accumulation_steps': 8, 'gradient_checkpointing_kwargs': {'use_reentrant': True}, 'optim': 'adamw_torch', 'dataloader_num_workers': 1, 'seed': 18, 'max_grad_norm': 2.0, 'load_best_model_at_end': False, 'push_to_hub': True, 'hub_private_repo': True, 'lr_scheduler_type': 'cosine', 'remove_unused_columns': False, 'ddp_find_unused_parameters': False, 'use_liger_kernel': True}, '_wandb': {}}
+2025-02-03 02:28:03,456 INFO    MainThread:1721 [wandb_init.py:init():789] starting backend
+2025-02-03 02:28:03,671 INFO    MainThread:1721 [wandb_init.py:init():793] sending inform_init request
+2025-02-03 02:28:03,689 INFO    MainThread:1721 [backend.py:_multiprocessing_setup():97] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-02-03 02:28:03,690 INFO    MainThread:1721 [wandb_init.py:init():808] backend started and connected
+2025-02-03 02:28:03,696 INFO    MainThread:1721 [wandb_init.py:init():901] updated telemetry
+2025-02-03 02:28:03,817 INFO    MainThread:1721 [wandb_init.py:init():926] communicating run to backend with 90.0 second timeout
+2025-02-03 02:28:03,981 INFO    MainThread:1721 [wandb_init.py:init():984] starting run threads in backend
+2025-02-03 02:28:04,176 INFO    MainThread:1721 [wandb_run.py:_console_start():2385] atexit reg
+2025-02-03 02:28:04,177 INFO    MainThread:1721 [wandb_run.py:_redirect():2235] redirect: wrap_raw
+2025-02-03 02:28:04,178 INFO    MainThread:1721 [wandb_run.py:_redirect():2300] Wrapping output streams.
+2025-02-03 02:28:04,178 INFO    MainThread:1721 [wandb_run.py:_redirect():2325] Redirects installed.
+2025-02-03 02:28:04,181 INFO    MainThread:1721 [wandb_init.py:init():1026] run started, returning control to user process
+2025-02-03 02:28:28,617 INFO    MainThread:1721 [wandb_run.py:_config_callback():1253] config_cb None None {'peft_config': {'default': {'task_type': 'CAUSAL_LM', 'peft_type': <PeftType.LORA: 'LORA'>, 'auto_mapping': None, 'base_model_name_or_path': 'mistralai/Ministral-8B-Instruct-2410', 'revision': None, 'inference_mode': False, 'r': 16, 'target_modules': {'q_proj', 'k_proj', 'o_proj', 'down_proj', 'v_proj', 'gate_proj', 'up_proj'}, 'exclude_modules': None, 'lora_alpha': 32, 'lora_dropout': 0.05, 'fan_in_fan_out': False, 'bias': 'none', 'use_rslora': True, 'modules_to_save': None, 'init_lora_weights': True, 'layers_to_transform': None, 'layers_pattern': None, 'rank_pattern': {}, 'alpha_pattern': {}, 'megatron_config': None, 'megatron_core': 'megatron.core', 'loftq_config': {}, 'eva_config': None, 'use_dora': False, 'layer_replication': None, 'runtime_config': {'ephemeral_gpu_offload': False}, 'lora_bias': False}}, 'vocab_size': 131072, 'max_position_embeddings': 32768, 'hidden_size': 4096, 'intermediate_size': 12288, 'num_hidden_layers': 36, 'num_attention_heads': 32, 'sliding_window': 32768, 'head_dim': 128, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'use_cache': False, 'rope_theta': 100000000.0, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['MistralForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': None, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'mistralai/Ministral-8B-Instruct-2410', '_attn_implementation_autoset': True, 'transformers_version': '4.48.2', 'model_type': 'mistral', 'wandb_id': 'dcc7zgc1', 'fold': 0, 'group': 'dipromats', 'dataset': 'nbroad/odesia-combined-dipromats-v1', 'output_dir': './', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 8, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 2.0, 'num_train_epochs': 2, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.1, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './runs/Feb03_02-28-02_465247ce1ab7', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 2, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 40, 'save_total_limit': 2, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 18, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 40, 'dataloader_num_workers': 1, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': 'loss', 'greater_is_better': False, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': False, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 'nbroad/nbroad-odesia-dipromats-dcc7zgc1', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': True, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': {'use_reentrant': True}, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': True, 'eval_use_gather_object': False, 'average_tokens_across_devices': False, 'dataset_text_field': 'text', 'packing': False, 'max_seq_length': 1024, 'dataset_num_proc': None, 'dataset_batch_size': 1000, 'model_init_kwargs': None, 'dataset_kwargs': {}, 'eval_packing': None, 'num_of_sequences': 1024, 'chars_per_token': '<CHARS_PER_TOKEN>', 'use_liger': False}
+2025-02-03 02:28:28,625 INFO    MainThread:1721 [wandb_config.py:__setitem__():154] config set model/num_parameters = 8063455232 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7f6ecf4e54d0>>
+2025-02-03 02:28:28,625 INFO    MainThread:1721 [wandb_run.py:_config_callback():1253] config_cb model/num_parameters 8063455232 None

wandb/run-20250203_022803-dcc7zgc1/run-dcc7zgc1.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00080771a880eb05a615cc361766e1c5fe5fa16d2a6a9c3f60c7dcb8022f5639
+size 294912