Added challenge inference script

Dockerfile

@@ -0,0 +1,22 @@
+FROM nvcr.io/nvidia/pytorch:22.12-py3
+# FROM python:3.8.10-slim
+# alternatively use python image as a base image, if PyTorch with GPU drivers is not needed
+
+#COPY requirements.txt /script/requirements.txt
+#COPY predict.py /script/predict.py
+#COPY evaluate.py /script/evaluate.py
+#COPY start.sh /script/start.sh
+#COPY ./FungiCLEF2023-ViT_base_patch16_224-100E.pth /script/FungiCLEF2023-ViT_base_patch16_224-100E.pth
+
+COPY . /script/
+
+# install python dependencies
+ENV SCRIPT_DIR='/script'
+WORKDIR $SCRIPT_DIR
+RUN pip install --no-cache-dir --upgrade pip build && \
+    pip install --no-cache-dir --compile -r requirements.txt && \
+    mim install "mmpretrain==1.0.0rc7" && \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/* /tmp/* /var/tmp/*
+
+# run script
+CMD bash start.sh

docker-compose.yml

@@ -0,0 +1,24 @@
+version: "3.8"
+
+services:
+  fungiclef:
+    image: fungiclef-example:latest
+    build: .
+    container_name: fungiclef-example
+    volumes:
+    - /media/Data-10T-1/Data/DF21_300:/Data
+    # settings related to nvcr.io/nvidia/pytorch:22.12-py3 docker image
+    ipc: "host"
+    ulimits:
+      memlock: -1
+      stack: 67108864
+    deploy:
+      resources:
+        limits:
+          cpus: "4.0"
+          memory: "12g"
+        reservations:
+          devices:
+            - capabilities: [ "gpu" ]
+              driver: nvidia
+              count: all

evaluate.py

@@ -0,0 +1,211 @@
+from typing import List
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import f1_score
+
+COLUMNS = ["observationID", "class_id"]
+poisonous_lvl = pd.read_csv(
+    "http://ptak.felk.cvut.cz/plants//DanishFungiDataset/poison_status_list.csv"
+)
+POISONOUS_SPECIES = poisonous_lvl[poisonous_lvl["poisonous"] == 1].class_id.unique()
+
+
+def classification_error_with_unknown(
+    merged_df, cost_unkwnown_misclassified=10, cost_misclassified_as_unknown=0.1
+):
+    num_misclassified_unknown = sum((merged_df.class_id_gt == -1) & (merged_df.class_id_pred != -1))
+    num_misclassified_as_unknown = sum(
+        (merged_df.class_id_gt != -1) & (merged_df.class_id_pred == -1)
+    )
+    num_misclassified_other = sum(
+        (merged_df.class_id_gt != merged_df.class_id_pred)
+        & (merged_df.class_id_pred != -1)
+        & (merged_df.class_id_gt != -1)
+    )
+    return (
+        num_misclassified_other
+        + num_misclassified_unknown * cost_unkwnown_misclassified
+        + num_misclassified_as_unknown * cost_misclassified_as_unknown
+    ) / len(merged_df)
+
+
+def classification_error(merged_df):
+    return classification_error_with_unknown(
+        merged_df, cost_misclassified_as_unknown=1, cost_unkwnown_misclassified=1
+    )
+
+
+def num_psc_decisions(merged_df):
+    # Number of observations that were misclassified as edible, when in fact they are poisonous
+    num_psc = sum(
+        merged_df.class_id_gt.isin(POISONOUS_SPECIES)
+        & ~merged_df.class_id_pred.isin(POISONOUS_SPECIES)
+    )
+    return num_psc
+
+
+def num_esc_decisions(merged_df):
+    # Number of observations that were misclassified as poisonus, when in fact they are edible
+    num_esc = sum(
+        ~merged_df.class_id_gt.isin(POISONOUS_SPECIES)
+        & merged_df.class_id_pred.isin(POISONOUS_SPECIES)
+    )
+    return num_esc
+
+
+def psc_esc_cost_score(merged_df, cost_psc=100, cost_esc=1):
+    return (
+        cost_psc * num_psc_decisions(merged_df) + cost_esc * num_esc_decisions(merged_df)
+    ) / len(merged_df)
+
+
+def evaluate_csv(test_annotation_file: str, user_submission_file: str) -> List[dict]:
+    # load gt annotations
+    gt_df = pd.read_csv(test_annotation_file, sep=",")
+    for col in COLUMNS:
+        assert col in gt_df, f"Test annotation file is missing column '{col}'."
+    # keep only observation-based predictions
+    gt_df = gt_df.drop_duplicates("observationID")
+
+    # load user predictions
+    try:
+        is_tsv = user_submission_file.endswith(".tsv")
+        user_pred_df = pd.read_csv(user_submission_file, sep="\t" if is_tsv else ",")
+    except Exception:
+        print("Could not read file submitted by the user.")
+        raise ValueError("Could not read file submitted by the user.")
+
+    # validate user predictions
+    for col in COLUMNS:
+        if col not in user_pred_df:
+            print(f"File submitted by the user is missing column '{col}'.")
+            raise ValueError(f"File submitted by the user is missing column '{col}'.")
+    if len(gt_df) != len(user_pred_df):
+        print(f"File submitted by the user should have {len(gt_df)} records.")
+        raise ValueError(f"File submitted by the user should have {len(gt_df)} records.")
+    missing_obs = gt_df.loc[
+        ~gt_df["observationID"].isin(user_pred_df["observationID"]),
+        "observationID",
+    ]
+    if len(missing_obs) > 0:
+        if len(missing_obs) > 3:
+            missing_obs_str = ", ".join(missing_obs.iloc[:3].astype(str)) + ", ..."
+        else:
+            missing_obs_str = ", ".join(missing_obs.astype(str))
+        print(f"File submitted by the user is missing observations: {missing_obs_str}")
+        raise ValueError(f"File submitted by the user is missing observations: {missing_obs_str}")
+
+    # merge dataframes
+    merged_df = pd.merge(
+        gt_df,
+        user_pred_df,
+        how="outer",
+        on="observationID",
+        validate="one_to_one",
+        suffixes=("_gt", "_pred"),
+    )
+
+    # evaluate accuracy_score and f1_score
+    cls_error = classification_error(merged_df)
+    cls_error_with_unknown = classification_error_with_unknown(merged_df)
+    psc_esc_cost = psc_esc_cost_score(merged_df)
+
+    result = [
+        {
+            "test_split": {
+                "F1 Score": np.round(
+                    f1_score(merged_df["class_id_gt"], merged_df["class_id_pred"], average="macro")
+                    * 100,
+                    2,
+                ),
+                "Track 1: Classification Error": np.round(cls_error, 4),
+                "Track 2: Cost for Poisonousness Confusion": np.round(psc_esc_cost, 4),
+                "Track 3: User-Focused Loss": np.round(cls_error + psc_esc_cost, 4),
+                "Track 4: Classification Error with Special Cost for Unknown": np.round(
+                    cls_error_with_unknown, 4
+                ),
+            }
+        }
+    ]
+
+    print(f"Evaluated scores: {result[0]['test_split']}")
+
+    return result
+
+
+def evaluate(test_annotation_file, user_submission_file, phase_codename, **kwargs):
+    """
+    Evaluates the submission for a particular challenge phase and returns score
+    Arguments:
+
+        `test_annotations_file`: Path to test_annotation_file on the server
+        `user_submission_file`: Path to file submitted by the user
+        `phase_codename`: Phase to which submission is made
+
+        `**kwargs`: keyword arguments that contains additional submission
+        metadata that challenge hosts can use to send slack notification.
+        You can access the submission metadata
+        with kwargs['submission_metadata']
+
+        Example: A sample submission metadata can be accessed like this:
+        >>> print(kwargs['submission_metadata'])
+        {
+            'status': u'running',
+            'when_made_public': None,
+            'participant_team': 5,
+            'input_file': 'https://abc.xyz/path/to/submission/file.json',
+            'execution_time': u'123',
+            'publication_url': u'ABC',
+            'challenge_phase': 1,
+            'created_by': u'ABC',
+            'stdout_file': 'https://abc.xyz/path/to/stdout/file.json',
+            'method_name': u'Test',
+            'stderr_file': 'https://abc.xyz/path/to/stderr/file.json',
+            'participant_team_name': u'Test Team',
+            'project_url': u'http://foo.bar',
+            'method_description': u'ABC',
+            'is_public': False,
+            'submission_result_file': 'https://abc.xyz/path/result/file.json',
+            'id': 123,
+            'submitted_at': u'2017-03-20T19:22:03.880652Z'
+        }
+    """
+    print("Starting Evaluation.....")
+    out = {}
+    if phase_codename == "prediction-based":
+        print("Evaluating for Prediction-based Phase")
+        out["result"] = evaluate_csv(test_annotation_file, user_submission_file)
+
+        # To display the results in the result file
+        out["submission_result"] = out["result"][0]["test_split"]
+        print("Completed evaluation")
+    return out
+
+
+if __name__ == "__main__":
+    import argparse
+    import json
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--test-annotation-file",
+        help="Path to test_annotation_file on the server.",
+        type=str,
+        required=True,
+    )
+    parser.add_argument(
+        "--user-submission-file",
+        help="Path to a file created by predict script.",
+        type=str,
+        required=True,
+    )
+    args = parser.parse_args()
+
+    result = evaluate(
+        test_annotation_file=args.test_annotation_file,
+        user_submission_file=args.user_submission_file,
+        phase_codename="prediction-based",
+    )
+    with open("scores.json", "w") as f:
+        json.dump(result, f)

predict.py

@@ -0,0 +1,70 @@
+import os
+import os.path as osp
+import subprocess
+import sys
+from pathlib import Path
+
+
+# custom script arguments
+CONFIG_PATH = 'work_dirs/swin_base_b32x4-fp16_fungi+val_res_384_cb_epochs_6/swin_base_b32x4-fp16_fungi+val-test_res_384_cb_epochs_6.py'
+CHECKPOINT_PATH = "work_dirs/swin_base_b32x4-fp16_fungi+val_res_384_cb_epochs_6/epoch_6.pth"
+SCORE_THRESHOLD = 0.2
+
+
+def run_inference(input_csv, output_csv, data_root_path):
+    """Load model and dataloader and run inference."""
+
+    if not data_root_path.endswith('/'):
+        data_root_path += '/'
+    data_cfg_opts = [
+        f'test_dataloader.dataset.data_root=',
+        f'test_dataloader.dataset.ann_file={input_csv}',
+        f'test_dataloader.dataset.data_prefix={data_root_path}']
+
+    inference = subprocess.Popen([
+        'python', '-m',
+        'tools.test_generate_result_pre-consensus',
+        CONFIG_PATH, CHECKPOINT_PATH,
+        output_csv,
+        '--threshold', str(SCORE_THRESHOLD),
+        '--no-scores',
+        '--cfg-options'] + data_cfg_opts)
+    return_code = inference.wait()
+    if return_code != 0:
+        print(f'Inference crashed with exit code {return_code}')
+        sys.exit(return_code)
+    print(f'Written {output_csv}')
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--input-file",
+        help="Path to a file with observation ids and image paths.",
+        type=str,
+        required=True,
+    )
+    parser.add_argument(
+        "--data-root-path",
+        help="Path to a directory where images are stored.",
+        type=str,
+        required=True,
+    )
+    parser.add_argument(
+        "--output-file",
+        help="Path to a file where predict script will store predictions.",
+        type=str,
+        required=True,
+    )
+    args = parser.parse_args()
+
+    output_csv = os.path.basename(args.output_file)
+    if not output_csv.endswith(".csv"):
+        output_csv = output_csv + ".csv"
+    run_inference(
+        input_csv=args.input_file,
+        output_csv=output_csv,
+        data_root_path=args.data_root_path,
+    )

requirements.txt

@@ -0,0 +1,3 @@
+openmim
+tensorboard
+future

start.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+METADATA_CSV='http://ptak.felk.cvut.cz/plants/DanishFungiDataset/FungiCLEF2023_val_metadata_PRODUCTION.csv'
+DATA_ROOT_PATH='/Data'
+python ./predict.py \
+    --input-file $METADATA_CSV \
+    --data-root-path $DATA_ROOT_PATH \
+    --output-file user_submission.csv && \
+python ./evaluate.py \
+    --test-annotation-file $METADATA_CSV \
+    --user-submission-file user_submission.csv

tools/test_generate_result_pre-consensus.py

@@ -17,21 +17,11 @@ import torch
 
 from torch.nn import DataParallel
 
-#from mmcv import DictAction
-from mmengine.model.wrappers import MMDistributedDataParallel
-#from mmcv.parallel import MMDataParallel, MMDistributedDataParallel
-#from mmcv.runner import (load_checkpoint,
-#                         wrap_fp16_model)
+from mmengine.config import DictAction
 from mmengine.runner import load_checkpoint
 from mmengine.registry import DefaultScope
-
-
-from mmpretrain.datasets import build_dataset
 from mmengine.runner import Runner
-#from mmcls.datasets import build_dataloader, build_dataset
 from mmpretrain.models import build_classifier
-#from mmcls.models import build_classifier
-#from mmcls.utils import get_root_logger, setup_multi_processes
 
 
 def parse_args():
@@ -39,21 +29,23 @@ def parse_args():
     parser.add_argument('config', help='test config file path')
     parser.add_argument('checkpoint', help='checkpoint file')
     parser.add_argument('out', help='output result file')
+    parser.add_argument('--threshold', default=None, type=float, help='open-set threshold')
+    parser.add_argument('--no-scores', action='store_true', help='don\'t write score .csv file')
     parser.add_argument(
         '--gpu-collect',
         action='store_true',
         help='whether to use gpu to collect results')
     parser.add_argument('--tmpdir', help='tmp dir for writing some results')
-    # parser.add_argument(
-    #     '--cfg-options',
-    #     nargs='+',
-    #     action=DictAction,
-    #     help='override some settings in the used config, the key-value pair '
-    #     'in xxx=yyy format will be merged into config file. If the value to '
-    #     'be overwritten is a list, it should be like key="[a,b]" or key=a,b '
-    #     'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" '
-    #     'Note that the quotation marks are necessary and that no white space '
-    #     'is allowed.')
+    parser.add_argument(
+        '--cfg-options',
+        nargs='+',
+        action=DictAction,
+        help='override some settings in the used config, the key-value pair '
+        'in xxx=yyy format will be merged into config file. If the value to '
+        'be overwritten is a list, it should be like key="[a,b]" or key=a,b '
+        'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" '
+        'Note that the quotation marks are necessary and that no white space '
+        'is allowed.')
     parser.add_argument(
         '--device', default=None, help='device used for testing. (Deprecated)')
     parser.add_argument(
@@ -103,8 +95,6 @@ def single_gpu_test(model,
     observation_ids = []
     for i, data in enumerate(data_loader):
         with torch.no_grad():
-            #result = model(return_loss=False, **data)
-            #imgs = data['inputs'].cuda()
             #data = model.module.data_preprocessor(data, training=False)
             imgs = data['inputs'].cuda()
             result = model.module.extract_feat(imgs)
@@ -112,12 +102,10 @@ def single_gpu_test(model,
         filenames = [x.img_path for x in data['data_samples']]
         obs_ids = [osp.basename(x).split('.')[0].split('-')[1] for x in filenames]
         result = list(zip(result[0], obs_ids))
-        #print(result)
 
         batch_size = len(result)
         results.extend(result)
 
-        #batch_size = data['img'].size(0)
         prog_bar.update(batch_size)
     return results
 
@@ -167,8 +155,8 @@ def main():
     default_scope = DefaultScope.get_instance('test', scope_name='mmpretrain')
 
     cfg = mmengine.Config.fromfile(args.config) #mmcv.Config.fromfile(args.config)
-    #if args.cfg_options is not None:
-    #    cfg.merge_from_dict(args.cfg_options)
+    if args.cfg_options is not None:
+        cfg.merge_from_dict(args.cfg_options)
 
     # set multi-process settings
     setup_multi_processes(cfg)
@@ -187,42 +175,12 @@ def main():
     else:
         cfg.gpu_ids = [args.gpu_id]
 
-    # dataset = build_dataset(cfg.data.test, default_args=dict(test_mode=True))
-
-    # # build the dataloader
-    # # The default loader config
-    # loader_cfg = dict(
-    #     # cfg.gpus will be ignored if distributed
-    #     num_gpus=len(cfg.gpu_ids),
-    #     dist=False,
-    #     round_up=True,
-    # )
-    # # The overall dataloader settings
-    # loader_cfg.update({
-    #     k: v
-    #     for k, v in cfg.data.items() if k not in [
-    #         'train', 'val', 'test', 'train_dataloader', 'val_dataloader',
-    #         'test_dataloader'
-    #     ]
-    # })
-    # test_loader_cfg = {
-    #     **loader_cfg,
-    #     'shuffle': False,  # Not shuffle by default
-    #     'sampler_cfg': None,  # Not use sampler by default
-    #     **cfg.data.get('test_dataloader', {}),
-    # }
-    # the extra round_up data will be removed during gpu/cpu collect
     data_loader = Runner.build_dataloader(cfg.test_dataloader)
 
     # build the model and load checkpoint
     model = build_classifier(cfg.model)
     checkpoint = load_checkpoint(model, args.checkpoint, map_location='cpu')
 
-    # if 'CLASSES' in checkpoint.get('meta', {}):
-    #     CLASSES = checkpoint['meta']['CLASSES']
-    # if CLASSES is None:
-    #     CLASSES = dataset.CLASSES
-
     if args.device == 'cpu':
         model = model.cpu()
     else:
@@ -231,30 +189,39 @@ def main():
             assert mmcv.digit_version(mmcv.__version__) >= (1, 4, 4), \
                 'To test with CPU, please confirm your mmcv version ' \
                 'is not lower than v1.4.4'
-    #model.CLASSES = CLASSES
+
     outputs = single_gpu_test(model, data_loader)
 
     results = defaultdict(list)
     for result, obs_id in outputs:
         results[obs_id].append(result)
+
+    if not args.no_scores:
+        with open(args.out + '.scores.csv', 'w') as f2:
+            for obs_id, result in results.items():
+                avg_feats = torch.mean(torch.stack(result, dim=0), dim=0, keepdim=True)
+                scores = model.module.head(avg_feats)
+                f2.write(f'{obs_id}')
+                for s in scores:
+                    f2.write(f',{s}')
+                f2.write('\n')
     
     dropped = 0
     total = 0
-    with open(args.out, 'w') as f, open(args.out + '.scores.csv', 'w') as f2:
-        f.write('observation_id,class_id\n')
+    with open(args.out, 'w') as f:
+        f.write('observationID,class_id\n')
         for obs_id, result in results.items():
             avg_feats = torch.mean(torch.stack(result, dim=0), dim=0, keepdim=True)
             scores = model.module.head(avg_feats)
-            class_id = np.argmax(scores.detach().cpu())
-            #if np.max(scores) < 0.1:
-            #    class_id = -1
-            #    dropped += 1
+            scores = scores.detach().cpu().numpy()
+            class_id = np.argmax(scores)
+            if args.threshold:
+                max_score = float(torch.max(torch.softmax(torch.from_numpy(scores), dim=0)))
+                if max_score < args.threshold:
+                    class_id = -1
+                    dropped += 1
             total += 1
             f.write(f'{obs_id},{float(class_id):.1f}\n')
-            f2.write(f'{obs_id}')
-            for s in scores:
-                f2.write(f',{s}')
-            f2.write('\n')
 
     print(f'dropped {dropped} out of {total}')