Upload pathology_tumor_detection version 0.6.3

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+training.csv filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

configs/inference.json

@@ -0,0 +1,131 @@
+{
+    "imports": [
+        "$import glob",
+        "$import os"
+    ],
+    "bundle_root": ".",
+    "output_dir": "$os.path.join(@bundle_root, 'eval')",
+    "dataset_dir": "/workspace/data/medical/pathology",
+    "testing_file": "$os.path.join(@bundle_root, 'testing.csv')",
+    "wsi_reader": "cuCIM",
+    "patch_size": [
+        224,
+        224
+    ],
+    "number_intensity_ch": 3,
+    "device": "$torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')",
+    "network_def": {
+        "_target_": "TorchVisionFCModel",
+        "model_name": "resnet18",
+        "num_classes": 1,
+        "use_conv": true,
+        "pretrained": true
+    },
+    "network": "$@network_def.to(@device)",
+    "preprocessing": {
+        "_target_": "Compose",
+        "transforms": [
+            {
+                "_target_": "CastToTyped",
+                "keys": "image",
+                "dtype": "float32"
+            },
+            {
+                "_target_": "ScaleIntensityRanged",
+                "keys": "image",
+                "a_min": 0.0,
+                "a_max": 255.0,
+                "b_min": -1.0,
+                "b_max": 1.0
+            },
+            {
+                "_target_": "ToTensord",
+                "keys": "image"
+            }
+        ]
+    },
+    "datalist": {
+        "_target_": "CSVDataset",
+        "src": "@testing_file",
+        "kwargs_read_csv": {
+            "names": [
+                "image"
+            ],
+            "header": null
+        },
+        "transform": {
+            "_target_": "Lambdad",
+            "keys": "image",
+            "func": "$lambda x: os.path.join(@dataset_dir, 'testing/images', x + '.tif')"
+        }
+    },
+    "dataset": {
+        "_target_": "MaskedPatchWSIDataset",
+        "data": "@datalist",
+        "mask_level": 6,
+        "patch_size": "@patch_size",
+        "transform": "@preprocessing",
+        "reader": "@wsi_reader"
+    },
+    "dataloader": {
+        "_target_": "DataLoader",
+        "dataset": "@dataset",
+        "batch_size": 400,
+        "shuffle": false,
+        "num_workers": 8
+    },
+    "inferer": {
+        "_target_": "SimpleInferer"
+    },
+    "postprocessing": {
+        "_target_": "Compose",
+        "transforms": [
+            {
+                "_target_": "EnsureTyped",
+                "keys": "pred"
+            },
+            {
+                "_target_": "Activationsd",
+                "keys": "pred",
+                "sigmoid": true
+            },
+            {
+                "_target_": "ToNumpyd",
+                "keys": "pred"
+            }
+        ]
+    },
+    "handlers": [
+        {
+            "_target_": "CheckpointLoader",
+            "load_path": "$@bundle_root + '/models/model.pt'",
+            "load_dict": {
+                "model": "@network"
+            }
+        },
+        {
+            "_target_": "StatsHandler",
+            "tag_name": "progress",
+            "iteration_print_logger": "$lambda engine: print(f'image: \"{engine.state.batch[\"image\"].meta[\"name\"][0]}\", iter: {engine.state.iteration}/{engine.state.epoch_length}') if engine.state.iteration % 100 == 0 else None",
+            "output_transform": "$lambda x: None"
+        },
+        {
+            "_target_": "monai.handlers.ProbMapProducer",
+            "output_dir": "@output_dir"
+        }
+    ],
+    "evaluator": {
+        "_target_": "SupervisedEvaluator",
+        "device": "@device",
+        "val_data_loader": "@dataloader",
+        "network": "@network",
+        "inferer": "@inferer",
+        "postprocessing": "@postprocessing",
+        "val_handlers": "@handlers",
+        "amp": true,
+        "decollate": false
+    },
+    "run": [
+        "[email protected]()"
+    ]
+}

configs/inference_trt.json

@@ -0,0 +1,10 @@
+{
+    "imports": [
+        "$import glob",
+        "$import os",
+        "$import torch_tensorrt"
+    ],
+    "handlers#0#_disabled_": true,
+    "network_def": "$torch.jit.load(@bundle_root + '/models/model_trt.ts')",
+    "evaluator#amp": false
+}

configs/logging.conf

@@ -0,0 +1,21 @@
+[loggers]
+keys=root
+
+[handlers]
+keys=consoleHandler
+
+[formatters]
+keys=fullFormatter
+
+[logger_root]
+level=INFO
+handlers=consoleHandler
+
+[handler_consoleHandler]
+class=StreamHandler
+level=INFO
+formatter=fullFormatter
+args=(sys.stdout,)
+
+[formatter_fullFormatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s

configs/metadata.json

@@ -0,0 +1,110 @@
+{
+    "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20240725.json",
+    "version": "0.6.3",
+    "changelog": {
+        "0.6.3": "update to huggingface hosting",
+        "0.6.2": "enhance readme for nccl timout issue",
+        "0.6.1": "fix multi-gpu issue",
+        "0.6.0": "use monai 1.4 and update large files",
+        "0.5.9": "update to use monai 1.3.1",
+        "0.5.8": "update readme to add memory warning",
+        "0.5.7": "update channel_def in metadata",
+        "0.5.6": "fix the wrong GPU index issue of multi-node",
+        "0.5.5": "modify mgpu logging level",
+        "0.5.4": "retrain using an internal pretrained ResNet18",
+        "0.5.3": "make the training bundle deterministic",
+        "0.5.2": "update TensorRT descriptions",
+        "0.5.1": "update the TensorRT part in the README file",
+        "0.5.0": "add the command of executing inference with TensorRT models",
+        "0.4.9": "adapt to BundleWorkflow interface",
+        "0.4.8": "update the readme file with TensorRT convert",
+        "0.4.7": "add name tag",
+        "0.4.6": "modify dataset key name",
+        "0.4.5": "update model weights and perfomance metrics",
+        "0.4.4": "restructure readme to match updated template",
+        "0.4.3": "fix wrong figure url",
+        "0.4.2": "update metadata with new metrics",
+        "0.4.1": "Fix inference print logger and froc",
+        "0.4.0": "add lesion FROC calculation and wsi_reader",
+        "0.3.3": "update to use monai 1.0.1",
+        "0.3.2": "enhance readme on commands example",
+        "0.3.1": "fix license Copyright error",
+        "0.3.0": "update license files",
+        "0.2.0": "unify naming",
+        "0.1.1": "fix location variable name change",
+        "0.1.0": "initialize release of the bundle"
+    },
+    "monai_version": "1.4.0",
+    "pytorch_version": "2.4.0",
+    "numpy_version": "1.24.4",
+    "required_packages_version": {
+        "cucim-cu12": "24.6.0",
+        "pandas": "2.2.1",
+        "torchvision": "0.19.0",
+        "pytorch-ignite": "0.4.11",
+        "tensorboard": "2.17.0"
+    },
+    "supported_apps": {},
+    "name": "Pathology tumor detection",
+    "task": "Pathology metastasis detection",
+    "description": "A pre-trained model for metastasis detection on Camelyon 16 dataset.",
+    "authors": "MONAI team",
+    "copyright": "Copyright (c) MONAI Consortium",
+    "data_source": "Camelyon dataset",
+    "data_type": "tiff",
+    "image_classes": "RGB image with intensity between 0 and 255",
+    "label_classes": "binary labels for each patch",
+    "pred_classes": "scalar probability",
+    "eval_metrics": {
+        "accuracy": 0.9,
+        "froc": 0.72
+    },
+    "intended_use": "This is an example, not to be used for diagnostic purposes",
+    "references": [
+        ""
+    ],
+    "network_data_format": {
+        "inputs": {
+            "image": {
+                "type": "image",
+                "format": "magnitude",
+                "num_channels": 3,
+                "spatial_shape": [
+                    224,
+                    224
+                ],
+                "dtype": "float32",
+                "value_range": [
+                    0,
+                    255
+                ],
+                "is_patch_data": true,
+                "channel_def": {
+                    "0": "R",
+                    "1": "G",
+                    "2": "B"
+                }
+            }
+        },
+        "outputs": {
+            "pred": {
+                "type": "probability",
+                "format": "classification",
+                "num_channels": 1,
+                "spatial_shape": [
+                    1,
+                    1
+                ],
+                "dtype": "float32",
+                "is_patch_data": true,
+                "value_range": [
+                    0,
+                    1
+                ],
+                "channel_def": {
+                    "0": "metastasis"
+                }
+            }
+        }
+    }
+}

configs/multi_gpu_train.json

@@ -0,0 +1,42 @@
+{
+    "device": "$torch.device('cuda:' + os.environ['LOCAL_RANK'])",
+    "network": {
+        "_target_": "torch.nn.parallel.DistributedDataParallel",
+        "module": "$@network_def.to(@device)",
+        "device_ids": [
+            "@device"
+        ]
+    },
+    "train#sampler": {
+        "_target_": "DistributedSampler",
+        "dataset": "@train#dataset",
+        "even_divisible": true,
+        "shuffle": true
+    },
+    "train#dataloader#sampler": "@train#sampler",
+    "train#dataloader#shuffle": false,
+    "train#trainer#train_handlers": "$@train#handlers[: -2 if dist.get_rank() > 0 else None]",
+    "validate#sampler": {
+        "_target_": "DistributedSampler",
+        "dataset": "@validate#dataset",
+        "even_divisible": false,
+        "shuffle": false
+    },
+    "validate#dataloader#sampler": "@validate#sampler",
+    "validate#evaluator#val_handlers": "$None if dist.get_rank() > 0 else @validate#handlers",
+    "initialize": [
+        "$import torch.distributed as dist",
+        "$dist.is_initialized() or dist.init_process_group(backend='nccl')",
+        "$torch.cuda.set_device(@device)",
+        "$monai.utils.set_determinism(seed=123)",
+        "$import logging",
+        "$@train#trainer.logger.setLevel(logging.WARNING if dist.get_rank() > 0 else logging.INFO)",
+        "$@validate#evaluator.logger.setLevel(logging.WARNING if dist.get_rank() > 0 else logging.INFO)"
+    ],
+    "run": [
+        "$@train#trainer.run()"
+    ],
+    "finalize": [
+        "$dist.destroy_process_group()"
+    ]
+}

configs/train.json

@@ -0,0 +1,380 @@
+{
+    "imports": [
+        "$import os",
+        "$import ignite"
+    ],
+    "lr": 0.001,
+    "num_epochs": 4,
+    "val_interval": 1,
+    "bundle_root": ".",
+    "ckpt_dir": "$os.path.join(@bundle_root, 'models')",
+    "output_dir": "$os.path.join(@bundle_root, 'log')",
+    "training_file": "$os.path.join(@bundle_root, 'training.csv')",
+    "validation_file": "$os.path.join(@bundle_root, 'validation.csv')",
+    "dataset_dir": "/workspace/data/medical/pathology",
+    "wsi_reader": "cuCIM",
+    "region_size": [
+        768,
+        768
+    ],
+    "patch_size": [
+        224,
+        224
+    ],
+    "grid_shape": [
+        3,
+        3
+    ],
+    "number_intensity_ch": 3,
+    "device": "$torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')",
+    "network_def": {
+        "_target_": "TorchVisionFCModel",
+        "model_name": "resnet18",
+        "num_classes": 1,
+        "use_conv": true,
+        "pretrained": false
+    },
+    "network": "$@network_def.to(@device)",
+    "loss": {
+        "_target_": "torch.nn.BCEWithLogitsLoss"
+    },
+    "optimizer": {
+        "_target_": "Novograd",
+        "params": "[email protected]()",
+        "lr": "@lr"
+    },
+    "lr_scheduler": {
+        "_target_": "torch.optim.lr_scheduler.CosineAnnealingLR",
+        "optimizer": "@optimizer",
+        "T_max": "@num_epochs"
+    },
+    "train": {
+        "preprocessing": {
+            "_target_": "Compose",
+            "transforms": [
+                {
+                    "_target_": "Lambdad",
+                    "keys": [
+                        "label"
+                    ],
+                    "func": "$lambda x: x.reshape((1, *@grid_shape))"
+                },
+                {
+                    "_target_": "GridSplitd",
+                    "keys": [
+                        "image",
+                        "label"
+                    ],
+                    "grid": "@grid_shape",
+                    "size": {
+                        "image": "@patch_size",
+                        "label": 1
+                    }
+                },
+                {
+                    "_target_": "ToTensord",
+                    "keys": "image"
+                },
+                {
+                    "_target_": "TorchVisiond",
+                    "keys": "image",
+                    "name": "ColorJitter",
+                    "brightness": 0.25,
+                    "contrast": 0.75,
+                    "saturation": 0.25,
+                    "hue": 0.04
+                },
+                {
+                    "_target_": "ToNumpyd",
+                    "keys": "image"
+                },
+                {
+                    "_target_": "RandFlipd",
+                    "keys": "image",
+                    "prob": 0.5
+                },
+                {
+                    "_target_": "RandRotate90d",
+                    "keys": "image",
+                    "prob": 0.5,
+                    "max_k": 3,
+                    "spatial_axes": [
+                        -2,
+                        -1
+                    ]
+                },
+                {
+                    "_target_": "CastToTyped",
+                    "keys": "image",
+                    "dtype": "float32"
+                },
+                {
+                    "_target_": "RandZoomd",
+                    "keys": "image",
+                    "prob": 0.5,
+                    "min_zoom": 0.9,
+                    "max_zoom": 1.1
+                },
+                {
+                    "_target_": "ScaleIntensityRanged",
+                    "keys": "image",
+                    "a_min": 0.0,
+                    "a_max": 255.0,
+                    "b_min": -1.0,
+                    "b_max": 1.0
+                },
+                {
+                    "_target_": "ToTensord",
+                    "keys": [
+                        "image",
+                        "label"
+                    ]
+                }
+            ]
+        },
+        "datalist": {
+            "_target_": "CSVDataset",
+            "src": "@training_file",
+            "col_groups": {
+                "image": 0,
+                "location": [
+                    2,
+                    1
+                ],
+                "label": [
+                    3,
+                    6,
+                    9,
+                    4,
+                    7,
+                    10,
+                    5,
+                    8,
+                    11
+                ]
+            },
+            "kwargs_read_csv": {
+                "header": null
+            },
+            "transform": {
+                "_target_": "Lambdad",
+                "keys": "image",
+                "func": "$lambda x: os.path.join(@dataset_dir, 'training/images', x + '.tif')"
+            }
+        },
+        "dataset": {
+            "_target_": "monai.data.wsi_datasets.PatchWSIDataset",
+            "data": "@train#datalist",
+            "patch_level": 0,
+            "patch_size": "@region_size",
+            "reader": "@wsi_reader",
+            "transform": "@train#preprocessing"
+        },
+        "dataloader": {
+            "_target_": "DataLoader",
+            "dataset": "@train#dataset",
+            "batch_size": 128,
+            "pin_memory": false,
+            "num_workers": 8
+        },
+        "inferer": {
+            "_target_": "SimpleInferer"
+        },
+        "postprocessing": {
+            "_target_": "Compose",
+            "transforms": [
+                {
+                    "_target_": "Activationsd",
+                    "keys": "pred",
+                    "sigmoid": true
+                },
+                {
+                    "_target_": "AsDiscreted",
+                    "keys": "pred",
+                    "threshold": 0.5
+                }
+            ]
+        },
+        "handlers": [
+            {
+                "_target_": "LrScheduleHandler",
+                "lr_scheduler": "@lr_scheduler",
+                "print_lr": true
+            },
+            {
+                "_target_": "ValidationHandler",
+                "validator": "@validate#evaluator",
+                "epoch_level": true,
+                "interval": "@val_interval"
+            },
+            {
+                "_target_": "StatsHandler",
+                "tag_name": "train_loss",
+                "output_transform": "$monai.handlers.from_engine(['loss'], first=True)"
+            },
+            {
+                "_target_": "TensorBoardStatsHandler",
+                "log_dir": "@output_dir",
+                "tag_name": "train_loss",
+                "output_transform": "$monai.handlers.from_engine(['loss'], first=True)"
+            }
+        ],
+        "key_metric": {
+            "train_acc": {
+                "_target_": "ignite.metrics.Accuracy",
+                "output_transform": "$monai.handlers.from_engine(['pred', 'label'])"
+            }
+        },
+        "trainer": {
+            "_target_": "SupervisedTrainer",
+            "device": "@device",
+            "max_epochs": "@num_epochs",
+            "train_data_loader": "@train#dataloader",
+            "network": "@network",
+            "optimizer": "@optimizer",
+            "loss_function": "@loss",
+            "inferer": "@train#inferer",
+            "amp": true,
+            "postprocessing": "@train#postprocessing",
+            "key_train_metric": "@train#key_metric",
+            "train_handlers": "@train#handlers"
+        }
+    },
+    "validate": {
+        "preprocessing": {
+            "_target_": "Compose",
+            "transforms": [
+                {
+                    "_target_": "Lambdad",
+                    "keys": "label",
+                    "func": "$lambda x: x.reshape((1, *@grid_shape))"
+                },
+                {
+                    "_target_": "GridSplitd",
+                    "keys": [
+                        "image",
+                        "label"
+                    ],
+                    "grid": "@grid_shape",
+                    "size": {
+                        "image": "@patch_size",
+                        "label": 1
+                    }
+                },
+                {
+                    "_target_": "CastToTyped",
+                    "keys": "image",
+                    "dtype": "float32"
+                },
+                {
+                    "_target_": "ScaleIntensityRanged",
+                    "keys": "image",
+                    "a_min": 0.0,
+                    "a_max": 255.0,
+                    "b_min": -1.0,
+                    "b_max": 1.0
+                },
+                {
+                    "_target_": "ToTensord",
+                    "keys": [
+                        "image",
+                        "label"
+                    ]
+                }
+            ]
+        },
+        "datalist": {
+            "_target_": "CSVDataset",
+            "src": "@validation_file",
+            "col_groups": {
+                "image": 0,
+                "location": [
+                    2,
+                    1
+                ],
+                "label": [
+                    3,
+                    6,
+                    9,
+                    4,
+                    7,
+                    10,
+                    5,
+                    8,
+                    11
+                ]
+            },
+            "kwargs_read_csv": {
+                "header": null
+            },
+            "transform": {
+                "_target_": "Lambdad",
+                "keys": "image",
+                "func": "$lambda x: os.path.join(@dataset_dir, 'training/images', x + '.tif')"
+            }
+        },
+        "dataset": {
+            "_target_": "monai.data.wsi_datasets.PatchWSIDataset",
+            "data": "@validate#datalist",
+            "patch_level": 0,
+            "patch_size": "@region_size",
+            "reader": "@wsi_reader",
+            "transform": "@validate#preprocessing"
+        },
+        "dataloader": {
+            "_target_": "DataLoader",
+            "dataset": "@validate#dataset",
+            "batch_size": 128,
+            "pin_memory": false,
+            "shuffle": false,
+            "num_workers": 8
+        },
+        "inferer": {
+            "_target_": "SimpleInferer"
+        },
+        "postprocessing": "%train#postprocessing",
+        "handlers": [
+            {
+                "_target_": "StatsHandler",
+                "iteration_log": false
+            },
+            {
+                "_target_": "TensorBoardStatsHandler",
+                "log_dir": "@output_dir",
+                "iteration_log": false
+            },
+            {
+                "_target_": "CheckpointSaver",
+                "save_dir": "@ckpt_dir",
+                "save_dict": {
+                    "model": "@network"
+                },
+                "save_key_metric": true,
+                "key_metric_filename": "model.pt"
+            }
+        ],
+        "key_metric": {
+            "valid_acc": {
+                "_target_": "ignite.metrics.Accuracy",
+                "output_transform": "$monai.handlers.from_engine(['pred', 'label'])"
+            }
+        },
+        "evaluator": {
+            "_target_": "SupervisedEvaluator",
+            "device": "@device",
+            "val_data_loader": "@validate#dataloader",
+            "network": "@network",
+            "inferer": "@validate#inferer",
+            "postprocessing": "@validate#postprocessing",
+            "key_val_metric": "@validate#key_metric",
+            "val_handlers": "@validate#handlers",
+            "amp": true
+        }
+    },
+    "initialize": [
+        "$monai.utils.set_determinism(seed=15)"
+    ],
+    "run": [
+        "$@train#trainer.run()"
+    ]
+}

docs/README.md

@@ -0,0 +1,194 @@
+# Model Overview
+
+A pre-trained model for automated detection of metastases in whole-slide histopathology images.
+
+The model is trained based on ResNet18 [1] with the last fully connected layer replaced by a 1x1 convolution layer.
+![Diagram showing the flow from model input, through the model architecture, and to model output](http://developer.download.nvidia.com/assets/Clara/Images/clara_pt_pathology_metastasis_detection_workflow.png)
+
+## Data
+
+All the data used to train, validate, and test this model is from [Camelyon-16 Challenge](https://camelyon16.grand-challenge.org/). You can download all the images for "CAMELYON16" data set from various sources listed [here](https://camelyon17.grand-challenge.org/Data/).
+
+Location information for training/validation patches (the location on the whole slide image where patches are extracted) are adopted from [NCRF/coords](https://github.com/baidu-research/NCRF/tree/master/coords).
+
+Annotation information are adopted from [NCRF/jsons](https://github.com/baidu-research/NCRF/tree/master/jsons).
+
+- Target: Tumor
+- Task: Detection
+- Modality: Histopathology
+- Size: 270 WSIs for training/validation, 48 WSIs for testing
+
+### Preprocessing
+
+This bundle expects the training/validation data (whole slide images) reside in a `{dataset_dir}/training/images`. By default `dataset_dir` is pointing to `/workspace/data/medical/pathology/` You can modify `dataset_dir` in the bundle config files to point to a different directory.
+
+To reduce the computation burden during the inference, patches are extracted only where there is tissue and ignoring the background according to a tissue mask. Please also create a directory for prediction output. By default `output_dir` is set to `eval` folder under the bundle root.
+
+Please refer to "Annotation" section of [Camelyon challenge](https://camelyon17.grand-challenge.org/Data/) to prepare ground truth images, which are needed for FROC computation. By default, this data set is expected to be at `/workspace/data/medical/pathology/ground_truths`. But it can be modified in `evaluate_froc.sh`.
+
+## Training configuration
+
+The training was performed with the following:
+
+- Config file: train.config
+- GPU: at least 16 GB of GPU memory.
+- Actual Model Input: 224 x 224 x 3
+- AMP: True
+- Optimizer: Novograd
+- Learning Rate: 1e-3
+- Loss: BCEWithLogitsLoss
+- Whole slide image reader: cuCIM (if running on Windows or Mac, please install `OpenSlide` on your system and change `wsi_reader` to "OpenSlide")
+
+### Pretrained Weights
+
+By setting the `"pretrained"` parameter of `TorchVisionFCModel` in the config file to `true`, ImageNet pre-trained weights will be used for training. Please note that these weights are for non-commercial use. Each user is responsible for checking the content of the models/datasets and the applicable licenses and determining if suitable for the intended use. In order to use other pretrained weights, you can use `CheckpointLoader` in train handlers section as the first handler:
+
+```json
+{
+    "_target_": "CheckpointLoader",
+    "load_path": "$@bundle_root + '/pretrained_resnet18.pth'",
+    "strict": false,
+    "load_dict": {
+        "model_new": "@network"
+    }
+}
+```
+
+### Input
+
+The training pipeline is a json file (dataset.json) which includes path to each WSI, the location and the label information for each training patch.
+
+### Output
+
+A probability number of the input patch being tumor or normal.
+
+### Memory Consumption Warning
+
+If you face memory issues in traning, you can lower the `batch_size` in the configurations to reduce the System RAM requirements.
+
+### Inference on a WSI
+
+Inference is performed on WSI in a sliding window manner with specified stride. A foreground mask is needed to specify the region where the inference will be performed on, given that background region which contains no tissue at all can occupy a significant portion of a WSI. Output of the inference pipeline is a probability map of size 1/stride of original WSI size.
+
+### Note on determinism
+
+By default this bundle use a deterministic approach to make the results reproducible. However, it comes at a cost of performance loss. Thus if you do not care about reproducibility, you can have a performance gain by replacing `"$monai.utils.set_determinism"` line with `"$setattr(torch.backends.cudnn, 'benchmark', True)"` in initialize section of training configuration (`configs/train.json` and `configs/multi_gpu_train.json` for single GPU and multi-GPU training respectively).
+
+## Performance
+
+FROC score is used for evaluating the performance of the model. After inference is done, `evaluate_froc.sh` needs to be run to evaluate FROC score based on predicted probability map (output of inference) and the ground truth tumor masks.
+Using an internal pretrained weights for ResNet18, this model deterministically achieves the 0.90 accuracy on validation patches, and FROC of 0.72 on the 48 Camelyon testing data that have ground truth annotations available.
+
+![A Graph showing Train Acc, Train Loss, and Validation Acc](https://developer.download.nvidia.com/assets/Clara/Images/monai_pathology_tumor_detection_train_and_val_metrics_v5.png)
+
+The `pathology_tumor_detection` bundle supports acceleration with TensorRT. The table below displays the speedup ratios observed on an A100 80G GPU.
+
+Please notice that the benchmark results are tested on one WSI image since the images are too large to benchmark. And the inference time in the end-to-end line stands for one patch of the whole image.
+
+| method | torch_fp32(ms) | torch_amp(ms) | trt_fp32(ms) | trt_fp16(ms) | speedup amp | speedup fp32 | speedup fp16 | amp vs fp16|
+| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
+| model computation |1.93 | 2.52 | 1.61 | 1.33 | 0.77 | 1.20 | 1.45 | 1.89 |
+| end2end |224.97 | 223.50 | 222.65 | 224.03 | 1.01 | 1.01 | 1.00 | 1.00 |
+
+Where:
+
+- `model computation` means the speedup ratio of model's inference with a random input without preprocessing and postprocessing
+- `end2end` means run the bundle end-to-end with the TensorRT based model.
+- `torch_fp32` and `torch_amp` are for the PyTorch models with or without `amp` mode.
+- `trt_fp32` and `trt_fp16` are for the TensorRT based models converted in corresponding precision.
+- `speedup amp`, `speedup fp32` and `speedup fp16` are the speedup ratios of corresponding models versus the PyTorch float32 model
+- `amp vs fp16` is the speedup ratio between the PyTorch amp model and the TensorRT float16 based model.
+
+This result is benchmarked under:
+
+- TensorRT: 8.5.3+cuda11.8
+- Torch-TensorRT Version: 1.4.0
+- CPU Architecture: x86-64
+- OS: ubuntu 20.04
+- Python version:3.8.10
+- CUDA version: 12.0
+- GPU models and configuration: A100 80G
+
+## MONAI Bundle Commands
+
+In addition to the Pythonic APIs, a few command line interfaces (CLI) are provided to interact with the bundle. The CLI supports flexible use cases, such as overriding configs at runtime and predefining arguments in a file.
+
+For more details usage instructions, visit the [MONAI Bundle Configuration Page](https://docs.monai.io/en/latest/config_syntax.html).
+
+#### Execute training
+
+```
+python -m monai.bundle run --config_file configs/train.json
+```
+
+Please note that if the default dataset path is not modified with the actual path in the bundle config files, you can also override it by using `--dataset_dir`:
+
+```
+python -m monai.bundle run --config_file configs/train.json --dataset_dir <actual dataset path>
+```
+
+#### Override the `train` config to execute multi-GPU training
+
+```
+torchrun --standalone --nnodes=1 --nproc_per_node=2 -m monai.bundle run --config_file "['configs/train.json','configs/multi_gpu_train.json']"
+```
+
+Please note that the distributed training-related options depend on the actual running environment; thus, users may need to remove `--standalone`, modify `--nnodes`, or do some other necessary changes according to the machine used. For more details, please refer to [pytorch's official tutorial](https://pytorch.org/tutorials/intermediate/ddp_tutorial.html).
+
+**Note:** When using a container based on [PyTorch 24.0x](https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes), you may encounter random NCCL timeout errors. To address this issue, consider the following adjustments:
+
+- Reduce the `num_workers`: Decreasing the number of data loader workers can help minimize these errors.
+- Set `pin_memory` to `False`: Disabling pinned memory may reduce the likelihood of timeouts.
+- Switch to the `gloo` backend: As a workaround, you can set the distributed training backend to `gloo` to avoid NCCL-related timeouts.
+
+You can implement these settings by adding flags like `--train#dataloader#num_workers 0` or `--train#dataloader#pin_memory false`.
+
+#### Execute inference
+
+```
+CUDA_LAUNCH_BLOCKING=1 python -m monai.bundle run --config_file configs/inference.json
+```
+
+#### Evaluate FROC metric
+
+```
+cd scripts && source evaluate_froc.sh
+```
+
+#### Export checkpoint to TorchScript file
+
+```
+python -m monai.bundle ckpt_export network_def --filepath models/model.ts --ckpt_file models/model.pt --meta_file configs/metadata.json --config_file configs/inference.json
+```
+
+#### Export checkpoint to TensorRT based models with fp32 or fp16 precision
+
+```
+python -m monai.bundle trt_export --net_id network_def --filepath models/model_trt.ts --ckpt_file models/model.pt --meta_file configs/metadata.json --config_file configs/inference.json --precision <fp32/fp16> --dynamic_batchsize "[1, 400, 600]"
+```
+
+#### Execute inference with the TensorRT model
+
+```
+python -m monai.bundle run --config_file "['configs/inference.json', 'configs/inference_trt.json']"
+```
+
+# References
+
+[1] He, Kaiming, et al, "Deep Residual Learning for Image Recognition." In Proceedings of the IEEE conference on computer vision and pattern recognition, pp. 770-778. 2016. <https://arxiv.org/pdf/1512.03385.pdf>
+
+# License
+
+Copyright (c) MONAI Consortium
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

docs/data_license.txt

@@ -0,0 +1,128 @@
+CAMELYON16 data set by Computational Pathology Group of Radboud University
+Medical Centre
+
+CAMELYON16 data set is available under CC0.
+
+==========================================================================
+
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+    HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display,
+     communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+     likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+     subject to the limitations in paragraph 4(a), below;
+  v. rights protecting the extraction, dissemination, use and reuse of data
+     in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+     European Parliament and of the Council of 11 March 1996 on the legal
+     protection of databases, and under any national implementation
+     thereof, including any amended or successor version of such
+     directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+     world based on applicable law or treaty, and any national
+     implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+    surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+    warranties of any kind concerning the Work, express, implied,
+    statutory or otherwise, including without limitation warranties of
+    title, merchantability, fitness for a particular purpose, non
+    infringement, or the absence of latent or other defects, accuracy, or
+    the present or absence of errors, whether or not discoverable, all to
+    the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+    that may apply to the Work or any use thereof, including without
+    limitation any person's Copyright and Related Rights in the Work.
+    Further, Affirmer disclaims responsibility for obtaining any necessary
+    consents, permissions or other rights required for any use of the
+    Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+    party to this document and has no duty or obligation with respect to
+    this CC0 or use of the Work.

models/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a0d9b9e714e18a90c1f7f7d9c7e47f807c59f9f8c681b84865fae208fcbb4d6
+size 44780565

scripts/evaluate_froc.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+LEVEL=6
+SPACING=0.243
+READER=openslide
+EVAL_DIR=../eval
+GROUND_TRUTH_DIR=/workspace/data/medical/pathology/testing/ground_truths
+
+echo "=> Level= ${LEVEL}"
+echo "=> Spacing = ${SPACING}"
+echo "=> WSI Reader: ${READER}"
+echo "=> Evaluation output directory: ${EVAL_DIR}"
+echo "=> Ground truth directory: ${GROUND_TRUTH_DIR}"
+
+python3 ./lesion_froc.py \
+    --level $LEVEL \
+    --spacing $SPACING \
+    --reader $READER \
+    --eval-dir ${EVAL_DIR} \
+    --ground-truth-dir ${GROUND_TRUTH_DIR}

scripts/lesion_froc.py

@@ -0,0 +1,56 @@
+import argparse
+import os
+
+from monai.apps.pathology import LesionFROC
+
+
+def full_path(dir: str, file: str):
+    return os.path.normpath(os.path.join(dir, file))
+
+
+def load_data(ground_truth_dir: str, eval_dir: str, level: int, spacing: float):
+    # Get the list of probability map result files
+    prob_files = os.listdir(eval_dir)
+
+    # read the dataset and create an eval_dataset based on that.
+    eval_dataset = []
+    for prob_name in prob_files:
+        if prob_name.endswith(".npy"):
+            sample = {
+                "tumor_mask": full_path(ground_truth_dir, prob_name.replace("npy", "tif")),
+                "prob_map": full_path(eval_dir, prob_name),
+                "level": level,
+                "pixel_spacing": spacing,
+            }
+
+            eval_dataset.append(sample)
+
+    return eval_dataset
+
+
+def evaluate_froc(data, reader):
+    lesion_froc = LesionFROC(data, image_reader_name=reader)
+    score = lesion_froc.evaluate()
+    return score
+
+
+if __name__ == "__main__":
+    # Parse command line arguments
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-s", "--spacing", type=float, default=0.243, dest="spacing")
+    parser.add_argument("-l", "--level", type=int, default=6, dest="level")
+    parser.add_argument("-r", "--reader", type=str, default="cucim", dest="reader")
+    parser.add_argument("-e", "--eval-dir", type=str, dest="eval_dir")
+    parser.add_argument("-g", "--ground-truth-dir", type=str, dest="ground_truth_dir")
+    args = parser.parse_args()
+
+    # prepare FROC input data
+    data = load_data(args.ground_truth_dir, args.eval_dir, args.level, args.spacing)
+    if len(data) < 1:
+        raise RuntimeError(f"No probability map result found in '{args.eval_dir}' with '.npy' extension.")
+
+    # evaluate FROC
+    score = evaluate_froc(data, args.reader)
+    with open(full_path(args.eval_dir, "froc_score.txt"), "w") as f:
+        f.write(f"FROC Score: {score}\n")
+    print(f"FROC Score: {score}")

testing.csv

@@ -0,0 +1,48 @@
+test_001
+test_002
+test_004
+test_008
+test_010
+test_011
+test_013
+test_016
+test_021
+test_026
+test_027
+test_029
+test_030
+test_033
+test_038
+test_040
+test_046
+test_048
+test_051
+test_052
+test_061
+test_064
+test_065
+test_066
+test_068
+test_069
+test_071
+test_073
+test_074
+test_075
+test_079
+test_082
+test_084
+test_090
+test_092
+test_094
+test_097
+test_099
+test_102
+test_104
+test_105
+test_108
+test_110
+test_113
+test_116
+test_117
+test_121
+test_122

training.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3e51c55b11485c51fea7f9dab7811edc120ab473da0f1349c4bf40c9d15d0b4
+size 20690435