Commit
·
df4bc88
1
Parent(s):
b12592e
Update Space (evaluate main: 8dfe0578)
Browse files- README.md +94 -5
- app.py +6 -0
- confusion_matrix.py +88 -0
- requirements.txt +2 -0
README.md
CHANGED
@@ -1,12 +1,101 @@
|
|
1 |
---
|
2 |
title: Confusion Matrix
|
3 |
-
emoji:
|
4 |
-
colorFrom:
|
5 |
-
colorTo:
|
6 |
sdk: gradio
|
7 |
-
sdk_version:
|
8 |
app_file: app.py
|
9 |
pinned: false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
10 |
---
|
11 |
|
12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
---
|
2 |
title: Confusion Matrix
|
3 |
+
emoji: 🤗
|
4 |
+
colorFrom: blue
|
5 |
+
colorTo: red
|
6 |
sdk: gradio
|
7 |
+
sdk_version: 3.19.1
|
8 |
app_file: app.py
|
9 |
pinned: false
|
10 |
+
tags:
|
11 |
+
- evaluate
|
12 |
+
- metric
|
13 |
+
description: >-
|
14 |
+
The confusion matrix evaluates classification accuracy.
|
15 |
+
|
16 |
+
Each row in a confusion matrix represents a true class and each column represents the instances in a predicted class.
|
17 |
---
|
18 |
|
19 |
+
# Metric Card for Confusion Matrix
|
20 |
+
|
21 |
+
|
22 |
+
## Metric Description
|
23 |
+
|
24 |
+
The confusion matrix evaluates classification accuracy. Each row in a confusion matrix represents a true class and each column represents the instances in a predicted class. Let's look at an example:
|
25 |
+
|
26 |
+
| | setosa | versicolor | virginica |
|
27 |
+
| ---------- | ------ | ---------- | --------- |
|
28 |
+
| setosa | 13 | 0 | 0 |
|
29 |
+
| versicolor | 0 | 10 | 6 |
|
30 |
+
| virginica | 0 | 0 | 9 |
|
31 |
+
|
32 |
+
What information does this confusion matrix provide?
|
33 |
+
|
34 |
+
* All setosa instances were properly predicted as such (true positives).
|
35 |
+
* The model always correctly classifies the setosa class (there are no false positives).
|
36 |
+
* 10 versicolor instances were properly classified, but 6 instances were misclassified as virginica.
|
37 |
+
* All virginica insances were properly classified as such.
|
38 |
+
|
39 |
+
|
40 |
+
## How to Use
|
41 |
+
|
42 |
+
At minimum, this metric requires predictions and references as inputs.
|
43 |
+
|
44 |
+
```python
|
45 |
+
>>> confusion_metric = evaluate.load("confusion_matrix")
|
46 |
+
>>> results = confusion_metric.compute(references=[0, 1, 1, 2, 0, 2, 2], predictions=[0, 2, 1, 1, 0, 2, 0])
|
47 |
+
>>> print(results)
|
48 |
+
{'confusion_matrix': [[2, 0, 0], [0, 1, 1], [1, 1, 1]]}
|
49 |
+
```
|
50 |
+
|
51 |
+
|
52 |
+
### Inputs
|
53 |
+
- **predictions** (`list` of `int`): Predicted labels.
|
54 |
+
- **references** (`list` of `int`): Ground truth labels.
|
55 |
+
- **labels** (`list` of `int`): List of labels to index the matrix. This may be used to reorder or select a subset of labels.
|
56 |
+
- **sample_weight** (`list` of `float`): Sample weights.
|
57 |
+
- **normalize** (`str`): Normalizes confusion matrix over the true (rows), predicted (columns) conditions or all the population.
|
58 |
+
|
59 |
+
|
60 |
+
### Output Values
|
61 |
+
- **confusion_matrix**(`list` of `list` of `str`): Confusion matrix. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`.
|
62 |
+
|
63 |
+
Output Example(s):
|
64 |
+
```python
|
65 |
+
{'confusion_matrix': [[2, 0, 0], [0, 1, 1], [1, 1, 1]]}
|
66 |
+
```
|
67 |
+
|
68 |
+
This metric outputs a dictionary, containing the confusion matrix.
|
69 |
+
|
70 |
+
|
71 |
+
### Examples
|
72 |
+
|
73 |
+
Example 1 - A simple example
|
74 |
+
|
75 |
+
```python
|
76 |
+
>>> confusion_metric = evaluate.load("confusion_matrix")
|
77 |
+
>>> results = confusion_metric.compute(references=[0, 1, 1, 2, 0, 2, 2], predictions=[0, 2, 1, 1, 0, 2, 0])
|
78 |
+
>>> print(results)
|
79 |
+
{'confusion_matrix': [[2, 0, 0], [0, 1, 1], [1, 1, 1]]}
|
80 |
+
```
|
81 |
+
|
82 |
+
## Citation(s)
|
83 |
+
```bibtex
|
84 |
+
@article{scikit-learn,
|
85 |
+
title={Scikit-learn: Machine Learning in {P}ython},
|
86 |
+
author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
|
87 |
+
and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
|
88 |
+
and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
|
89 |
+
Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
|
90 |
+
journal={Journal of Machine Learning Research},
|
91 |
+
volume={12},
|
92 |
+
pages={2825--2830},
|
93 |
+
year={2011}
|
94 |
+
}
|
95 |
+
```
|
96 |
+
|
97 |
+
|
98 |
+
## Further References
|
99 |
+
|
100 |
+
* https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html
|
101 |
+
* https://en.wikipedia.org/wiki/Confusion_matrix
|
app.py
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import evaluate
|
2 |
+
from evaluate.utils import launch_gradio_widget
|
3 |
+
|
4 |
+
|
5 |
+
module = evaluate.load("confusion_matrix")
|
6 |
+
launch_gradio_widget(module)
|
confusion_matrix.py
ADDED
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
|
2 |
+
#
|
3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4 |
+
# you may not use this file except in compliance with the License.
|
5 |
+
# You may obtain a copy of the License at
|
6 |
+
#
|
7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8 |
+
#
|
9 |
+
# Unless required by applicable law or agreed to in writing, software
|
10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12 |
+
# See the License for the specific language governing permissions and
|
13 |
+
# limitations under the License.
|
14 |
+
"""Confusion Matrix."""
|
15 |
+
|
16 |
+
import datasets
|
17 |
+
from sklearn.metrics import confusion_matrix
|
18 |
+
|
19 |
+
import evaluate
|
20 |
+
|
21 |
+
|
22 |
+
_DESCRIPTION = """
|
23 |
+
The confusion matrix evaluates classification accuracy. Each row in a confusion matrix represents a true class and each column represents the instances in a predicted class
|
24 |
+
"""
|
25 |
+
|
26 |
+
_KWARGS_DESCRIPTION = """
|
27 |
+
Args:
|
28 |
+
predictions (`list` of `int`): Predicted labels.
|
29 |
+
references (`list` of `int`): Ground truth labels.
|
30 |
+
labels (`list` of `int`): List of labels to index the matrix. This may be used to reorder or select a subset of labels.
|
31 |
+
sample_weight (`list` of `float`): Sample weights.
|
32 |
+
normalize (`str`): Normalizes confusion matrix over the true (rows), predicted (columns) conditions or all the population.
|
33 |
+
|
34 |
+
Returns:
|
35 |
+
confusion_matrix (`list` of `list` of `int`): Confusion matrix whose i-th row and j-th column entry indicates the number of samples with true label being i-th class and predicted label being j-th class.
|
36 |
+
|
37 |
+
Examples:
|
38 |
+
|
39 |
+
Example 1-A simple example
|
40 |
+
>>> confusion_matrix_metric = evaluate.load("confusion_matrix")
|
41 |
+
>>> results = confusion_matrix_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0])
|
42 |
+
>>> print(results) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
|
43 |
+
{'confusion_matrix': array([[1, 0, 1], [0, 2, 0], [1, 1, 0]][...])}
|
44 |
+
"""
|
45 |
+
|
46 |
+
|
47 |
+
_CITATION = """
|
48 |
+
@article{scikit-learn,
|
49 |
+
title={Scikit-learn: Machine Learning in {P}ython},
|
50 |
+
author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
|
51 |
+
and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
|
52 |
+
and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
|
53 |
+
Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
|
54 |
+
journal={Journal of Machine Learning Research},
|
55 |
+
volume={12},
|
56 |
+
pages={2825--2830},
|
57 |
+
year={2011}
|
58 |
+
}
|
59 |
+
"""
|
60 |
+
|
61 |
+
|
62 |
+
@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
|
63 |
+
class ConfusionMatrix(evaluate.Metric):
|
64 |
+
def _info(self):
|
65 |
+
return evaluate.MetricInfo(
|
66 |
+
description=_DESCRIPTION,
|
67 |
+
citation=_CITATION,
|
68 |
+
inputs_description=_KWARGS_DESCRIPTION,
|
69 |
+
features=datasets.Features(
|
70 |
+
{
|
71 |
+
"predictions": datasets.Sequence(datasets.Value("int32")),
|
72 |
+
"references": datasets.Sequence(datasets.Value("int32")),
|
73 |
+
}
|
74 |
+
if self.config_name == "multilabel"
|
75 |
+
else {
|
76 |
+
"predictions": datasets.Value("int32"),
|
77 |
+
"references": datasets.Value("int32"),
|
78 |
+
}
|
79 |
+
),
|
80 |
+
reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html"],
|
81 |
+
)
|
82 |
+
|
83 |
+
def _compute(self, predictions, references, labels=None, sample_weight=None, normalize=None):
|
84 |
+
return {
|
85 |
+
"confusion_matrix": confusion_matrix(
|
86 |
+
references, predictions, labels=labels, sample_weight=sample_weight, normalize=normalize
|
87 |
+
)
|
88 |
+
}
|
requirements.txt
ADDED
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
1 |
+
git+https://github.com/huggingface/evaluate@8dfe05784099fb9af55b8e77793205a3b7c86465
|
2 |
+
scikit-learn
|