Spaces:
Sleeping
Sleeping
File size: 4,461 Bytes
d916065 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 |
import unittest
from nltk.metrics.agreement import AnnotationTask
class TestDisagreement(unittest.TestCase):
"""
Class containing unit tests for nltk.metrics.agreement.Disagreement.
"""
def test_easy(self):
"""
Simple test, based on
https://github.com/foolswood/krippendorffs_alpha/raw/master/krippendorff.pdf.
"""
data = [
("coder1", "dress1", "YES"),
("coder2", "dress1", "NO"),
("coder3", "dress1", "NO"),
("coder1", "dress2", "YES"),
("coder2", "dress2", "NO"),
("coder3", "dress3", "NO"),
]
annotation_task = AnnotationTask(data)
self.assertAlmostEqual(annotation_task.alpha(), -0.3333333)
def test_easy2(self):
"""
Same simple test with 1 rating removed.
Removal of that rating should not matter: K-Apha ignores items with
only 1 rating.
"""
data = [
("coder1", "dress1", "YES"),
("coder2", "dress1", "NO"),
("coder3", "dress1", "NO"),
("coder1", "dress2", "YES"),
("coder2", "dress2", "NO"),
]
annotation_task = AnnotationTask(data)
self.assertAlmostEqual(annotation_task.alpha(), -0.3333333)
def test_advanced(self):
"""
More advanced test, based on
http://www.agreestat.com/research_papers/onkrippendorffalpha.pdf
"""
data = [
("A", "1", "1"),
("B", "1", "1"),
("D", "1", "1"),
("A", "2", "2"),
("B", "2", "2"),
("C", "2", "3"),
("D", "2", "2"),
("A", "3", "3"),
("B", "3", "3"),
("C", "3", "3"),
("D", "3", "3"),
("A", "4", "3"),
("B", "4", "3"),
("C", "4", "3"),
("D", "4", "3"),
("A", "5", "2"),
("B", "5", "2"),
("C", "5", "2"),
("D", "5", "2"),
("A", "6", "1"),
("B", "6", "2"),
("C", "6", "3"),
("D", "6", "4"),
("A", "7", "4"),
("B", "7", "4"),
("C", "7", "4"),
("D", "7", "4"),
("A", "8", "1"),
("B", "8", "1"),
("C", "8", "2"),
("D", "8", "1"),
("A", "9", "2"),
("B", "9", "2"),
("C", "9", "2"),
("D", "9", "2"),
("B", "10", "5"),
("C", "10", "5"),
("D", "10", "5"),
("C", "11", "1"),
("D", "11", "1"),
("C", "12", "3"),
]
annotation_task = AnnotationTask(data)
self.assertAlmostEqual(annotation_task.alpha(), 0.743421052632)
def test_advanced2(self):
"""
Same more advanced example, but with 1 rating removed.
Again, removal of that 1 rating should not matter.
"""
data = [
("A", "1", "1"),
("B", "1", "1"),
("D", "1", "1"),
("A", "2", "2"),
("B", "2", "2"),
("C", "2", "3"),
("D", "2", "2"),
("A", "3", "3"),
("B", "3", "3"),
("C", "3", "3"),
("D", "3", "3"),
("A", "4", "3"),
("B", "4", "3"),
("C", "4", "3"),
("D", "4", "3"),
("A", "5", "2"),
("B", "5", "2"),
("C", "5", "2"),
("D", "5", "2"),
("A", "6", "1"),
("B", "6", "2"),
("C", "6", "3"),
("D", "6", "4"),
("A", "7", "4"),
("B", "7", "4"),
("C", "7", "4"),
("D", "7", "4"),
("A", "8", "1"),
("B", "8", "1"),
("C", "8", "2"),
("D", "8", "1"),
("A", "9", "2"),
("B", "9", "2"),
("C", "9", "2"),
("D", "9", "2"),
("B", "10", "5"),
("C", "10", "5"),
("D", "10", "5"),
("C", "11", "1"),
("D", "11", "1"),
("C", "12", "3"),
]
annotation_task = AnnotationTask(data)
self.assertAlmostEqual(annotation_task.alpha(), 0.743421052632)
|