Spaces:
Sleeping
Sleeping
File size: 3,634 Bytes
d916065 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 |
import unittest
from nltk.corpus import brown
from nltk.jsontags import JSONTaggedDecoder, JSONTaggedEncoder
from nltk.tag import (
AffixTagger,
BigramTagger,
BrillTagger,
BrillTaggerTrainer,
DefaultTagger,
NgramTagger,
PerceptronTagger,
RegexpTagger,
TrigramTagger,
UnigramTagger,
)
from nltk.tag.brill import nltkdemo18
class TestJSONSerialization(unittest.TestCase):
def setUp(self):
self.corpus = brown.tagged_sents()[:35]
self.decoder = JSONTaggedDecoder()
self.encoder = JSONTaggedEncoder()
self.default_tagger = DefaultTagger("NN")
def test_default_tagger(self):
encoded = self.encoder.encode(self.default_tagger)
decoded = self.decoder.decode(encoded)
self.assertEqual(repr(self.default_tagger), repr(decoded))
self.assertEqual(self.default_tagger._tag, decoded._tag)
def test_regexp_tagger(self):
tagger = RegexpTagger([(r".*", "NN")], backoff=self.default_tagger)
encoded = self.encoder.encode(tagger)
decoded = self.decoder.decode(encoded)
self.assertEqual(repr(tagger), repr(decoded))
self.assertEqual(repr(tagger.backoff), repr(decoded.backoff))
self.assertEqual(tagger._regexps, decoded._regexps)
def test_affix_tagger(self):
tagger = AffixTagger(self.corpus, backoff=self.default_tagger)
encoded = self.encoder.encode(tagger)
decoded = self.decoder.decode(encoded)
self.assertEqual(repr(tagger), repr(decoded))
self.assertEqual(repr(tagger.backoff), repr(decoded.backoff))
self.assertEqual(tagger._affix_length, decoded._affix_length)
self.assertEqual(tagger._min_word_length, decoded._min_word_length)
self.assertEqual(tagger._context_to_tag, decoded._context_to_tag)
def test_ngram_taggers(self):
unitagger = UnigramTagger(self.corpus, backoff=self.default_tagger)
bitagger = BigramTagger(self.corpus, backoff=unitagger)
tritagger = TrigramTagger(self.corpus, backoff=bitagger)
ntagger = NgramTagger(4, self.corpus, backoff=tritagger)
encoded = self.encoder.encode(ntagger)
decoded = self.decoder.decode(encoded)
self.assertEqual(repr(ntagger), repr(decoded))
self.assertEqual(repr(tritagger), repr(decoded.backoff))
self.assertEqual(repr(bitagger), repr(decoded.backoff.backoff))
self.assertEqual(repr(unitagger), repr(decoded.backoff.backoff.backoff))
self.assertEqual(
repr(self.default_tagger), repr(decoded.backoff.backoff.backoff.backoff)
)
def test_perceptron_tagger(self):
tagger = PerceptronTagger(load=False)
tagger.train(self.corpus)
encoded = self.encoder.encode(tagger)
decoded = self.decoder.decode(encoded)
self.assertEqual(tagger.model.weights, decoded.model.weights)
self.assertEqual(tagger.tagdict, decoded.tagdict)
self.assertEqual(tagger.classes, decoded.classes)
def test_brill_tagger(self):
trainer = BrillTaggerTrainer(
self.default_tagger, nltkdemo18(), deterministic=True
)
tagger = trainer.train(self.corpus, max_rules=30)
encoded = self.encoder.encode(tagger)
decoded = self.decoder.decode(encoded)
self.assertEqual(repr(tagger._initial_tagger), repr(decoded._initial_tagger))
self.assertEqual(tagger._rules, decoded._rules)
self.assertEqual(tagger._training_stats, decoded._training_stats)
|