Spaces:
Sleeping
Sleeping
| import unittest | |
| from nltk.corpus import brown | |
| from nltk.jsontags import JSONTaggedDecoder, JSONTaggedEncoder | |
| from nltk.tag import ( | |
| AffixTagger, | |
| BigramTagger, | |
| BrillTagger, | |
| BrillTaggerTrainer, | |
| DefaultTagger, | |
| NgramTagger, | |
| PerceptronTagger, | |
| RegexpTagger, | |
| TrigramTagger, | |
| UnigramTagger, | |
| ) | |
| from nltk.tag.brill import nltkdemo18 | |
| class TestJSONSerialization(unittest.TestCase): | |
| def setUp(self): | |
| self.corpus = brown.tagged_sents()[:35] | |
| self.decoder = JSONTaggedDecoder() | |
| self.encoder = JSONTaggedEncoder() | |
| self.default_tagger = DefaultTagger("NN") | |
| def test_default_tagger(self): | |
| encoded = self.encoder.encode(self.default_tagger) | |
| decoded = self.decoder.decode(encoded) | |
| self.assertEqual(repr(self.default_tagger), repr(decoded)) | |
| self.assertEqual(self.default_tagger._tag, decoded._tag) | |
| def test_regexp_tagger(self): | |
| tagger = RegexpTagger([(r".*", "NN")], backoff=self.default_tagger) | |
| encoded = self.encoder.encode(tagger) | |
| decoded = self.decoder.decode(encoded) | |
| self.assertEqual(repr(tagger), repr(decoded)) | |
| self.assertEqual(repr(tagger.backoff), repr(decoded.backoff)) | |
| self.assertEqual(tagger._regexps, decoded._regexps) | |
| def test_affix_tagger(self): | |
| tagger = AffixTagger(self.corpus, backoff=self.default_tagger) | |
| encoded = self.encoder.encode(tagger) | |
| decoded = self.decoder.decode(encoded) | |
| self.assertEqual(repr(tagger), repr(decoded)) | |
| self.assertEqual(repr(tagger.backoff), repr(decoded.backoff)) | |
| self.assertEqual(tagger._affix_length, decoded._affix_length) | |
| self.assertEqual(tagger._min_word_length, decoded._min_word_length) | |
| self.assertEqual(tagger._context_to_tag, decoded._context_to_tag) | |
| def test_ngram_taggers(self): | |
| unitagger = UnigramTagger(self.corpus, backoff=self.default_tagger) | |
| bitagger = BigramTagger(self.corpus, backoff=unitagger) | |
| tritagger = TrigramTagger(self.corpus, backoff=bitagger) | |
| ntagger = NgramTagger(4, self.corpus, backoff=tritagger) | |
| encoded = self.encoder.encode(ntagger) | |
| decoded = self.decoder.decode(encoded) | |
| self.assertEqual(repr(ntagger), repr(decoded)) | |
| self.assertEqual(repr(tritagger), repr(decoded.backoff)) | |
| self.assertEqual(repr(bitagger), repr(decoded.backoff.backoff)) | |
| self.assertEqual(repr(unitagger), repr(decoded.backoff.backoff.backoff)) | |
| self.assertEqual( | |
| repr(self.default_tagger), repr(decoded.backoff.backoff.backoff.backoff) | |
| ) | |
| def test_perceptron_tagger(self): | |
| tagger = PerceptronTagger(load=False) | |
| tagger.train(self.corpus) | |
| encoded = self.encoder.encode(tagger) | |
| decoded = self.decoder.decode(encoded) | |
| self.assertEqual(tagger.model.weights, decoded.model.weights) | |
| self.assertEqual(tagger.tagdict, decoded.tagdict) | |
| self.assertEqual(tagger.classes, decoded.classes) | |
| def test_brill_tagger(self): | |
| trainer = BrillTaggerTrainer( | |
| self.default_tagger, nltkdemo18(), deterministic=True | |
| ) | |
| tagger = trainer.train(self.corpus, max_rules=30) | |
| encoded = self.encoder.encode(tagger) | |
| decoded = self.decoder.decode(encoded) | |
| self.assertEqual(repr(tagger._initial_tagger), repr(decoded._initial_tagger)) | |
| self.assertEqual(tagger._rules, decoded._rules) | |
| self.assertEqual(tagger._training_stats, decoded._training_stats) | |