Spaces:
Build error
Build error
File size: 1,234 Bytes
e62781a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
#!/usr/bin/env python3
# Copyright 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
import os
DEFAULTS = {
'corenlp_classpath': os.getenv('CLASSPATH')
}
def set_default(key, value):
global DEFAULTS
DEFAULTS[key] = value
from .corenlp_tokenizer import CoreNLPTokenizer
from .regexp_tokenizer import RegexpTokenizer
from .simple_tokenizer import SimpleTokenizer
# Spacy is optional
try:
from .spacy_tokenizer import SpacyTokenizer
except ImportError:
pass
def get_class(name):
if name == 'spacy':
return SpacyTokenizer
if name == 'corenlp':
return CoreNLPTokenizer
if name == 'regexp':
return RegexpTokenizer
if name == 'simple':
return SimpleTokenizer
raise RuntimeError('Invalid tokenizer: %s' % name)
def get_annotators_for_args(args):
annotators = set()
if args.use_pos:
annotators.add('pos')
if args.use_lemma:
annotators.add('lemma')
if args.use_ner:
annotators.add('ner')
return annotators
def get_annotators_for_model(model):
return get_annotators_for_args(model.args)
|