Hugging Face's logo

sakharamg
/
NMTKD

Model card Files Community
NMTKD
File size: 840 Bytes 
158b61b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
 
#pragma once

#include <string>
#include <vector>

#ifdef TOKENIZER_NAMESPACE
namespace TOKENIZER_NAMESPACE {
#endif

struct Parameters
{
    std::string lang_iso;
    std::vector<std::string> args;
    std::string out_path;
    int nthreads;
    int chunksize;
    const char *cfg_path;
    bool verbose_p;
    bool detag_p;
    bool alltag_p;
    bool entities_p;
    bool escape_p;
    bool aggro_p;
    bool supersub_p;
    bool url_p;
    bool downcase_p;
    bool normalize_p;
    bool penn_p;
    bool words_p;
    bool denumber_p;
    bool narrow_latin_p;
    bool narrow_kana_p;
    bool refined_p;
    bool unescape_p;
    bool drop_bad_p;
    bool split_p;
    bool notokenization_p;
    bool para_marks_p;
    bool split_breaks_p;

	Parameters();

    Parameters(const Parameters& _);
};


#ifdef TOKENIZER_NAMESPACE
}
#endif