mgelard commited on
Commit
1051fcd
·
verified ·
1 Parent(s): f11f568

Add tokenizer config

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +11 -6
tokenizer_config.json CHANGED
@@ -1,12 +1,17 @@
1
  {
2
- "added_tokens_decoder": {},
 
 
 
 
 
 
 
 
3
  "auto_map": {
4
  "AutoTokenizer": [
5
  "tokenizer.BinnedOmicTokenizer",
6
  null
7
  ]
8
- },
9
- "clean_up_tokenization_spaces": true,
10
- "model_max_length": 1000000000000000019884624838656,
11
- "tokenizer_class": "BinnedOmicTokenizer"
12
- }
 
1
  {
2
+ "tokenizer_class": "BinnedOmicTokenizer",
3
+ "n_expressions_bins": 64,
4
+ "min_omic_value": 0.0,
5
+ "max_omic_value": 1.0,
6
+ "use_max_normalization": true,
7
+ "normalization_factor": 5.547176906585117,
8
+ "prepend_cls_token": false,
9
+ "fixed_sequence_length": null,
10
+ "unpadded_length": null,
11
  "auto_map": {
12
  "AutoTokenizer": [
13
  "tokenizer.BinnedOmicTokenizer",
14
  null
15
  ]
16
+ }
17
+ }