Update spaCy pipeline

README.md

@@ -5,12 +5,60 @@ language:
 - en
 model-index:
 - name: en_engagement_spl_RoBERTa_acad
-  results: []
+  results:
+  - task:
+      name: NER
+      type: token-classification
+    metrics:
+    - name: NER Precision
+      type: precision
+      value: 0.0
+    - name: NER Recall
+      type: recall
+      value: 0.0
+    - name: NER F Score
+      type: f_score
+      value: 0.0
+  - task:
+      name: TAG
+      type: token-classification
+    metrics:
+    - name: TAG (XPOS) Accuracy
+      type: accuracy
+      value: 0.0
+  - task:
+      name: LEMMA
+      type: token-classification
+    metrics:
+    - name: Lemma Accuracy
+      type: accuracy
+      value: 0.0
+  - task:
+      name: UNLABELED_DEPENDENCIES
+      type: token-classification
+    metrics:
+    - name: Unlabeled Attachment Score (UAS)
+      type: f_score
+      value: 0.0
+  - task:
+      name: LABELED_DEPENDENCIES
+      type: token-classification
+    metrics:
+    - name: Labeled Attachment Score (LAS)
+      type: f_score
+      value: 0.0
+  - task:
+      name: SENTS
+      type: token-classification
+    metrics:
+    - name: Sentences F-Score
+      type: f_score
+      value: 0.9181469871
 ---
 | Feature | Description |
 | --- | --- |
 | **Name** | `en_engagement_spl_RoBERTa_acad` |
-| **Version** | `0.2.2.1228` |
+| **Version** | `0.3.0.1228` |
 | **spaCy** | `>=3.4.4,<3.5.0` |
 | **Default Pipeline** | `transformer`, `parser`, `trainable_transformer`, `spancat` |
 | **Components** | `transformer`, `parser`, `trainable_transformer`, `spancat` |
@@ -42,14 +90,14 @@ model-index:
 | `SENTS_P` | 90.27 |
 | `SENTS_R` | 93.41 |
 | `SENTS_F` | 91.81 |
-| `SPANS_SC_F` | 69.28 |
-| `SPANS_SC_P` | 72.16 |
-| `SPANS_SC_R` | 66.63 |
+| `SPANS_SC_F` | 69.58 |
+| `SPANS_SC_P` | 72.23 |
+| `SPANS_SC_R` | 67.11 |
 | `ENTS_PER_TYPE` | 0.00 |
 | `TAG_ACC` | 0.00 |
 | `ENTS_F` | 0.00 |
 | `ENTS_P` | 0.00 |
 | `ENTS_R` | 0.00 |
 | `LEMMA_ACC` | 0.00 |
-| `TRAINABLE_TRANSFORMER_LOSS` | 149.48 |
-| `SPANCAT_LOSS` | 106919.98 |
+| `TRAINABLE_TRANSFORMER_LOSS` | 344.32 |
+| `SPANCAT_LOSS` | 51942.04 |

config.cfg

@@ -10,8 +10,8 @@ seed = 0
 
 [nlp]
 lang = "en"
-pipeline = ["transformer","span_finder","spancat"]
-batch_size = 16
+pipeline = ["transformer","parser","trainable_transformer","spancat"]
+batch_size = 64
 disabled = []
 before_creation = null
 after_creation = null
@@ -20,32 +20,28 @@ tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
 
 [components]
 
-[components.span_finder]
-factory = "experimental_span_finder"
-max_length = 0
-min_length = 0
-predicted_key = "span_candidates"
-threshold = 0.2
-training_key = ${vars.spans_key}
-
-[components.span_finder.model]
-@architectures = "spacy-experimental.SpanFinder.v1"
-
-[components.span_finder.model.scorer]
-@layers = "spacy.LinearLogistic.v1"
-nO = 2
-nI = null
+[components.parser]
+factory = "parser"
+learn_tokens = false
+min_action_freq = 30
+moves = null
+scorer = {"@scorers":"spacy.parser_scorer.v1"}
+update_with_oracle_cut_size = 100
+
+[components.parser.model]
+@architectures = "spacy.TransitionBasedParser.v2"
+state_type = "parser"
+extra_state_tokens = false
+hidden_width = 64
+maxout_pieces = 2
+use_upper = false
+nO = null
 
-[components.span_finder.model.tok2vec]
+[components.parser.model.tok2vec]
 @architectures = "spacy-transformers.TransformerListener.v1"
 grad_factor = 1.0
+upstream = "transformer"
 pooling = {"@layers":"reduce_mean.v1"}
-upstream = "*"
-
-[components.span_finder.scorer]
-@scorers = "spacy-experimental.span_finder_scorer.v1"
-predicted_key = ${components.span_finder.predicted_key}
-training_key = ${vars.spans_key}
 
 [components.spancat]
 factory = "spancat"
@@ -58,8 +54,9 @@ threshold = 0.4
 @architectures = "spacy.SpanCategorizer.v1"
 
 [components.spancat.model.reducer]
-@layers = "spacy.mean_max_reducer.v1"
-hidden_size = 384
+@layers = "mean_max_reducer.v1.5"
+hidden_size = 256
+dropout = 0.2
 
 [components.spancat.model.scorer]
 @layers = "spacy.LinearLogistic.v1"
@@ -70,29 +67,51 @@ nI = null
 @architectures = "spacy-transformers.TransformerListener.v1"
 grad_factor = 1.0
 pooling = {"@layers":"reduce_mean.v1"}
-upstream = "*"
+upstream = "trainable_transformer"
 
 [components.spancat.suggester]
-@misc = "spacy-experimental.span_finder_suggester.v1"
-candidates_key = ${components.span_finder.predicted_key}
+@misc = "spacy-experimental.ngram_subtree_suggester.v1"
+sizes = [1,2,3,4,5,6,7,8,9,10,11,12]
 
-[components.transformer]
+[components.trainable_transformer]
 factory = "transformer"
 max_batch_items = 4096
 set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"}
 
-[components.transformer.model]
+[components.trainable_transformer.model]
 @architectures = "spacy-transformers.TransformerModel.v1"
 name = "egumasa/roberta-base-finetuned-academic"
 
-[components.transformer.model.get_spans]
+[components.trainable_transformer.model.get_spans]
 @span_getters = "spacy-transformers.strided_spans.v1"
-window = 169
+window = 196
 stride = 128
 
+[components.trainable_transformer.model.tokenizer_config]
+use_fast = true
+
+[components.transformer]
+factory = "transformer"
+max_batch_items = 4096
+set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"}
+
+[components.transformer.model]
+@architectures = "spacy-transformers.TransformerModel.v3"
+name = "roberta-base"
+mixed_precision = false
+
+[components.transformer.model.get_spans]
+@span_getters = "spacy-transformers.strided_spans.v1"
+window = 128
+stride = 96
+
+[components.transformer.model.grad_scaler_config]
+
 [components.transformer.model.tokenizer_config]
 use_fast = true
 
+[components.transformer.model.transformer_config]
+
 [corpora]
 
 [corpora.dev]
@@ -117,13 +136,13 @@ train_corpus = "corpora.train"
 seed = ${system.seed}
 gpu_allocator = ${system.gpu_allocator}
 dropout = 0.1
-accumulate_gradient = 1
-patience = 5000
+accumulate_gradient = 4
+patience = 6000
 max_epochs = 0
 max_steps = 20000
 eval_frequency = 200
-frozen_components = []
-annotating_components = ["span_finder"]
+frozen_components = ["transformer","parser"]
+annotating_components = ["parser"]
 before_to_disk = null
 
 [training.batcher]
@@ -134,8 +153,8 @@ get_length = null
 
 [training.batcher.size]
 @schedules = "compounding.v1"
-start = 300
-stop = 1000
+start = 100
+stop = 600
 compound = 1.0002
 t = 0.0
 
@@ -156,16 +175,25 @@ eps = 0.00000001
 [training.optimizer.learn_rate]
 @schedules = "warmup_linear.v1"
 warmup_steps = 250
-total_steps = 20000
+total_steps = 40000
 initial_rate = 0.00005
 
 [training.score_weights]
-span_finder_span_candidates_f = 0.0
-span_finder_span_candidates_p = 0.0
-span_finder_span_candidates_r = 0.2
-spans_sc_f = 0.6
+dep_uas = null
+dep_las = null
+dep_las_per_type = null
+sents_p = null
+sents_r = null
+sents_f = null
+spans_sc_f = 0.5
 spans_sc_p = 0.0
-spans_sc_r = 0.2
+spans_sc_r = 0.5
+ents_per_type = null
+tag_acc = null
+ents_f = null
+ents_p = null
+ents_r = null
+lemma_acc = null
 
 [pretraining]
 

custom_functions.py

@@ -59,7 +59,8 @@ def build_mean_max_reducer2(hidden_size: int,
             cast(Model[Ragged, Floats2d], reduce_first()),
             reduce_mean(),
             reduce_max(),
-        ), Maxout(nO=hidden_size, normalize=True, dropout=dropout),
+        ), 
+        Maxout(nO=hidden_size, normalize=True, dropout=dropout),
         Maxout(nO=hidden_size, normalize=True, dropout=dropout))
 
 

en_engagement_spl_RoBERTa_acad-any-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9060a8e1c2d993e693cb362144f58ea88767481933403c14a0f6f5ca30cd1af
-size 942481184
+oid sha256:2772317b3e8261f1ca495961af24180ca0910be23cea34ca00020666836a3fee
+size 932543299

meta.json

@@ -1,14 +1,14 @@
 {
   "lang":"en",
   "name":"engagement_spl_RoBERTa_acad",
-  "version":"0.2.2.1228",
+  "version":"0.3.0.1228",
   "description":"",
   "author":"",
   "email":"",
   "url":"",
   "license":"",
   "spacy_version":">=3.4.4,<3.5.0",
-  "spacy_git_version":"77833bfef",
+  "spacy_git_version":"Unknown",
   "vectors":{
     "width":0,
     "vectors":0,
@@ -19,7 +19,54 @@
     "transformer":[
 
     ],
-    "span_finder":[
+    "parser":[
+      "ROOT",
+      "acl",
+      "acomp",
+      "advcl",
+      "advmod",
+      "agent",
+      "amod",
+      "appos",
+      "attr",
+      "aux",
+      "auxpass",
+      "case",
+      "cc",
+      "ccomp",
+      "compound",
+      "conj",
+      "csubj",
+      "csubjpass",
+      "dative",
+      "dep",
+      "det",
+      "dobj",
+      "expl",
+      "intj",
+      "mark",
+      "meta",
+      "neg",
+      "nmod",
+      "npadvmod",
+      "nsubj",
+      "nsubjpass",
+      "nummod",
+      "oprd",
+      "parataxis",
+      "pcomp",
+      "pobj",
+      "poss",
+      "preconj",
+      "predet",
+      "prep",
+      "prt",
+      "punct",
+      "quantmod",
+      "relcl",
+      "xcomp"
+    ],
+    "trainable_transformer":[
 
     ],
     "spancat":[
@@ -39,30 +86,40 @@
   },
   "pipeline":[
     "transformer",
-    "span_finder",
+    "parser",
+    "trainable_transformer",
     "spancat"
   ],
   "components":[
     "transformer",
-    "span_finder",
+    "parser",
+    "trainable_transformer",
     "spancat"
   ],
   "disabled":[
 
   ],
   "performance":{
-    "span_finder_span_candidates_f":0.199127869,
-    "span_finder_span_candidates_p":0.1139467472,
-    "span_finder_span_candidates_r":0.7887877534,
-    "spans_sc_f":0.7057724252,
-    "spans_sc_p":0.7084201751,
-    "spans_sc_r":0.7031443939,
-    "transformer_loss":1457529913378.0666503906,
-    "span_finder_loss":622.8571459684,
-    "spancat_loss":513.4672946446
+    "dep_uas":0.0,
+    "dep_las":0.0,
+    "dep_las_per_type":0.0,
+    "sents_p":0.902739726,
+    "sents_r":0.9340892984,
+    "sents_f":0.9181469871,
+    "spans_sc_f":0.6957640751,
+    "spans_sc_p":0.722333556,
+    "spans_sc_r":0.6710798511,
+    "ents_per_type":0.0,
+    "tag_acc":0.0,
+    "ents_f":0.0,
+    "ents_p":0.0,
+    "ents_r":0.0,
+    "lemma_acc":0.0,
+    "trainable_transformer_loss":3.4431899822,
+    "spancat_loss":519.4204027318
   },
   "requirements":[
-    "spacy-experimental>=0.6.1,<0.7.0",
-    "spacy-transformers>=1.1.8,<1.2.0"
+    "spacy-transformers>=1.1.8,<1.2.0",
+    "spacy-experimental>=0.6.1,<0.7.0"
   ]
 }

spancat/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64ac1f8c1b52aadba9236caab8a5438c2cb72abf47d95248029664bb15eadd5d
-size 14183772
+oid sha256:f6760360b1c0cbf336fb1d473bd0db2ea6562f91f9bd42b0a41140ba1ad0c129
+size 9456476

trainable_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d01afabc36b8071df731d74137b8364e09c4fa6c42d0de52b0bcbd84a669581
-size 502027906
+oid sha256:aa52feb953fba36c73a8277246dd4235d80239fefa4fb4c04b6891db4cd77db0
+size 502030722

transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a76c5f803a0dc9d96af11cce2c5e9d93194c6c2b0df2bb31fc1e9c7401bdef08
-size 502027906
+oid sha256:5f3bd9c07fd7e10e50c3eea3a9f90a08f4f1d12f0ad4839b6949b3e9d3d91662
+size 502030652