Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+finite.a filter=lfs diff=lfs merge=lfs -text
+lemma.a filter=lfs diff=lfs merge=lfs -text
+morph.a filter=lfs diff=lfs merge=lfs -text
+root.a filter=lfs diff=lfs merge=lfs -text
+root.ca filter=lfs diff=lfs merge=lfs -text

BUILT

@@ -0,0 +1 @@
+2024-11-28T13:37:11.407789

GIT_REV

@@ -0,0 +1 @@
+76701cc

GIT_REV_LEX

@@ -0,0 +1 @@
+76701cc

README.md

@@ -0,0 +1,409 @@
+---
+language: de
+library_name: sfst
+license: gpl-2.0
+tags:
+- sfst
+- dwdsmor
+- token-classification
+- lemmatisation
+model-index:
+- name: dwdsmor
+  results:
+  - task:
+      type: token-classification
+      name: Lemmatisation
+    dataset:
+      name: Universal Dependencies Treebank (de-hdt)
+      type: universal_dependencies
+      config: de_hdt
+      split: train
+    metrics:
+    - type: coverage
+      value: 84.15293963067323
+      name: Lemma
+    - type: coverage
+      value: 100.0
+      name: Lemma ($()
+    - type: coverage
+      value: 100.0
+      name: Lemma ($,)
+    - type: coverage
+      value: 99.99580703997988
+      name: Lemma ($.)
+    - type: coverage
+      value: 77.40301552167969
+      name: Lemma (ADJA)
+    - type: coverage
+      value: 75.48407611333322
+      name: Lemma (ADJD)
+    - type: coverage
+      value: 96.82621529723873
+      name: Lemma (ADV)
+    - type: coverage
+      value: 99.89939637826963
+      name: Lemma (APPO)
+    - type: coverage
+      value: 93.08645050358152
+      name: Lemma (APPR)
+    - type: coverage
+      value: 99.67651071695788
+      name: Lemma (APPRART)
+    - type: coverage
+      value: 79.16666666666666
+      name: Lemma (APZR)
+    - type: coverage
+      value: 99.99603964317186
+      name: Lemma (ART)
+    - type: coverage
+      value: 96.13524039049265
+      name: Lemma (CARD)
+    - type: coverage
+      value: 13.320473120462967
+      name: Lemma (FM)
+    - type: coverage
+      value: 71.42857142857143
+      name: Lemma (ITJ)
+    - type: coverage
+      value: 100.0
+      name: Lemma (KOKOM)
+    - type: coverage
+      value: 99.95274949083503
+      name: Lemma (KON)
+    - type: coverage
+      value: 100.0
+      name: Lemma (KOUI)
+    - type: coverage
+      value: 98.58579967925354
+      name: Lemma (KOUS)
+    - type: coverage
+      value: 6.1808081211782095
+      name: Lemma (NE)
+    - type: coverage
+      value: 74.40482047389456
+      name: Lemma (NN)
+    - type: coverage
+      value: 97.99275737196068
+      name: Lemma (PDAT)
+    - type: coverage
+      value: 99.95682832062167
+      name: Lemma (PDS)
+    - type: coverage
+      value: 98.79094306440976
+      name: Lemma (PIAT)
+    - type: coverage
+      value: 100.0
+      name: Lemma (PIDAT)
+    - type: coverage
+      value: 99.51910051476564
+      name: Lemma (PIS)
+    - type: coverage
+      value: 99.9888876541838
+      name: Lemma (PPER)
+    - type: coverage
+      value: 100.0
+      name: Lemma (PPOSAT)
+    - type: coverage
+      value: 100.0
+      name: Lemma (PPOSS)
+    - type: coverage
+      value: 100.0
+      name: Lemma (PRELAT)
+    - type: coverage
+      value: 100.0
+      name: Lemma (PRELS)
+    - type: coverage
+      value: 100.0
+      name: Lemma (PRF)
+    - type: coverage
+      value: 98.61938278289118
+      name: Lemma (PROAV)
+    - type: coverage
+      value: 30.821337849280273
+      name: Lemma (PTKA)
+    - type: coverage
+      value: 100.0
+      name: Lemma (PTKANT)
+    - type: coverage
+      value: 100.0
+      name: Lemma (PTKNEG)
+    - type: coverage
+      value: 77.05097087378641
+      name: Lemma (PTKVZ)
+    - type: coverage
+      value: 0.0
+      name: Lemma (PTKZU)
+    - type: coverage
+      value: 95.51166965888689
+      name: Lemma (PWAT)
+    - type: coverage
+      value: 99.37264742785446
+      name: Lemma (PWAV)
+    - type: coverage
+      value: 99.46524064171123
+      name: Lemma (PWS)
+    - type: coverage
+      value: 100.0
+      name: Lemma (VAFIN)
+    - type: coverage
+      value: 100.0
+      name: Lemma (VAIMP)
+    - type: coverage
+      value: 100.0
+      name: Lemma (VAINF)
+    - type: coverage
+      value: 100.0
+      name: Lemma (VAPP)
+    - type: coverage
+      value: 100.0
+      name: Lemma (VMFIN)
+    - type: coverage
+      value: 100.0
+      name: Lemma (VMINF)
+    - type: coverage
+      value: 100.0
+      name: Lemma (VMPP)
+    - type: coverage
+      value: 88.6487187323461
+      name: Lemma (VVFIN)
+    - type: coverage
+      value: 95.96122778675283
+      name: Lemma (VVIMP)
+    - type: coverage
+      value: 82.1453501900256
+      name: Lemma (VVINF)
+    - type: coverage
+      value: 82.9683698296837
+      name: Lemma (VVIZU)
+    - type: coverage
+      value: 79.96866513473992
+      name: Lemma (VVPP)
+    - type: coverage
+      value: 41.48471615720524
+      name: Lemma (XY)
+---
+
+# DWDSmor
+
+_SFST/SMOR/DWDS-based German morphology_
+
+
+
+
+
+DWDSmor implements the lemmatisation and morphological analysis of
+word forms as well as the generation of paradigms of lexical words in
+written German.
+
+## Usage
+
+DWDSmor is available via PyPI:
+
+``` plaintext
+pip install dwdsmor
+```
+
+For lemmatisation:
+
+``` python-console
+>>> import dwsdmor
+>>> lemmatizer = dwdsmor.lemmatizer()
+>>> assert lemmatizer("getestet", pos={"+V"}) == "testen"
+>>> assert lemmatizer("getestet", pos={"+ADJ"}) == "getestet"
+```
+
+…
+
+## Development
+
+This repository provides source code for building DWDSmor lexica and transducers
+as well as for using DWDSmor transducers for morphological analysis and paradigm
+generation:
+
+* `dwdsmor/` contains Python packages for using DWDSmor, including
+  scripts for morphological analysis and for paradigm generation by
+  means of DWDSmor transducers.
+* `share/` contains XSLT stylesheets for extracting lexical entries in SMORLemma
+  format form XML sources of DWDS articles. Sample inputs and outputs can be
+  found in `samples/`.
+* `lexicon/dwds/` contains scripts for building DWDSmor lexica by means of the
+  XSLT stylesheets in `share/` and DWDS sources in `lexicon/dwds/wb/`, which are
+  not part of this repository.
+* `lexicon/sample/` contains scripts for building sample DWDSmor lexica by means
+  of the XSLT stylesheets in `share/` and the sample lexicon in
+  `lexicon/sample/wb/`.
+* `grammar/` contains an FST grammar derived from SMORLemma, providing the
+  morphology for building DWDSmor automata from DWDSmor lexica.
+* `test/` implements a test suite for the DWDSmor transducers.
+
+DWDSmor is in active development. In its current stage, DWDSmor supports most
+inflection classes and some productive word-formation patterns of written
+German. Note that the sample lexicon in `lexicon/sample/wb/` only covers a
+sketchy subset of the German vocabulary, and so do the DWDSmor automata compiled
+from it.
+
+
+## Prerequisites
+
+[GNU/Linux](https://www.debian.org/)
+: Development, builds and tests of DWDSmor are performed
+  on [Debian GNU/Linux](https://debian.org/). While other UNIX-like operating
+  systems such as MacOS should work, too, they are not actively supported.
+
+[Python >= v3.9](https://www.python.org/)
+: DWDSmor targets Python as its primary runtime environment. The DWDSmor
+  transducers can be used via SFST's commandline tools, queried in Python
+  applications via language-specific
+  [bindings](https://github.com/gremid/sfst-transduce), or used by the Python
+  scripts `dwdsmor.py` and `paradigm.py` for morphological analysis and for
+  paradigm generation.
+
+[Saxon-HE](https://www.saxonica.com/)
+: The extraction of lexical entries from XML sources of DWDS articles is
+  implemented in XSLT 2, for which Saxon-HE is used as the runtime environment.
+
+[Java (JDK) >= v8](https://openjdk.java.net/)
+:  Saxon requires a Java runtime.
+
+[SFST](https://www.cis.uni-muenchen.de/~schmid/tools/SFST/)
+: a C++ library and toolbox for finite-state transducers (FSTs); please take a
+  look at its homepage for installation and usage instructions.
+
+On a Debian-based distribution, install the following packages:
+
+```sh
+apt install python3 default-jdk libsaxonhe-java sfst
+```
+
+Set up a virtual environment for project builds, for example via Python's `venv`:
+
+```sh
+python3 -m venv .venv
+source .venv/bin/activate
+```
+
+Then run the DWDSmor setup routine in order to install Python dependencies:
+
+```sh
+pip install -e .[dev]
+```
+
+
+## Building DWDSmor lexica and transducers
+
+For building DWDSmor lexica and transducers, run:
+
+```sh
+make all
+```
+
+Alternatively, you can run:
+
+```sh
+make dwds && make dwds-install && make dwdsmor
+```
+
+Note that these commands require DWDS sources in `lexicon/dwds/wb/`, which are
+not part of this repository.
+
+Alternatively, you can build sample DWDSmor lexica and transducers from the
+sample lexicon in `lexicon/sample/wb/` by running:
+
+```sh
+make sample && make sample-install && make dwdsmor
+```
+
+After building DWDSmor transducers, install them into `lib/`, where the
+Python scripts `dwdsmor` and `dwdsmor-paradigm` expect them by default:
+
+```sh
+make install
+```
+
+The installed DWDSmor transducers are:
+
+* `lib/dwdsmor.{a,ca}`: transducer with inflection and word-formation
+  components, for lemmatisation and morphological analysis of word forms in
+  terms of grammatical categories
+* `lib/dwdsmor-morph.{a,ca}`: transducer with inflection and word-formation
+  components, for the generation of morphologically segmented word forms
+* `lib/dwdsmor-finite.{a,ca}`: transducer with an inflection component and a
+  finite word-formation component, for testing purposes
+* `lib/dwdsmor-root.{a,ca}`: transducer with inflection and word-formation
+  components, for lexical analysis of word forms in terms of root lemmas (i.e.,
+  lemmas of ultimate word-formation bases), word-formation process,
+  word-formation means, and grammatical categories in term of the
+  Pattern-and-Restriction Theory of word formation (Nolda 2022)
+* `lib/dwdsmor-index.{a,ca}`: transducer with an inflection component only with
+  DWDS homographic lemma indices, for paradigm generation
+
+
+## Testing DWDSmor
+
+Run
+
+    pytest
+
+in order to test basic transducer usage and for potential regressions.
+
+## Contact
+
+Feel free to contact [Andreas Nolda](mailto:[email protected]) for
+questions regarding the lexicon or the grammar and
+[Gregor Middell](mailto:[email protected]) for question related
+to the integration of DWDSmor into your corpus-annotation pipeline.
+
+
+## License
+
+As the original SMOR and SMORLemma grammars, the DWDSmor grammar is
+licensed under the GNU General Public Licence v2.0. The same applies
+to the rest of this project.
+
+## Credits
+
+DWSDmor is based on the following software and datasets:
+
+1. [SFST](https://www.cis.uni-muenchen.de/~schmid/tools/SFST/), a C++ library
+   and toolbox for finite-state transducers (FSTs) (Schmidt 2006)
+2. [SMORLemma](https://github.com/rsennrich/SMORLemma) (Sennrich and Kunz 2014),
+   a modified version of the Stuttgart Morphology
+   ([SMOR](https://www.cis.lmu.de/~schmid/tools/SMOR/)) (Schmid, Fitschen, and
+   Heid 2004) with an alternative lemmatisation component
+3. the [DWDS dictionary](https://www.dwds.de/) (BBAW n.d.) replacing the
+   [IMSLex](https://www.ims.uni-stuttgart.de/forschung/ressourcen/lexika/imslex/)
+   (Fitschen 2004) as the lexical data source for German words, their grammatical
+   categories, and their morphological properties.
+
+## Bibliography
+
+* Berlin-Brandenburg Academy of Sciences and Humanities (BBAW) (ed.) (n.d.).
+  DWDS – Digitales Wörterbuch der deutschen Sprache: Das Wortauskunftssystem zur
+  deutschen Sprache in Geschichte und Gegenwart.
+  https://www.dwds.de
+* Fitschen, Arne (2004). Ein computerlinguistisches Lexikon als komplexes
+  System. Ph.D. thesis, Universität Stuttgart.
+  [PDF](http://www.ims.uni-stuttgart.de/forschung/ressourcen/lexika/IMSLex/fitschendiss.pdf)
+* Nolda, Andreas (2022). Headedness as an epiphenomenon: Case studies on
+  compounding and blending in German. In *Headedness and/or Grammatical
+  Anarchy?*, ed. by Ulrike Freywald, Horst Simon, and Stefan Müller, Empirically
+  Oriented Theoretical Morphology and Syntax 11, Berlin: Language Science Press,
+  343–376.
+  [PDF](https://zenodo.org/record/7142720/files/336-FreywaldSimonMüller-2022-11.pdf).
+* Schmid, Helmut (2006). A programming language for finite state transducers. In
+  *Finite-State Methods and Natural Language Processing: 5th International
+  Workshop, FSMNLP 2005, Helsinki, Finland, September 1–2, 2005*, ed. by Anssi
+  Yli-Jyrä, Lauri Karttunen, and Juhani Karhumäki, Lecture Notes in Artificial
+  Intelligence 4002, Berlin: Springer, 1263–1266.
+  [PDF](https://www.cis.uni-muenchen.de/~schmid/papers/SFST-PL.pdf).
+* Schmid, Helmut, Arne Fitschen, and Ulrich Heid (2004). SMOR: A German
+  computational morphology covering derivation, composition, and inflection. In
+  LREC 2004: Fourth International Conference on Language Resources and
+  Evaluation, ed. by Maria T. Lino *et al.*, European Language Resources
+  Association, 1263–1266.
+  [PDF](http://www.lrec-conf.org/proceedings/lrec2004/pdf/468.pdf)
+* Sennrich, Rico and Beta Kunz (2014). Zmorge: A German morphological lexicon
+  extracted from Wiktionary. In LREC 2014: Ninth International Conference on
+  Language Resources and Evaluation, ed. by Nicoletta Calzolari *et al.*,
+  European Language Resources Association, 1063–1067.
+  [PDF](http://www.lrec-conf.org/proceedings/lrec2014/pdf/116_Paper.pdf).

finite.a

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a90dbeb10d36b610bb58a58c8720d1d0793508459796f38e85877a9b65b78316
+size 1134309

lemma.a

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b41ad7a8e276d80c356cb29e2a2d4a71e0ab3947a407274174b24c9420ce86f
+size 1233648

morph.a

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11153b2ad849789b455ba27f7c801604007b38e6e4eba223184d855268fa039c
+size 1241182

root.a

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de16583efbd2441ca98d843cb2bd519bf4c925ac405d65396b6572a4bd3c9bdf
+size 6980498

root.ca

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c73e66ea1433a035e929e0d06d7081a6b03a2c5d60b74f5950d99f98871a55c7
+size 3632222