init

README.md

@@ -0,0 +1,70 @@
+---
+language:
+  - ar
+
+thumbnail: "url to a thumbnail used in social sharing"
+tags:
+  - ner
+  - token-classification
+  - Arabic-NER
+
+metrics:
+  - accuracy
+  - f1
+  - precision
+  - recall
+
+widget:
+  - text: "النجم محمد صلاح لاعب المنتخب المصري يعيش في مصر بالتحديد من نجريج, الشرقية"
+    example_title: "Mohamed Salah"
+  - text: "انا ساكن في حدايق الزتون و بدرس في جامعه عين شمس"
+    example_title: "Egyptian Dialect"
+  - text: "يقع نهر الأمازون في قارة أمريكا الجنوبية"
+    example_title: "Standard Arabic"
+
+datasets:
+  - Fine-grained-Arabic-Named-Entity-Corpora
+---
+
+# Arabic Named Entity Recognition
+
+This project is made to enrich the Arabic Named Entity Recognition(ANER). Arabic is a tough language to deal with and has alot of difficulties.
+We managed to made a model based on Arabert to support 50 entities.
+
+## Paper
+
+Here's the paper that contains all the details for our model, our approach, and the training results
+
+- [ANER Paper](https://drive.google.com/file/d/1jJn3iWqOeLzaNvO-6aKfgidzJlWOtvti/view?usp=sharing)
+
+# Usage
+
+The model is available in HuggingFace model page under the name: [boda/ANER](https://huggingface.co/boda/ANER). Checkpoints are available only in PyTorch at the time.
+
+### Use in python:
+
+```python
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+
+tokenizer = AutoTokenizer.from_pretrained("boda/ANER")
+
+model = AutoModelForTokenClassification.from_pretrained("boda/ANER")
+```
+
+# Dataset
+
+- [Fine-grained Arabic Named Entity Corpora](https://fsalotaibi.kau.edu.sa/Pages-Arabic-NE-Corpora.aspx)
+
+# Acknowledgments
+
+Thanks for [Arabert](https://github.com/aub-mind/arabert) for providing the Arabic Bert model, which we used as a base model for our work.
+
+We also would like to thank [Prof. Fahd Saleh S Alotaibi](https://fsalotaibi.kau.edu.sa/Pages-Arabic-NE-Corpora.aspx) at Faculty of Computing and Information Technology King Abdulaziz University, for providing the dataset which we used to train our model with.
+
+# Contacts
+
+**Abdelrahman Atef**
+
+- [LinkedIn](linkedin.com/in/boda-sadalla)
+- [Github](https://github.com/BodaSadalla98)
+- <[email protected]>

config.json

@@ -0,0 +1,182 @@
+{
+  "_name_or_path": ".",
+  "architectures": [
+    "BertForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "O",
+    "1": " \u0641\u0646\u0627\u0646  ",
+    "2": " \u0641\u0646\u0627\u0646  ",
+    "3": " \u0635\u0648\u062a  ",
+    "4": " \u0635\u0648\u062a  ",
+    "5": " \u062a\u0639\u0644\u064a\u0645\u064a  ",
+    "6": " \u062a\u0639\u0644\u064a\u0645\u064a  ",
+    "7": " \u0623\u0631\u0627\u0636\u064a \u0627\u0644\u0628\u0646\u0627\u0621  ",
+    "8": " \u0623\u0631\u0627\u0636\u064a \u0627\u0644\u0628\u0646\u0627\u0621  ",
+    "9": " \u0645\u0631\u0643\u0632 \u0633\u0643\u0646\u064a  ",
+    "10": " \u0634\u0639\u0628(\u0623\u0645\u0629)  ",
+    "11": " \u0648\u0644\u0627\u064a\u0629 \u0623\u0648 \u0645\u0642\u0627\u0637\u0639\u0629  ",
+    "12": " \u0648\u0644\u0627\u064a\u0629 \u0623\u0648 \u0645\u0642\u0627\u0637\u0639\u0629  ",
+    "13": " \u0645\u0633\u0637\u062d \u0645\u0627\u0626\u064a  ",
+    "14": " \u0645\u0633\u0637\u062d \u0645\u0627\u0626\u064a  ",
+    "15": " \u0623\u0631\u0636 \u0637\u0628\u064a\u0639\u064a\u0629  ",
+    "16": " \u0623\u0631\u0636 \u0637\u0628\u064a\u0639\u064a\u0629  ",
+    "17": " \u0633\u0648\u0641\u062a\u0648\u064a\u0631(\u0628\u0631\u0645\u062c\u064a\u0627\u062a)  ",
+    "18": " \u0633\u0648\u0641\u062a\u0648\u064a\u0631(\u0628\u0631\u0645\u062c\u064a\u0627\u062a)  ",
+    "19": " \u0639\u0627\u0644\u0645  ",
+    "20": " \u0643\u062a\u0627\u0628  ",
+    "21": " \u0643\u062a\u0627\u0628  ",
+    "22": " \u0639\u0627\u0644\u0645  ",
+    "23": " \u0645\u062c\u0645\u0648\u0639\u0629  ",
+    "24": " \u0633\u0645\u0627\u0648\u064a  ",
+    "25": " \u0634\u0631\u0637\u0629  ",
+    "26": " \u0634\u0631\u0637\u0629  ",
+    "27": " \u0645\u0631\u0643\u0632 \u0633\u0643\u0646\u064a  ",
+    "28": " \u0633\u0645\u0627\u0648\u064a  ",
+    "29": " \u0645\u0647\u0646\u062f\u0633  ",
+    "30": " \u0645\u0647\u0646\u062f\u0633  ",
+    "31": " \u0642\u0630\u064a\u0641\u0629  ",
+    "32": " \u062d\u0643\u0648\u0645\u0629  ",
+    "33": " \u062d\u0643\u0648\u0645\u0629  ",
+    "34": " \u062a\u062c\u0627\u0631\u064a  ",
+    "35": " \u062a\u062c\u0627\u0631\u064a  ",
+    "36": " \u0642\u0627\u0631\u0629  ",
+    "37": " \u0647\u0648\u0627\u0621  ",
+    "38": " \u0647\u0648\u0627\u0621  ",
+    "39": " \u0634\u062e\u0635  ",
+    "40": " \u0634\u062e\u0635  ",
+    "41": " \u0645\u062c\u0645\u0648\u0639\u0629  ",
+    "42": " \u0633\u064a\u0627\u0633\u064a  ",
+    "43": " \u0633\u064a\u0627\u0633\u064a  ",
+    "44": " \u0631\u064a\u0627\u0636\u064a  ",
+    "45": " \u0631\u064a\u0627\u0636\u064a  ",
+    "46": " \u0645\u0624\u0633\u0633\u0629 \u062f\u064a\u0646\u064a\u0629  ",
+    "47": " \u0645\u0624\u0633\u0633\u0629 \u062f\u064a\u0646\u064a\u0629  ",
+    "48": " \u0637\u0631\u064a\u0642  ",
+    "49": " \u0637\u0631\u064a\u0642  ",
+    "50": " \u0625\u0639\u0644\u0627\u0645  ",
+    "51": " \u0625\u0639\u0644\u0627\u0645  ",
+    "52": " \u063a\u064a\u0631 \u062d\u0643\u0648\u0645\u064a  ",
+    "53": " \u063a\u064a\u0631 \u062d\u0643\u0648\u0645\u064a  ",
+    "54": " \u0645\u062f\u064a\u0646\u0629 \u0623\u0648 \u0636\u0627\u062d\u064a\u0629  ",
+    "55": " \u0645\u062f\u064a\u0646\u0629 \u0623\u0648 \u0636\u0627\u062d\u064a\u0629  ",
+    "56": " \u0631\u062c\u0644 \u0623\u0639\u0645\u0627\u0644  ",
+    "57": " \u0645\u062d\u0627\u0645\u064a  ",
+    "58": " \u0645\u062d\u0627\u0645\u064a  ",
+    "59": "   ",
+    "60": "   ",
+    "61": " \u0634\u0639\u0628(\u0623\u0645\u0629)  ",
+    "62": " \u0634\u062e\u0635 \u062f\u064a\u0646\u064a  ",
+    "63": " \u0634\u062e\u0635 \u062f\u064a\u0646\u064a  ",
+    "64": " \u0631\u062c\u0644 \u0623\u0639\u0645\u0627\u0644  ",
+    "65": " \u0639\u0644\u0648\u0645 \u0637\u0628\u064a\u0629  ",
+    "66": " \u0639\u0644\u0648\u0645 \u0637\u0628\u064a\u0629  ",
+    "67": " \u0641\u064a\u0644\u0645  ",
+    "68": " \u0641\u064a\u0644\u0645  ",
+    "69": " \u0645\u0627\u0621  ",
+    "70": " \u0645\u0627\u0621  ",
+    "71": " \u062f\u0648\u0627\u0621  ",
+    "72": " \u0639\u062a\u0627\u062f  ",
+    "73": " \u0639\u062a\u0627\u062f  ",
+    "74": " \u0645\u0646\u0634\u0623\u0629 \u0645\u0646\u0637\u0642\u0629 \u0641\u0631\u0639\u064a\u0629  ",
+    "75": " \u0645\u0646\u0634\u0623\u0629 \u0645\u0646\u0637\u0642\u0629 \u0641\u0631\u0639\u064a\u0629  ",
+    "76": " \u0641\u0638  ",
+    "77": " \u0645\u0637\u0627\u0631  ",
+    "78": " \u0641\u0638  ",
+    "79": " \u062f\u0648\u0627\u0621  ",
+    "80": " \u0631\u064a\u0627\u0636\u0629  ",
+    "81": " \u0631\u064a\u0627\u0636\u0629  ",
+    "82": " \u0631\u0645\u0627\u064a\u0629  ",
+    "83": " \u0631\u0645\u0627\u064a\u0629  ",
+    "84": " \u0637\u0639\u0627\u0645  ",
+    "85": " \u0637\u0639\u0627\u0645  ",
+    "86": " \u0642\u0627\u0631\u0629  ",
+    "87": " \u0646\u0648\u0648\u064a  ",
+    "88": " \u0646\u0648\u0648\u064a  ",
+    "89": " \u062a\u0631\u0641\u064a\u0647  ",
+    "90": " \u062a\u0631\u0641\u064a\u0647  ",
+    "91": " \u0642\u0630\u064a\u0641\u0629  ",
+    "92": " \u0623\u0631\u0636  ",
+    "93": " \u062d\u0627\u062f  ",
+    "94": " \u0645\u0637\u0627\u0631  ",
+    "95": " \u0623\u0631\u0636  ",
+    "96": " \u0646\u0628\u0627\u062a  ",
+    "97": " \u0646\u0628\u0627\u062a  ",
+    "98": " \u0645\u0646\u0641\u062c\u0631  ",
+    "99": " \u0645\u0646\u0641\u062c\u0631  ",
+    "100": " \u0643\u064a\u0645\u064a\u0627\u0626\u064a  ",
+    "101": " \u0643\u064a\u0645\u064a\u0627\u0626\u064a  "
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "   ": 60,
+    " \u0623\u0631\u0627\u0636\u064a \u0627\u0644\u0628\u0646\u0627\u0621  ": 8,
+    " \u0623\u0631\u0636  ": 95,
+    " \u0623\u0631\u0636 \u0637\u0628\u064a\u0639\u064a\u0629  ": 16,
+    " \u0625\u0639\u0644\u0627\u0645  ": 51,
+    " \u062a\u062c\u0627\u0631\u064a  ": 35,
+    " \u062a\u0631\u0641\u064a\u0647  ": 90,
+    " \u062a\u0639\u0644\u064a\u0645\u064a  ": 6,
+    " \u062d\u0627\u062f  ": 93,
+    " \u062d\u0643\u0648\u0645\u0629  ": 33,
+    " \u062f\u0648\u0627\u0621  ": 79,
+    " \u0631\u062c\u0644 \u0623\u0639\u0645\u0627\u0644  ": 64,
+    " \u0631\u0645\u0627\u064a\u0629  ": 83,
+    " \u0631\u064a\u0627\u0636\u0629  ": 81,
+    " \u0631\u064a\u0627\u0636\u064a  ": 45,
+    " \u0633\u0645\u0627\u0648\u064a  ": 28,
+    " \u0633\u0648\u0641\u062a\u0648\u064a\u0631(\u0628\u0631\u0645\u062c\u064a\u0627\u062a)  ": 18,
+    " \u0633\u064a\u0627\u0633\u064a  ": 43,
+    " \u0634\u062e\u0635  ": 40,
+    " \u0634\u062e\u0635 \u062f\u064a\u0646\u064a  ": 63,
+    " \u0634\u0631\u0637\u0629  ": 26,
+    " \u0634\u0639\u0628(\u0623\u0645\u0629)  ": 61,
+    " \u0635\u0648\u062a  ": 4,
+    " \u0637\u0631\u064a\u0642  ": 49,
+    " \u0637\u0639\u0627\u0645  ": 85,
+    " \u0639\u0627\u0644\u0645  ": 22,
+    " \u0639\u062a\u0627\u062f  ": 73,
+    " \u0639\u0644\u0648\u0645 \u0637\u0628\u064a\u0629  ": 66,
+    " \u063a\u064a\u0631 \u062d\u0643\u0648\u0645\u064a  ": 53,
+    " \u0641\u0638  ": 78,
+    " \u0641\u0646\u0627\u0646  ": 2,
+    " \u0641\u064a\u0644\u0645  ": 68,
+    " \u0642\u0627\u0631\u0629  ": 86,
+    " \u0642\u0630\u064a\u0641\u0629  ": 91,
+    " \u0643\u062a\u0627\u0628  ": 21,
+    " \u0643\u064a\u0645\u064a\u0627\u0626\u064a  ": 101,
+    " \u0645\u0624\u0633\u0633\u0629 \u062f\u064a\u0646\u064a\u0629  ": 47,
+    " \u0645\u0627\u0621  ": 70,
+    " \u0645\u062c\u0645\u0648\u0639\u0629  ": 41,
+    " \u0645\u062d\u0627\u0645\u064a  ": 58,
+    " \u0645\u062f\u064a\u0646\u0629 \u0623\u0648 \u0636\u0627\u062d\u064a\u0629  ": 55,
+    " \u0645\u0631\u0643\u0632 \u0633\u0643\u0646\u064a  ": 27,
+    " \u0645\u0633\u0637\u062d \u0645\u0627\u0626\u064a  ": 14,
+    " \u0645\u0637\u0627\u0631  ": 94,
+    " \u0645\u0646\u0634\u0623\u0629 \u0645\u0646\u0637\u0642\u0629 \u0641\u0631\u0639\u064a\u0629  ": 75,
+    " \u0645\u0646\u0641\u062c\u0631  ": 99,
+    " \u0645\u0647\u0646\u062f\u0633  ": 30,
+    " \u0646\u0628\u0627\u062a  ": 97,
+    " \u0646\u0648\u0648\u064a  ": 88,
+    " \u0647\u0648\u0627\u0621  ": 38,
+    " \u0648\u0644\u0627\u064a\u0629 \u0623\u0648 \u0645\u0642\u0627\u0637\u0639\u0629  ": 12,
+    "O": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.5.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 64000
+}

helpers/download_model.py

@@ -0,0 +1,32 @@
+import requests
+
+def download_file_from_google_drive(id, destination):
+    URL = "https://docs.google.com/uc?export=download"
+
+    session = requests.Session()
+
+    response = session.get(URL, params = { 'id' : id }, stream = True)
+    token = get_confirm_token(response)
+
+    if token:
+        params = { 'id' : id, 'confirm' : token }
+        response = session.get(URL, params = params, stream = True)
+
+    save_response_content(response, destination)    
+
+def get_confirm_token(response):
+    for key, value in response.cookies.items():
+        if key.startswith('download_warning'):
+            return value
+
+    return None
+
+def save_response_content(response, destination):
+    CHUNK_SIZE = 32768
+
+    with open(destination, "wb") as f:
+        for chunk in response.iter_content(CHUNK_SIZE):
+            if chunk: # filter out keep-alive new chunks
+                f.write(chunk)
+
+

helpers/helper.py

@@ -0,0 +1,180 @@
+from numpy import string_
+import re
+
+en_to_ar_camel = {
+    'B-LOC' : 'مكان',
+    'B-ORG': 'مؤسسة', 
+    'B-PERS': 'شخص',  
+    'B-MISC': 'معنى بموضوعات متنوعة', 
+    'I-LOC': 'مكان', 
+    'I-ORG': 'مؤسسة', 
+    'I-PERS': 'شحص',   
+    'I-MISC': 'معنى بموضوعات متنوعة', 
+}
+
+en_to_ar = {
+    "B-Artist" : "فنان",
+    "I-Artist" :"فنان",
+    "B-Sound": "صوت",
+    "I-Sound":"صوت",
+    "B-Educational": "تعليمي",
+    "I-Educational":"تعليمي",
+    "B-Building-Grounds":"أراضي البناء",
+    "I-Building-Grounds":"أراضي البناء",
+    "B-Population-Center":"مركز سكني",
+    "B-Nation":"شعب(أمة)",
+    "B-State-or-Province":"ولاية أو مقاطعة",
+    "I-State-or-Province": "ولاية أو مقاطعة",
+    "B-Water-Body": "مسطح مائي",
+    "I-Water-Body":"مسطح مائي",
+    "B-Land-Region-Natural": "أرض طبيعية",
+    "I-Land-Region-Natural":"أرض طبيعية",
+    "B-Software":"سوفتوير(برمجيات)",
+    "I-Software":"سوفتوير(برمجيات)",
+    "B-Scientist": "عالم",
+    "B-Book":"كتاب",
+    "I-Book":"كتاب",
+    "I-Scientist":"عالم",
+    "B-Group":"مجموعة",
+    "B-Celestial":"سماوي",
+    "B-Police":"شرطة",
+    "I-Police":"شرطة",
+    "I-Population-Center":"مركز سكني",
+    "I-Celestial":"سماوي",
+    "B-Engineer":"مهندس",
+    "I-Engineer":"مهندس",
+    "B-Projectile":"قذيفة",
+    "B-Government":"حكومة",
+    "I-Government":"حكومة",
+    "B-Commercial":"تجاري",
+    "I-Commercial":"تجاري",
+    "B-Continent":"قارة",
+    "B-Air":"هواء",
+    "I-Air":"هواء",
+    "B-Other_PER":"شخص",
+    "I-Other_PER":"شخص",
+    "I-Group":"مجموعة",
+    "B-Politician":"سياسي",
+    "I-Politician":"سياسي",
+    "B-Athlete":"رياضي",
+    "I-Athlete":"رياضي",
+    "B-Religious_ORG":"مؤسسة دينية",
+    "I-Religious_ORG":"مؤسسة دينية",
+    "B-Path":"طريق",
+    "I-Path":"طريق",
+    "B-Media":"إعلام",
+    "I-Media":"إعلام",
+    "B-Non-Governmental":"غير حكومي",
+    "I-Non-Governmental":"غير حكومي",
+    "B-County-or-District":"مدينة أو ضاحية",
+    "I-County-or-District":"مدينة أو ضاحية",
+    "B-Businessperson":"رجل أعمال",
+    "B-Lawyer":"محامي",
+    "I-Lawyer":"محامي",
+    "B-GPE-Cluster":"",
+    "I-GPE-Cluster":"",
+    "I-Nation":"شعب(أمة)",
+    "B-Religious_PER":"شخص ديني",
+    "I-Religious_PER":"شخص ديني",
+    "I-Businessperson":"رجل أعمال",
+    "B-Medical-Science":"علوم طبية",
+    "I-Medical-Science":"علوم طبية",
+    "B-Movie":"فيلم",
+    "I-Movie":"فيلم",
+    "B-Water":"ماء",
+    "I-Water":"ماء",
+    "B-Drug":"دواء",
+    "B-Hardware":"عتاد",
+    "I-Hardware":"عتاد",
+    "B-Subarea-Facility":"منشأة منطقة فرعية",
+    "I-Subarea-Facility":"منشأة منطقة فرعية",
+    "B-Blunt":"فظ",
+    "B-Airport":"مطار",
+    "I-Blunt": "فظ",
+    "I-Drug":"دواء",
+    "B-Sports":"رياضة",
+    "I-Sports":"رياضة",
+    "B-Shooting":"رماية",
+    "I-Shooting":"رماية",
+    "B-Food":"طعام",
+    "I-Food":"طعام",
+    "I-Continent":"قارة",
+    "B-Nuclear":"نووي",
+    "I-Nuclear":"نووي",
+    "B-Entertainment":"ترفيه",
+    "I-Entertainment":"ترفيه",
+    "I-Projectile":"قذيفة",
+    "B-Land":"أرض",
+    "B-Sharp":"حاد",
+    "I-Airport":"مطار",
+    "I-Land":"أرض",
+    "B-Plant":"نبات",
+    "I-Plant":"نبات",
+    "B-Exploding":"منفجر",
+    "I-Exploding":"منفجر",
+    "B-Chemical":"كيميائي",
+    "I-Chemical": "كيميائي",
+}
+
+
+
+
+def get_separate_entities(labels, tokens):
+    """
+        takes labels and token , return full name entity (mohamed, salah --> "mohamed salah")
+        this will be used to search in wikipedia
+    """
+    res = []                                          
+    b_before = False
+    temp = ""
+    key_value = ()
+    for i in range(len(labels)):
+        print(res)
+        curr = labels[i]
+        
+        if("B-" in curr):
+            if(b_before):
+                key_value = (temp[:-1], 1)
+                res.append(key_value)
+                temp = tokens[i] + ' '
+            else:
+                b_before = True
+                temp += tokens[i] + ' '
+                if(i == len(labels)-1):
+                    key_value = (temp[:-1], 1)
+                    res.append(key_value)
+                # print("temp is:" + str(temp))
+
+        elif("I-" in curr):
+            temp += tokens[i] + ' '
+            if(i == len(labels)-1):
+                key_value = (temp[:-1], 1)
+                res.append(key_value)
+
+        else:
+            if(temp == ""):
+                key_value = (tokens[i], 0)
+                res.append(key_value) 
+            else:
+                key_value = (temp[:-1], 1)
+                res.append(key_value)
+                key_value = (tokens[i], 0)
+                res.append(key_value) 
+                temp = ""
+                b_before = False
+    
+   
+
+    print(res)
+    return res 
+
+
+
+
+
+
+
+
+
+
+

main.py

@@ -0,0 +1,22 @@
+from transformers import AutoModelForTokenClassification, AutoTokenizer
+from helpers import   helper
+
+
+MODEL_NAME = 'boda/ANER'
+
+
+# Load model and tokenizer
+model = AutoModelForTokenClassification.from_pretrained('.')
+tokenizer = AutoTokenizer.from_pretrained('.')
+
+ # change in the model labels
+# model.config.id2label = {i: ' '+v+' ' for i, v in model.config.id2label.items()  if i != 0 }
+# model.config.id2label[0] = 'O'
+# model.config.label2id = {label: i for i, label in model.config.id2label.items()}
+
+
+
+
+# save model after finish 
+# model.save_pretrained('.')
+

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "max_len": 512, "do_basic_tokenize": true, "never_split": ["[بريد]", "[مستخدم]", "[رابط]"], "special_tokens_map_file": null, "name_or_path": "aubmindlab/bert-base-arabertv02"}