Delete Model/MultimodelNER

Model/MultimodelNER/MNER_2016.py

@@ -1,106 +0,0 @@
-import streamlit as st
-from spacy import displacy
-from Model.NER.VLSP2021.Predict_Ner import ViTagger
-import re
-from thunghiemxuly import save_uploaded_image,convert_text_to_txt,add_string_to_txt
-
-import os
-from transformers import AutoTokenizer, BertConfig
-from Model.MultimodelNER.VLSP2016.train_umt_2016 import load_model,predict
-from Model.MultimodelNER.Ner_processing import format_predictions,process_predictions,combine_entities,remove_B_prefix,combine_i_tags
-
-from Model.MultimodelNER.predict import get_test_examples_predict
-from Model.MultimodelNER import resnet as resnet
-from Model.MultimodelNER.resnet_utils import myResnet
-import torch
-import numpy as np
-from Model.MultimodelNER.VLSP2016.dataset_roberta import MNERProcessor_2016
-
-
-CONFIG_NAME = 'bert_config.json'
-WEIGHTS_NAME = 'pytorch_model.bin'
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-
-net = getattr(resnet, 'resnet152')()
-net.load_state_dict(torch.load(os.path.join('E:/demo_datn/pythonProject1/Model/Resnet/', 'resnet152.pth')))
-encoder = myResnet(net, True, device)
-def process_text(text):
-    # Loại bỏ dấu cách thừa và dấu cách ở đầu và cuối văn bản
-    processed_text = re.sub(r'\s+', ' ', text.strip())
-    return processed_text
-
-
-
-def show_mner_2016():
-    multimodal_text = st.text_area("Enter your text for MNER:", height=300)
-    multimodal_text = process_text(multimodal_text)  # Xử lý văn bản
-    image = st.file_uploader("Upload an image (only jpg):", type=["jpg"])
-    if st.button("Process Multimodal NER"):
-            save_image = 'E:/demo_datn/pythonProject1/Model/MultimodelNER/VLSP2016/Image'
-            save_txt = 'E:/demo_datn/pythonProject1/Model/MultimodelNER/VLSP2016/Filetxt/test.txt'
-            image_name = image.name
-            save_uploaded_image(image, save_image)
-            convert_text_to_txt(multimodal_text, save_txt)
-            add_string_to_txt(image_name, save_txt)
-            st.image(image, caption="Uploaded Image", use_column_width=True)
-
-            bert_model='vinai/phobert-base-v2'
-            output_dir='E:/demo_datn/pythonProject1/Model/MultimodelNER/VLSP2016/best_model'
-            output_model_file = os.path.join(output_dir, WEIGHTS_NAME)
-            output_encoder_file = os.path.join(output_dir, "pytorch_encoder.bin")
-            processor = MNERProcessor_2016()
-            label_list = processor.get_labels()
-            auxlabel_list = processor.get_auxlabels()
-            num_labels = len(label_list) + 1
-            auxnum_labels = len(auxlabel_list) + 1
-            trans_matrix = np.zeros((auxnum_labels, num_labels), dtype=float)
-            trans_matrix[0, 0] = 1  # pad to pad
-            trans_matrix[1, 1] = 1  # O to O
-            trans_matrix[2, 2] = 0.25  # B to B-MISC
-            trans_matrix[2, 4] = 0.25  # B to B-PER
-            trans_matrix[2, 6] = 0.25  # B to B-ORG
-            trans_matrix[2, 8] = 0.25  # B to B-LOC
-            trans_matrix[3, 3] = 0.25  # I to I-MISC
-            trans_matrix[3, 5] = 0.25  # I to I-PER
-            trans_matrix[3, 7] = 0.25  # I to I-ORG
-            trans_matrix[3, 9] = 0.25  # I to I-LOC
-            trans_matrix[4, 10] = 1  # X to X
-            trans_matrix[5, 11] = 1  # [CLS] to [CLS]
-            trans_matrix[6, 12] = 1
-            tokenizer = AutoTokenizer.from_pretrained(bert_model, do_lower_case=False)
-            model_umt, encoder_umt = load_model(output_model_file, output_encoder_file, encoder,num_labels,auxnum_labels)
-            eval_examples = get_test_examples_predict('E:/demo_datn/pythonProject1/Model/MultimodelNER/VLSP2016/Filetxt/')
-
-            y_pred, a = predict(model_umt, encoder_umt, eval_examples, tokenizer, device,save_image,trans_matrix)
-            formatted_output = format_predictions(a, y_pred[0])
-            final = process_predictions(formatted_output)
-            final2 = combine_entities(final)
-            final3 = remove_B_prefix(final2)
-            final4 = combine_i_tags(final3)
-            words_and_labels = final4
-            # Tạo danh sách từ
-            words = [word for word, _ in words_and_labels]
-            # Tạo danh sách thực thể và nhãn cho mỗi từ, loại bỏ nhãn 'O'
-            entities = [{'start': sum(len(word) + 1 for word, _ in words_and_labels[:i]),
-                         'end': sum(len(word) + 1 for word, _ in words_and_labels[:i + 1]), 'label': label} for
-                        i, (word, label)
-                        in enumerate(words_and_labels) if label != 'O']
-            # print(entities)
-
-            # Render the visualization without color for 'O' labels
-            html = displacy.render(
-                {"text": " ".join(words), "ents": entities, "title": None},
-                style="ent",
-                manual=True,
-                options={"colors": {"MISC": "#806699",
-                                    "ORG": "#ff6666",
-                                    "LOC": "#66cc66",
-                                    "PER": "#bf80ff",
-                                    "O": None}}
-            )
-            # print(html)
-            st.markdown(html, unsafe_allow_html=True)
-
-
-###Ví dụ 1 : Một trận hỗn chiến đã xảy ra tại trận đấu khúc côn cầu giữa  Penguins và Islanders ở Mỹ (image:penguin)

Model/MultimodelNER/best_model/bert_config.json

@@ -1,28 +0,0 @@
-{
-  "_name_or_path": "vinai/phobert-base-v2",
-  "architectures": [
-    "RobertaForMaskedLM"
-  ],
-  "attention_probs_dropout_prob": 0.1,
-  "bos_token_id": 0,
-  "classifier_dropout": null,
-  "eos_token_id": 2,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 768,
-  "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "layer_norm_eps": 1e-05,
-  "max_position_embeddings": 258,
-  "model_type": "roberta",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 12,
-  "pad_token_id": 1,
-  "position_embedding_type": "absolute",
-  "tokenizer_class": "PhobertTokenizer",
-  "torch_dtype": "float32",
-  "transformers_version": "4.35.2",
-  "type_vocab_size": 1,
-  "use_cache": true,
-  "vocab_size": 64001
-}

Model/MultimodelNER/best_model/eval_results.txt

@@ -1,11 +0,0 @@
-              precision    recall  f1-score   support
-
-         LOC     0.9570    0.9618    0.9594       996
-        MISC     0.9143    0.8889    0.9014        36
-         ORG     0.8129    0.7975    0.8051       158
-         PER     0.9835    0.9788    0.9812       851
-
-   micro avg     0.9563    0.9549    0.9556      2041
-   macro avg     0.9169    0.9068    0.9118      2041
-weighted avg     0.9561    0.9549    0.9555      2041
-Overall: 0.9563297350343474 0.9549240568348849 0.9556263790144643

Model/MultimodelNER/best_model/model_config.json

@@ -1 +0,0 @@
-{"bert_model": "vinai/phobert-base-v2", "do_lower": false, "max_seq_length": 256, "num_labels": 13, "label_map": {"1": "B-ORG", "2": "B-MISC", "3": "I-PER", "4": "I-ORG", "5": "B-LOC", "6": "I-MISC", "7": "I-LOC", "8": "O", "9": "B-PER", "10": "X", "11": "<s>", "12": "</s>"}}

Model/MultimodelNER/best_model/pytorch_encoder.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ab29aaf11c3beb874e34fc9bccaa1fb838d94701cf4a4189c37d768a7678e958
-size 241699561

Model/MultimodelNER/best_model/pytorch_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c950c331c48a229744b1b727a49d3dc248f28377ba8efbd86612daf2721e4368
-size 699285929

Model/MultimodelNER/cache/models--vinai--phobert-base-v2/.no_exist/2b51e367d92093c9688112098510e6a58bab67cd/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
-size 0

Model/MultimodelNER/cache/models--vinai--phobert-base-v2/refs/main

@@ -1 +0,0 @@
-2b51e367d92093c9688112098510e6a58bab67cd

Model/MultimodelNER/cache/models--vinai--phobert-base-v2/snapshots/2b51e367d92093c9688112098510e6a58bab67cd/config.json

@@ -1,27 +0,0 @@
-{
-  "architectures": [
-    "RobertaForMaskedLM"
-  ],
-  "attention_probs_dropout_prob": 0.1,
-  "bos_token_id": 0,
-  "classifier_dropout": null,
-  "eos_token_id": 2,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 768,
-  "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "layer_norm_eps": 1e-05,
-  "max_position_embeddings": 258,
-  "model_type": "roberta",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 12,
-  "pad_token_id": 1,
-  "position_embedding_type": "absolute",
-  "tokenizer_class": "PhobertTokenizer",
-  "torch_dtype": "float32",
-  "transformers_version": "4.26.1",
-  "type_vocab_size": 1,
-  "use_cache": true,
-  "vocab_size": 64001
-}

Model/MultimodelNER/cache/models--vinai--phobert-base-v2/snapshots/2b51e367d92093c9688112098510e6a58bab67cd/pytorch_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7ba09eb4c244a5b3a49ad76d52d129ac085b61f5c6287de7f99508b02be589f9
-size 540322347

Model/MultimodelNER/dataset_roberta.py

@@ -1,452 +0,0 @@
-import torch
-import logging
-import os
-
-logger = logging.getLogger(__name__)
-from torchvision import transforms
-from PIL import Image
-
-
-class SBInputExample(object):
-    """A single training/test example for simple sequence classification."""
-
-    def __init__(self, guid, text_a, text_b, img_id, label=None, auxlabel=None):
-        """Constructs a InputExample.
-
-        Args:
-            guid: Unique id for the example.
-            text_a: string. The untokenized text of the first sequence. For single
-            sequence tasks, only this sequence must be specified.
-            text_b: (Optional) string. The untokenized text of the second sequence.
-            Only must be specified for sequence pair tasks.
-            label: (Optional) string. The label of the example. This should be
-            specified for train and dev examples, but not for test examples.
-        """
-        self.guid = guid
-        self.text_a = text_a
-        self.text_b = text_b
-        self.img_id = img_id
-        self.label = label
-        # Please note that the auxlabel is not used in SB
-        # it is just kept in order not to modify the original code
-        self.auxlabel = auxlabel
-
-
-class SBInputFeatures(object):
-    """A single set of features of data"""
-
-    def __init__(self, input_ids, input_mask, added_input_mask, segment_ids, img_feat, label_id, auxlabel_id):
-        self.input_ids = input_ids
-        self.input_mask = input_mask
-        self.added_input_mask = added_input_mask
-        self.segment_ids = segment_ids
-        self.img_feat = img_feat
-        self.label_id = label_id
-        self.auxlabel_id = auxlabel_id
-
-
-def sbreadfile(filename):
-    '''
-    Đọc dữ liệu từ tệp và trả về dưới dạng danh sách các cặp từ và nhãn, cùng với danh sách hình ảnh và nhãn phụ.
-    '''
-    print("Chuẩn bị dữ liệu cho ", filename)
-    f = open(filename, encoding='utf8')
-    data = []
-    imgs = []
-    auxlabels = []
-    sentence = []
-    label = []
-    auxlabel = []
-    imgid = ''
-
-    for line in f:
-        line = line.strip()  # Loại bỏ các dấu cách thừa ở đầu và cuối dòng
-        if line.startswith('IMGID:'):
-            imgid = line.split('IMGID:')[1] + '.jpg'
-            continue
-        if line == '':
-            if len(sentence) > 0:
-                data.append((sentence, label))
-                imgs.append(imgid)
-                auxlabels.append(auxlabel)
-                sentence = []
-                label = []
-                auxlabel = []
-                imgid = ''
-            continue
-        splits = line.split('\t')
-        if len(splits) == 2:  # Đảm bảo dòng có ít nhất một từ và một nhãn
-            word, cur_label = splits
-            sentence.append(word)
-            label.append(cur_label)
-            auxlabel.append(cur_label[0])  # Lấy ký tự đầu tiên của nhãn làm nhãn phụ
-
-    if len(sentence) > 0:  # Xử lý dữ liệu cuối cùng trong tệp
-        data.append((sentence, label))
-        imgs.append(imgid)
-        auxlabels.append(auxlabel)
-
-    print("Số lượng mẫu: " + str(len(data)))
-    print("Số lượng hình ảnh: " + str(len(imgs)))
-    return data, imgs, auxlabels
-
-
-# def sbreadfile(filename): #code gốc
-#     '''
-#     read file
-#     return format :
-#     [ ['EU', 'B-ORG'], ['rejects', 'O'], ['German', 'B-MISC'], ['call', 'O'], ['to', 'O'], ['boycott', 'O'], ['British', 'B-MISC'], ['lamb', 'O'], ['.', 'O'] ]
-#     '''
-#     print("prepare data for ",filename)
-#     f = open(filename,encoding='utf8')
-#     data = []
-#     imgs = []
-#     auxlabels = []
-#     sentence = []
-#     label = []
-#     auxlabel = []
-#     imgid = ''
-#     a = 0
-#     for line in f:
-#         if line.startswith('IMGID:'):
-#             imgid = line.strip().split('IMGID:')[1] + '.jpg'
-#             continue
-#         if line[0] == "\n":
-#             if len(sentence) > 0:
-#                 data.append((sentence, label))
-#                 imgs.append(imgid)
-#                 auxlabels.append(auxlabel)
-#                 sentence = []
-#                 label = []
-#                 imgid = ''
-#                 auxlabel = []
-#             continue
-#         splits = line.split('\t')
-#         sentence.append(splits[0])
-#         cur_label = splits[-1][:-1]
-#         # if cur_label == 'B-OTHER':
-#         #     cur_label = 'B-MISC'
-#         # elif cur_label == 'I-OTHER':
-#         #     cur_label = 'I-MISC'
-#         label.append(cur_label)
-#         auxlabel.append(cur_label[0])
-
-#     if len(sentence) > 0:
-#         data.append((sentence, label))
-#         imgs.append(imgid)
-#         auxlabels.append(auxlabel)
-#         sentence = []
-#         label = []
-#         auxlabel = []
-
-#     print("The number of samples: " + str(len(data)))
-#     print("The number of images: " + str(len(imgs)))
-#     return data, imgs, auxlabels
-
-class DataProcessor(object):
-    """Base class for data converters for sequence classification data sets."""
-
-    def get_train_examples(self, data_dir):
-        """Gets a collection of `InputExample`s for the train set."""
-        raise NotImplementedError()
-
-    def get_dev_examples(self, data_dir):
-        """Gets a collection of `InputExample`s for the dev set."""
-        raise NotImplementedError()
-
-    def get_labels(self):
-        """Gets the list of labels for this data set."""
-        raise NotImplementedError()
-
-    @classmethod
-    def _read_sbtsv(cls, input_file, quotechar=None):
-        """Reads a tab separated value file."""
-        return sbreadfile(input_file)
-
-
-class MNERProcessor_2016(DataProcessor):
-    """Processor for the CoNLL-2003 data set."""
-
-    def get_train_examples(self, data_dir):
-        """See base class."""
-        data, imgs, auxlabels = self._read_sbtsv(os.path.join(data_dir, "train.txt"))
-        return self._create_examples(data, imgs, auxlabels, "train")
-
-    def get_dev_examples(self, data_dir):
-        """See base class."""
-        data, imgs, auxlabels = self._read_sbtsv(os.path.join(data_dir, "dev.txt"))
-        return self._create_examples(data, imgs, auxlabels, "dev")
-
-    def get_test_examples(self, data_dir):
-        """See base class."""
-        data, imgs, auxlabels = self._read_sbtsv(os.path.join(data_dir, "test.txt"))
-        return self._create_examples(data, imgs, auxlabels, "test")
-
-    def get_labels(self):
-        #         return [
-        # "O","I-PRODUCT-AWARD",
-        # "B-MISCELLANEOUS",
-        # "B-QUANTITY-NUM",
-        # "B-ORGANIZATION-SPORTS",
-        # "B-DATETIME",
-        # "I-ADDRESS",
-        # "I-PERSON",
-        # "I-EVENT-SPORT",
-        # "B-ADDRESS",
-        # "B-EVENT-NATURAL",
-        # "I-LOCATION-GPE",
-        # "B-EVENT-GAMESHOW",
-        # "B-DATETIME-TIMERANGE",
-        # "I-QUANTITY-NUM",
-        # "I-QUANTITY-AGE",
-        # "B-EVENT-CUL",
-        # "I-QUANTITY-TEM",
-        # "I-PRODUCT-LEGAL",
-        # "I-LOCATION-STRUC",
-        # "I-ORGANIZATION",
-        # "B-PHONENUMBER",
-        # "B-IP",
-        # "B-QUANTITY-AGE",
-        # "I-DATETIME-TIME",
-        # "I-DATETIME",
-        # "B-ORGANIZATION-MED",
-        # "B-DATETIME-SET",
-        # "I-EVENT-CUL",
-        # "B-QUANTITY-DIM",
-        # "I-QUANTITY-DIM",
-        # "B-EVENT",
-        # "B-DATETIME-DATERANGE",
-        # "I-EVENT-GAMESHOW",
-        # "B-PRODUCT-AWARD",
-        # "B-LOCATION-STRUC",
-        # "B-LOCATION",
-        # "B-PRODUCT",
-        # "I-MISCELLANEOUS",
-        # "B-SKILL",
-        # "I-QUANTITY-ORD",
-        # "I-ORGANIZATION-STOCK",
-        # "I-LOCATION-GEO",
-        # "B-PERSON",
-        # "B-PRODUCT-COM",
-        # "B-PRODUCT-LEGAL",
-        # "I-LOCATION",
-        # "B-QUANTITY-TEM",
-        # "I-PRODUCT",
-        # "B-QUANTITY-CUR",
-        # "I-QUANTITY-CUR",
-        # "B-LOCATION-GPE",
-        # "I-PHONENUMBER",
-        # "I-ORGANIZATION-MED",
-        # "I-EVENT-NATURAL",
-        # "I-EMAIL",
-        # "B-ORGANIZATION",
-        # "B-URL",
-        # "I-DATETIME-TIMERANGE",
-        # "I-QUANTITY",
-        # "I-IP",
-        # "B-EVENT-SPORT",
-        # "B-PERSONTYPE",
-        # "B-QUANTITY-PER",
-        # "I-QUANTITY-PER",
-        # "I-PRODUCT-COM",
-        # "I-DATETIME-DURATION",
-        # "B-LOCATION-GPE-GEO",
-        # "B-QUANTITY-ORD",
-        # "I-EVENT",
-        # "B-DATETIME-TIME",
-        # "B-QUANTITY",
-        # "I-DATETIME-SET",
-        # "I-LOCATION-GPE-GEO",
-        # "B-ORGANIZATION-STOCK",
-        # "I-ORGANIZATION-SPORTS",
-        # "I-SKILL",
-        # "I-URL",
-        # "B-DATETIME-DURATION",
-        # "I-DATETIME-DATE",
-        # "I-PERSONTYPE",
-        # "B-DATETIME-DATE",
-        # "I-DATETIME-DATERANGE",
-        # "B-LOCATION-GEO",
-        # "B-EMAIL","X","<s>", "</s>"]
-
-        # vlsp2016
-        return [
-            "B-ORG", "B-MISC",
-            "I-PER",
-            "I-ORG",
-            "B-LOC",
-            "I-MISC",
-            "I-LOC",
-            "O",
-            "B-PER",
-            "X",
-            "<s>",
-            "</s>"]
-
-        # vlsp2018
-        # return [
-        #         "O","I-ORGANIZATION",
-        #         "B-ORGANIZATION",
-        #         "I-LOCATION",
-        #         "B-MISCELLANEOUS",
-        #         "I-PERSON",
-        #         "B-PERSON",
-        #         "I-MISCELLANEOUS",
-        #         "B-LOCATION",
-        #         "X",
-        #         "<s>",
-        #         "</s>"]
-
-    def get_auxlabels(self):
-        return ["O", "B", "I", "X", "<s>", "</s>"]
-
-    def get_start_label_id(self):
-        label_list = self.get_labels()
-        label_map = {label: i for i, label in enumerate(label_list, 1)}
-        return label_map['<s>']
-
-    def get_stop_label_id(self):
-        label_list = self.get_labels()
-        label_map = {label: i for i, label in enumerate(label_list, 1)}
-        return label_map['</s>']
-
-    def _create_examples(self, lines, imgs, auxlabels, set_type):
-        examples = []
-        for i, (sentence, label) in enumerate(lines):
-            guid = "%s-%s" % (set_type, i)
-            text_a = ' '.join(sentence)
-            text_b = None
-            img_id = imgs[i]
-            label = label
-            auxlabel = auxlabels[i]
-            examples.append(
-                SBInputExample(guid=guid, text_a=text_a, text_b=text_b, img_id=img_id, label=label, auxlabel=auxlabel))
-        return examples
-
-
-def image_process(image_path, transform):
-    image = Image.open(image_path).convert('RGB')
-    image = transform(image)
-    return image
-
-
-def convert_mm_examples_to_features(examples, label_list, auxlabel_list,
-                                    max_seq_length, tokenizer, crop_size, path_img):
-    label_map = {label: i for i, label in enumerate(label_list, 1)}
-    auxlabel_map = {label: i for i, label in enumerate(auxlabel_list, 1)}
-
-    features = []
-    count = 0
-
-    transform = transforms.Compose([
-        transforms.Resize([256, 256]),
-        transforms.RandomCrop(crop_size),  # args.crop_size, by default it is set to be 224
-        transforms.RandomHorizontalFlip(),
-        transforms.ToTensor(),
-        transforms.Normalize((0.485, 0.456, 0.406),
-                             (0.229, 0.224, 0.225))])
-
-    for (ex_index, example) in enumerate(examples):
-        textlist = example.text_a.split(' ')
-        labellist = example.label
-        auxlabellist = example.auxlabel
-        tokens = []
-        labels = []
-        auxlabels = []
-        for i, word in enumerate(textlist):
-            token = tokenizer.tokenize(word)
-            tokens.extend(token)
-            label_1 = labellist[i]
-            auxlabel_1 = auxlabellist[i]
-            for m in range(len(token)):
-                if m == 0:
-                    labels.append(label_1)
-                    auxlabels.append(auxlabel_1)
-                else:
-                    labels.append("X")
-                    auxlabels.append("X")
-        if len(tokens) >= max_seq_length - 1:
-            tokens = tokens[0:(max_seq_length - 2)]
-            labels = labels[0:(max_seq_length - 2)]
-            auxlabels = auxlabels[0:(max_seq_length - 2)]
-        ntokens = []
-        segment_ids = []
-        label_ids = []
-        auxlabel_ids = []
-        ntokens.append("<s>")
-        segment_ids.append(0)
-        label_ids.append(label_map["<s>"])
-        auxlabel_ids.append(auxlabel_map["<s>"])
-        for i, token in enumerate(tokens):
-            ntokens.append(token)
-            segment_ids.append(0)
-            label_ids.append(label_map[labels[i]])
-            auxlabel_ids.append(auxlabel_map[auxlabels[i]])
-        ntokens.append("</s>")
-        segment_ids.append(0)
-        label_ids.append(label_map["</s>"])
-        auxlabel_ids.append(auxlabel_map["</s>"])
-        input_ids = tokenizer.convert_tokens_to_ids(ntokens)
-        input_mask = [1] * len(input_ids)
-        added_input_mask = [1] * (len(input_ids) + 49)  # 1 or 49 is for encoding regional image representations
-
-        while len(input_ids) < max_seq_length:
-            input_ids.append(0)
-            input_mask.append(0)
-            added_input_mask.append(0)
-            segment_ids.append(0)
-            label_ids.append(0)
-            auxlabel_ids.append(0)
-
-        assert len(input_ids) == max_seq_length
-        assert len(input_mask) == max_seq_length
-        assert len(segment_ids) == max_seq_length
-        assert len(label_ids) == max_seq_length
-        assert len(auxlabel_ids) == max_seq_length
-
-        image_name = example.img_id
-        image_path = os.path.join(path_img, image_name)
-
-        if not os.path.exists(image_path):
-            if 'NaN' not in image_path:
-                print(image_path)
-        try:
-            image = image_process(image_path, transform)
-        except:
-            count += 1
-            image_path_fail = os.path.join(path_img, 'background.jpg')
-            image = image_process(image_path_fail, transform)
-
-        else:
-            if ex_index < 2:
-                logger.info("*** Example ***")
-                logger.info("guid: %s" % (example.guid))
-                logger.info("tokens: %s" % " ".join(
-                    [str(x) for x in tokens]))
-                logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
-                logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
-                logger.info(
-                    "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
-                logger.info("label: %s" % " ".join([str(x) for x in label_ids]))
-                logger.info("auxlabel: %s" % " ".join([str(x) for x in auxlabel_ids]))
-
-            features.append(
-                SBInputFeatures(input_ids=input_ids, input_mask=input_mask, added_input_mask=added_input_mask,
-                                segment_ids=segment_ids, img_feat=image, label_id=label_ids, auxlabel_id=auxlabel_ids))
-
-    print('the number of problematic samples: ' + str(count))
-    return features
-
-
-# if __name__ == "__main__":
-#     processor = MNERProcessor_2016()
-#     label_list = processor.get_labels()
-#     auxlabel_list = processor.get_auxlabels()
-#     num_labels = len(label_list) + 1  # label 0 corresponds to padding, label in label_list starts from 1
-#
-#     start_label_id = processor.get_start_label_id()
-#     stop_label_id = processor.get_stop_label_id()
-#
-#     data_dir = r'sample_data'
-#     train_examples = processor.get_train_examples(data_dir)
-#     print(train_examples[0].img_id)

Model/MultimodelNER/list.txt

@@ -1,5 +0,0 @@
-IMGID:namngo
-Toi
-ten
-la
-Minh

Model/MultimodelNER/test.txt

@@ -1,78 +0,0 @@
-IMGID:014716
-“	O
-Tôi	O
-xin	O
-cám_ơn	O
-thượng_sĩ	O
-Nguyễn	B-PER
-Trung	I-PER
-Hiếu	I-PER
-(	O
-người	O
-phiên_dịch	O
-tiếng	B-MISC
-Anh	I-MISC
-cho	O
-đơn_vị	O
-tình_báo	O
-quân_sự	O
-số	O
-635	O
-của	O
-quân_đội	O
-Mỹ	B-LOC
-biên_chế	O
-bên	O
-cạnh	O
-lữ_đoàn	B-ORG
-bộ_binh	I-ORG
-số	I-ORG
-11	I-ORG
-,	O
-sư_đoàn	B-ORG
-bộ_binh	I-ORG
-23	I-ORG
-)	O
-,	O
-người	O
-đã	O
-cứu	O
-cuốn	O
-nhật_ký	O
-của	O
-chị	O
-tôi	O
-khỏi	O
-bị	O
-quẳng	O
-vào	O
-đống	O
-lửa	O
-bởi	O
-anh	O
-đã	O
-nhận	O
-ra	O
-trong	O
-cuốn	O
-sổ	O
-này	O
-đã	O
-chứa_đựng	O
-lửa	O
-rồi	O
-để	O
-anh	O
-trao	O
-lại	O
-nó	O
-cho	O
-Fred	B-PER
-như	O
-một	O
-lời	O
-uỷ_thác	O
-từ	O
-chị	O
-tôi	O
-.	O

Model/MultimodelNER/train_umt_2016.py

@@ -1,352 +0,0 @@
-import os
-import sys
-
-os.environ["CUDA_VISIBLE_DEVICES"] = "0"
-import argparse
-
-import logging
-import random
-import numpy as np
-import torch
-import torch.nn.functional as F
-from transformers import AutoTokenizer, BertConfig
-from Model.MultimodelNER.UMT import UMT
-from Model.MultimodelNER import resnet as resnet
-from Model.MultimodelNER.resnet_utils import myResnet
-from Model.MultimodelNER.VLSP2016.dataset_roberta import convert_mm_examples_to_features, MNERProcessor_2016
-from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
-                              TensorDataset)
-from pytorch_pretrained_bert.optimization import BertAdam, warmup_linear
-from Model.MultimodelNER.ner_evaluate import evaluate_each_class,evaluate
-from seqeval.metrics import classification_report
-from tqdm import tqdm, trange
-import json
-from Model.MultimodelNER.predict import convert_mm_examples_to_features_predict, get_test_examples_predict
-from Model.MultimodelNER.Ner_processing import *
-CONFIG_NAME = 'bert_config.json'
-WEIGHTS_NAME = 'pytorch_model.bin'
-
-logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
-                    datefmt='%m/%d/%Y %H:%M:%S',
-                    level=logging.INFO)
-logger = logging.getLogger(__name__)
-parser = argparse.ArgumentParser()
-## Required parameters
-parser.add_argument("--negative_rate",
-                    default=16,
-                    type=int,
-                    help="the negative samples rate")
-
-parser.add_argument('--lamb',
-                    default=0.62,
-                    type=float)
-
-parser.add_argument('--temp',
-                    type=float,
-                    default=0.179,
-                    help="parameter for CL training")
-
-parser.add_argument('--temp_lamb',
-                    type=float,
-                    default=0.7,
-                    help="parameter for CL training")
-
-parser.add_argument("--data_dir",
-                    default='./data/twitter2017',
-                    type=str,
-
-                    help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
-parser.add_argument("--bert_model", default='vinai/phobert-base-v2', type=str)
-parser.add_argument("--task_name",
-                    default='sonba',
-                    type=str,
-
-                    help="The name of the task to train.")
-parser.add_argument("--output_dir",
-                    default='E:/demo_datn/pythonProject1/Model/MultimodelNER/VLSP2016/best_model/',
-                    type=str,
-                    help="The output directory where the model predictions and checkpoints will be written.")
-
-## Other parameters
-parser.add_argument("--cache_dir",
-                    default="",
-                    type=str,
-                    help="Where do you want to store the pre-trained models downloaded from s3")
-
-parser.add_argument("--max_seq_length",
-                    default=128,
-                    type=int,
-                    help="The maximum total input sequence length after WordPiece tokenization. \n"
-                         "Sequences longer than this will be truncated, and sequences shorter \n"
-                         "than this will be padded.")
-
-parser.add_argument("--do_train",
-                    action='store_true',
-                    help="Whether to run training.")
-
-parser.add_argument("--do_eval",
-                    action='store_true',
-                    help="Whether to run eval on the dev set.")
-
-parser.add_argument("--do_lower_case",
-                    action='store_true',
-                    help="Set this flag if you are using an uncased model.")
-
-parser.add_argument("--train_batch_size",
-                    default=64,
-                    type=int,
-                    help="Total batch size for training.")
-
-parser.add_argument("--eval_batch_size",
-                    default=16,
-                    type=int,
-                    help="Total batch size for eval.")
-
-parser.add_argument("--learning_rate",
-                    default=5e-5,
-                    type=float,
-                    help="The initial learning rate for Adam.")
-
-parser.add_argument("--num_train_epochs",
-                    default=12.0,
-                    type=float,
-                    help="Total number of training epochs to perform.")
-
-parser.add_argument("--warmup_proportion",
-                    default=0.1,
-                    type=float,
-                    help="Proportion of training to perform linear learning rate warmup for. "
-                         "E.g., 0.1 = 10%% of training.")
-
-parser.add_argument("--no_cuda",
-                    action='store_true',
-                    help="Whether not to use CUDA when available")
-
-parser.add_argument("--local_rank",
-                    type=int,
-                    default=-1,
-                    help="local_rank for distributed training on gpus")
-
-parser.add_argument('--seed',
-                    type=int,
-                    default=37,
-                    help="random seed for initialization")
-
-parser.add_argument('--gradient_accumulation_steps',
-                    type=int,
-                    default=1,
-                    help="Number of updates steps to accumulate before performing a backward/update pass.")
-
-parser.add_argument('--fp16',
-                    action='store_true',
-                    help="Whether to use 16-bit float precision instead of 32-bit")
-
-parser.add_argument('--loss_scale',
-                    type=float, default=0,
-                    help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
-                         "0 (default value): dynamic loss scaling.\n"
-                         "Positive power of 2: static loss scaling value.\n")
-
-parser.add_argument('--mm_model', default='MTCCMBert', help='model name')  # 'MTCCMBert', 'NMMTCCMBert'
-parser.add_argument('--layer_num1', type=int, default=1, help='number of txt2img layer')
-parser.add_argument('--layer_num2', type=int, default=1, help='number of img2txt layer')
-parser.add_argument('--layer_num3', type=int, default=1, help='number of txt2txt layer')
-parser.add_argument('--fine_tune_cnn', action='store_true', help='fine tune pre-trained CNN if True')
-parser.add_argument('--resnet_root', default='E:/demo_datn/pythonProject1/Model/Resnet/', help='path the pre-trained cnn models')
-parser.add_argument('--crop_size', type=int, default=224, help='crop size of image')
-parser.add_argument('--path_image', default='E:/demo_datn/pythonProject1/Model/MultimodelNER/VLSP2016/Image', help='path to images')
-# parser.add_argument('--mm_model', default='TomBert', help='model name') #
-parser.add_argument('--server_ip', type=str, default='', help="Can be used for distant debugging.")
-parser.add_argument('--server_port', type=str, default='', help="Can be used for distant debugging.")
-args = parser.parse_args()
-
-
-
-processors = {
-    "twitter2015": MNERProcessor_2016,
-    "twitter2017": MNERProcessor_2016,
-    "sonba": MNERProcessor_2016
-}
-
-
-
-random.seed(args.seed)
-np.random.seed(args.seed)
-torch.manual_seed(args.seed)
-
-
-task_name = args.task_name.lower()
-
-
-
-processor = processors[task_name]()
-label_list = processor.get_labels()
-auxlabel_list = processor.get_auxlabels()
-num_labels = len(label_list) + 1  # label 0 corresponds to padding, label in label_list starts from 1
-auxnum_labels = len(auxlabel_list) + 1  # label 0 corresponds to padding, label in label_list starts from 1
-
-start_label_id = processor.get_start_label_id()
-stop_label_id = processor.get_stop_label_id()
-
-# ''' initialization of our conversion matrix, in our implementation, it is a 7*12 matrix initialized as follows:
-trans_matrix = np.zeros((auxnum_labels, num_labels), dtype=float)
-trans_matrix[0, 0] = 1  # pad to pad
-trans_matrix[1, 1] = 1  # O to O
-trans_matrix[2, 2] = 0.25  # B to B-MISC
-trans_matrix[2, 4] = 0.25  # B to B-PER
-trans_matrix[2, 6] = 0.25  # B to B-ORG
-trans_matrix[2, 8] = 0.25  # B to B-LOC
-trans_matrix[3, 3] = 0.25  # I to I-MISC
-trans_matrix[3, 5] = 0.25  # I to I-PER
-trans_matrix[3, 7] = 0.25  # I to I-ORG
-trans_matrix[3, 9] = 0.25  # I to I-LOC
-trans_matrix[4, 10] = 1  # X to X
-trans_matrix[5, 11] = 1  # [CLS] to [CLS]
-trans_matrix[6, 12] = 1  # [SEP] to [SEP]
-'''
-trans_matrix = np.zeros((num_labels, auxnum_labels), dtype=float)
-trans_matrix[0,0]=1 # pad to pad
-trans_matrix[1,1]=1
-trans_matrix[2,2]=1
-trans_matrix[4,2]=1
-trans_matrix[6,2]=1
-trans_matrix[8,2]=1
-trans_matrix[3,3]=1
-trans_matrix[5,3]=1
-trans_matrix[7,3]=1
-trans_matrix[9,3]=1
-trans_matrix[10,4]=1
-trans_matrix[11,5]=1
-trans_matrix[12,6]=1
-'''
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-tokenizer = AutoTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)
-
-
-
-net = getattr(resnet, 'resnet152')()
-net.load_state_dict(torch.load(os.path.join(args.resnet_root, 'resnet152.pth')))
-encoder = myResnet(net, args.fine_tune_cnn, device)
-
-
-output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
-# output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
-output_encoder_file = os.path.join(args.output_dir, "pytorch_encoder.bin")
-
-temp = args.temp
-temp_lamb = args.temp_lamb
-lamb = args.lamb
-negative_rate = args.negative_rate
-# # loadmodel
-#     model = UMT.from_pretrained(args.bert_model,
-#                                 cache_dir=args.cache_dir, layer_num1=args.layer_num1,
-#                                 layer_num2=args.layer_num2,
-#                                 layer_num3=args.layer_num3,
-#                                 num_labels_=num_labels, auxnum_labels=auxnum_labels)
-#     model.load_state_dict(torch.load(output_model_file,map_location=torch.device('cpu')))
-#     model.to(device)
-#     encoder_state_dict = torch.load(output_encoder_file,map_location=torch.device('cpu'))
-#     encoder.load_state_dict(encoder_state_dict)
-#     encoder.to(device)
-#     print(model)
-
-def load_model(output_model_file, output_encoder_file,encoder,num_labels,auxnum_labels):
-    model = UMT.from_pretrained(args.bert_model,
-                                cache_dir=args.cache_dir, layer_num1=args.layer_num1,
-                                layer_num2=args.layer_num2,
-                                layer_num3=args.layer_num3,
-                                num_labels_=num_labels, auxnum_labels=auxnum_labels)
-    model.load_state_dict(torch.load(output_model_file, map_location=torch.device('cpu')))
-    model.to(device)
-    encoder_state_dict = torch.load(output_encoder_file, map_location=torch.device('cpu'))
-    encoder.load_state_dict(encoder_state_dict)
-    encoder.to(device)
-    return model, encoder
-
-model_umt,encoder_umt=load_model(output_model_file, output_encoder_file,encoder,num_labels,auxnum_labels)
-#
-#     # sentence = 'Thương biết_mấy những Thuận, những Liên, những Luận, Xuân, Nghĩa mỗi người một hoàn_cảnh nhưng đều rất giống nhau: rất ham học, rất cố_gắng để đạt mức hiểu biết cao nhất.'
-#     # image_path = '/kaggle/working/data/014715.jpg'
-#     # # crop_size = 224'
-path_image='E:\demo_datn\pythonProject1\Model\MultimodelNER\VLSP2016\Image'
-trans_matrix = np.zeros((auxnum_labels,num_labels), dtype=float)
-trans_matrix[0,0]=1 # pad to pad
-trans_matrix[1,1]=1 # O to O
-trans_matrix[2,2]=0.25 # B to B-MISC
-trans_matrix[2,4]=0.25 # B to B-PER
-trans_matrix[2,6]=0.25 # B to B-ORG
-trans_matrix[2,8]=0.25 # B to B-LOC
-trans_matrix[3,3]=0.25 # I to I-MISC
-trans_matrix[3,5]=0.25 # I to I-PER
-trans_matrix[3,7]=0.25 # I to I-ORG
-trans_matrix[3,9]=0.25 # I to I-LOC
-trans_matrix[4,10]=1   # X to X
-trans_matrix[5,11]=1   # [CLS] to [CLS]
-trans_matrix[6,12]=1   # [SE
-path_image='E:\demo_datn\pythonProject1\Model\MultimodelNER\VLSP2016\Image'
-
-def predict(model_umt, encoder_umt, eval_examples, tokenizer, device,path_image,trans_matrix):
-
-    features = convert_mm_examples_to_features_predict(eval_examples, 256, tokenizer, 224,path_image)
-
-    input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
-    input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
-    added_input_mask = torch.tensor([f.added_input_mask for f in features], dtype=torch.long)
-    segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
-    img_feats = torch.stack([f.img_feat for f in features])
-    print(img_feats)
-    eval_data = TensorDataset(input_ids, input_mask, added_input_mask, segment_ids, img_feats)
-    eval_sampler = SequentialSampler(eval_data)
-    eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=16)
-
-    model_umt.eval()
-    encoder_umt.eval()
-
-    y_pred = []
-    label_map = {i: label for i, label in enumerate(label_list, 1)}
-    label_map[0] = "<pad>"
-
-    for input_ids, input_mask, added_input_mask, segment_ids, img_feats in tqdm(eval_dataloader, desc="Evaluating"):
-        input_ids = input_ids.to(device)
-        input_mask = input_mask.to(device)
-        added_input_mask = added_input_mask.to(device)
-        segment_ids = segment_ids.to(device)
-        img_feats = img_feats.to(device)
-
-        with torch.no_grad():
-            imgs_f, img_mean, img_att = encoder_umt(img_feats)
-            predicted_label_seq_ids = model_umt(input_ids, segment_ids, input_mask, added_input_mask, img_att,
-                                                trans_matrix)
-
-        logits = predicted_label_seq_ids
-        input_mask = input_mask.to('cpu').numpy()
-
-        for i, mask in enumerate(input_mask):
-            temp_1 = []
-            for j, m in enumerate(mask):
-                if j == 0:
-                    continue
-                if m:
-                    if label_map[logits[i][j]] not in ["<pad>", "<s>", "</s>", "X"]:
-                        temp_1.append(label_map[logits[i][j]])
-                else:
-                    break
-            y_pred.append(temp_1)
-
-    a = eval_examples[0].text_a.split(" ")
-
-    return y_pred, a
-
-# eval_examples = get_test_examples_predict('E:/demo_datn/pythonProject1/Model/MultimodelNER/VLSP2016/Filetxt/')
-# y_pred, a = predict(model_umt, encoder_umt, eval_examples, tokenizer, device,path_image,trans_matrix)
-# print(y_pred)
-# formatted_output = format_predictions(a, y_pred[0])
-# print(formatted_output)
-# final= process_predictions(formatted_output)
-# final2= combine_entities(final)
-# final3= remove_B_prefix(final2)
-# final4=combine_i_tags(final3)
-#
-# print(final4)
-