feat(app.py): update app.py

.gitignore

@@ -0,0 +1 @@
+**/__pycache__

app.py

@@ -0,0 +1,43 @@
+from predictor import Predictor
+from transformers import pipeline
+from huggingface_hub import login
+from datetime import date
+import os
+import gradio as gr
+
+login(os.environ["HF_Token"])    
+paths = [
+    "data/W020230619818476939351.xls",
+    "data/W020230619818476975218.xls"
+]
+predictor = Predictor(
+    pipelines={
+        "name": pipeline("nerpipe", model="minskiter/resume-token-classification-name-0708",trust_remote_code=True,use_auth_token=True),
+        "common": pipeline("nerpipe",model="minskiter/resume-token-classification",trust_remote_code=True,use_auth_token=True)
+    },
+    paths=paths,
+    today=date(2023,4,1)
+)
+
+def ner_predictor_gradio(input):
+    entities = predictor(input)
+    # flattern entities
+    flatterns = []
+    for key in entities:
+        if isinstance(entities[key],list):
+            for item in entities[key]:
+                if isinstance(item,list):
+                    for subitem in item:
+                        flatterns.append(subitem)
+                else:
+                    flatterns.append(item)
+    return {"text":input, "entities": flatterns}
+
+demo = gr.Interface(
+    fn=ner_predictor_gradio, 
+    inputs=gr.Textbox(lines=5, label="输入你的简历"), 
+    outputs=gr.HighlightedText(label="简历识别结果"),
+)
+demo.launch()
+
+            

predictor/__init__.py

@@ -0,0 +1,443 @@
+from typing import Any, Dict,List
+from transformers import Pipeline
+import requests
+import re
+from io import BytesIO
+import pandas as pd
+import math
+import queue
+from datetime import date
+
+class Predictor():
+
+    def __init__(
+            self,
+            pipelines: Dict[str, Pipeline] = {},
+            paths: List[str] = [],
+            today: date = date.today()
+        ) -> None:
+        if "name" not in pipelines:
+            raise ValueError("'name' pipeline is None")
+        if "common" not in pipelines:
+            raise ValueError("'common' pipeline is None")
+        self.pipelines = pipelines
+        self.today = today
+        self.__init_split_data()
+        self.__init_schools_data(paths)
+        self.__init_patterns()
+
+    def __init_patterns(
+        self
+    ):
+        last_name = r"[赵,钱,孙,李,周,吴,郑,王,冯,陈,楮,卫,蒋,沈,韩,杨,朱,秦,尤,许,何,吕,施,张,孔,曹,严,华,金,魏,陶,姜,戚,谢,邹,喻,"\
+            +r"柏,水,窦,章,云,苏,潘,葛,奚,范,彭,郎,鲁,韦,昌,马,苗,凤,花,方,俞,任,袁,柳,酆,鲍,史,唐,费,廉,岑,薛,雷,贺,倪,汤,滕,殷,罗," \
+                    + r"毕,郝,邬,安,常,乐,于,时,傅,皮,卞,齐,康,伍,余,元,卜,顾,孟,平,黄,和,穆,萧,尹,姚,邵,湛,汪,祁,毛,禹,狄,米,贝,明,臧,计,伏,成,戴,谈,宋,茅," \
+                    + r"庞,熊,纪,舒,屈,项,祝,董,梁,杜,阮,蓝,闽,席,季,麻,强,贾,路,娄,危,江,童,颜,郭,梅,盛,林,刁,锺,徐,丘,骆,高,夏,蔡,田,樊,胡,凌,霍,虞,万,支," \
+                    + r"柯,昝,管,卢,莫,经,房,裘,缪,干,解,应,宗,丁,宣,贲,邓,郁,单,杭,洪,包,诸,左,石,崔,吉,钮,龚,程,嵇,邢,滑,裴,陆,荣,翁,荀,羊,於,惠,甄,麹,家," \
+                    + r"封,芮,羿,储,靳,汲,邴,糜,松,井,段,富,巫,乌,焦,巴,弓,牧,隗,山,谷,车,侯,宓,蓬,全,郗,班,仰,秋,仲,伊,宫,宁,仇,栾,暴,甘,斜,厉,戎,祖,武,符," \
+                    + r"刘,景,詹,束,龙,叶,幸,司,韶,郜,黎,蓟,薄,印,宿,白,怀,蒲,邰,从,鄂,索,咸,籍,赖,卓,蔺,屠,蒙,池,乔,阴,郁,胥,能,苍,双,闻,莘,党,翟,谭,贡,劳," \
+                    + r"逄,姬,申,扶,堵,冉,宰,郦,雍,郤,璩,桑,桂,濮,牛,寿,通,边,扈,燕,冀,郏,浦,尚,农,温,别,庄,晏,柴,瞿,阎,充,慕,连,茹,习,宦,艾,鱼,容,向,古,易," \
+                    + r"慎,戈,廖,庾,终,暨,居,衡,步,都,耿,满,弘,匡,国,文,寇,广,禄,阙,东,欧,殳,沃,利,蔚,越,夔,隆,师,巩,厍,聂,晁,勾,敖,融,冷,訾,辛,阚,那,简,饶," \
+                    + r"空,曾,毋,沙,乜,养,鞠,须,丰,巢,关,蒯,相,查,后,荆,红,游,竺,权,逑,盖,益,桓,公,万俟,司马,上官,欧阳,夏侯,诸葛,闻人,东方,赫连,皇甫,尉迟," \
+                    + r"公羊,澹台,公冶,宗政,濮阳,淳于,单于,太叔,申屠,公孙,仲孙,轩辕,令狐,锺离,宇文,长孙,慕容,鲜于,闾丘,司徒,司空,丌官,司寇,仉,督,子车," \
+                    + r"颛孙,端木,巫马,公西,漆雕,乐正,壤驷,公良,拓拔,夹谷,宰父,谷梁,晋,楚,阎,法,汝,鄢,涂,钦,段干,百里,东郭,南门,呼延,归,海,羊舌,微生,岳," \
+                    + r"帅,缑,亢,况,后,有,琴,梁丘,左丘,东门,西门,商,牟,佘,佴,伯,赏,南宫,墨,哈,谯,笪,年,爱,阳,佟,第五,言,福,邱]"
+        first_name = r' {0,3}[\u4e00-\u9fa5]( {0,3}[\u4e00-\u9fa5]){0,3}'
+        self.name_pattern = re.compile(last_name + first_name)
+        self.phone_pattern = re.compile(r'1 {0,4}(3 {0,4}\d|4 {0,4}[5-9]|5 {0,4}[0-35-9]|6 {0,4}[2567]|7 {0,4}[0-8]|8 {0,4}\d|9 {0,4}[0-35-9]) {0,4}(\d {0,4}){8}')
+        self.email_pattern = re.compile(r'([a-zA-Z0-9_-] {0,4})+@([a-zA-Z0-9_-] {0,4})+(\. {0,4}([a-zA-Z0-9_-] {0,4})+)+')
+        self.gender_pattern = re.compile(r'(性 {0,8}别.*?)?\s*?(男|女)\s+?')
+        self.age_patterns = [
+            re.compile(r"(\d{1,2})岁|年龄.{0,10}(\d{1,2})"),
+            re.compile(r"生.{0,12}(([12]\d{3})[年|.]?(([01]?\d)[月|.]?)?(([0-3]?\d)[日|.]?)?)"),
+        ]
+        self.works_key_pattern = re.compile("工作经(历|验)|experience",re.M|re.I)
+        self.job_time_patterns = re.compile('([1-2]\d{3}(\D?[01]?\d){0,2})\D?([1-2]\d{3}(\D?[01]?\d){0,2}|至今)')
+        self.edu_index =  ["博士","硕士","研究生","学士","本科","大专","专科","中专","高中","初中","小学"]
+        self.edu_patterns = list(re.compile(i) for i in self.edu_index)
+        self.school_pattern = re.compile(r"([a-zA-Z0-9 \u4e00-\u9fa5]{1,18}(学院|大学|中学|小学|学校|Unverisity|College))")
+
+    def _is_url(self, path: str):
+        return path.startswith('http://') or path.startswith('https://')
+
+    def __init_schools_data(
+        self,
+        paths: List[str],
+    ):
+        schools = {}
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36",
+        }
+        for path in paths:
+            stream = None
+            if self._is_url(path):
+                res = requests.get(path,headers=headers)
+                
+                if res.status_code==200:
+                    stream = BytesIO(res.content)
+            else:
+                with open(path, 'rb') as f:
+                    stream = BytesIO(f.read())
+            df = pd.read_excel(stream)
+            for row in df.iterrows():
+                if isinstance(row[1][1],float) and math.isnan(row[1][1]):
+                    continue
+                if row[1][1]=='学校名称':
+                    continue
+                # [学校] = 学历(本科、专科)
+                if len(row[1])>5:
+                    schools[row[1][1]] = row[1][5]
+                else:
+                    schools[row[1][1]] = "成人学校"
+        self.schools = schools
+        if len(schools)==0:
+            raise ValueError("学校数据为空")
+
+    def __init_split_data(
+        self
+    ):
+        self.splits = {'\\', '_', '"', '%', '{', '《', ')', '$', '(', '\n', '~', '*', ':', '!', '；', '”', '’', '\t', '？', '-', ';', '》', '】', '`', '、', '+', '“', '[', '—', '·', '）', '=', '‘', '}', '?', '，', '&', '@', '#', ']', '——', ' ', '.', '【', "'", '>', ',', '/', '：', '。', '...', '^', '（', '<', '|', '……', '！'}
+
+    def to_date(self, datestr:str):
+        if re.match("^\d{4}$",datestr):
+            return date(int(datestr),1,1)
+        match = re.match("^\d{4}(\D)\d{1,2}",datestr)
+        if match is not None:
+            try:
+                m = min(max(int(datestr.split(match.group(1))[1]),1),12)
+                return date(int(datestr.split(match.group(1))[0]),m,1)
+            except ValueError:
+                print(int(datestr.split(match.group(1))[0]),int(datestr.split(match.group(1))[1]))
+                raise
+        if datestr=="至今":
+            return self.today
+        return self.today
+
+    def split_to_blocks(
+        self,
+        text: str,
+        max_block_len: int = 510,
+        overlap: bool = True,
+        max_overlap_len: int = 20,
+    ):
+        block = {
+            "start": -1,
+            "end": -1,
+            "text": "",
+        }
+        blocks = []
+        overlap_end = queue.Queue()
+        for i in range(len(text)):
+            if text[i] in self.splits:
+                if block["start"]==-1:
+                    continue
+                if block["end"]!=-1 and i-block['start']>=max_block_len:
+                    block["text"] = text[block["start"]:block["end"]]
+                    blocks.append(block)
+                    block = {
+                        "start": overlap_end.queue[0]+1 if overlap else block['end']+1,
+                        "end": -1,
+                        "text": "",
+                    }
+                block["end"] = i
+                while overlap_end.qsize()>0 and overlap_end.queue[0]+max_overlap_len<=i:
+                    overlap_end.get()
+                overlap_end.put(i)
+            else:
+                if block["start"]==-1:
+                    block["start"] = i
+        # last block
+        if block["start"]!=-1:
+            block["end"] = len(text)
+            block["text"] = text[block["start"]:block["end"]]
+            blocks.append(block)
+        return blocks
+    
+    def get_expand_span(
+        self, 
+        text: str, 
+        start: int, 
+        end: int,
+        max_expand_length=20,
+    ):
+        expand_l,expand_r = start,end
+        for l in range(max(start-max_expand_length,0), start):
+            if text[l] in self.splits:
+                expand_l = l+1
+                break
+        for r in range(min(end+max_expand_length,len(text)-1), end, -1):
+            if text[r] in self.splits:
+                expand_r = r
+                break
+        return text[expand_l:expand_r], expand_l, expand_r
+    
+    def remove_blanks(
+        self,
+        text: str,
+        blank_pattern: re.Pattern,
+    ):
+        index_mapper = {}
+        new_text = []
+        for i in range(len(text)):
+            if blank_pattern.match(text[i]) is not None:
+                continue
+            index_mapper[len(new_text)] = i
+            new_text.append(text[i])
+        return ''.join(new_text), index_mapper
+    
+    def process(self, text)->Dict[str, Any]:
+        return_obj = {
+            "name": [],
+            "age": [],
+            "gender": [],
+            "phone": [],
+            "email": [],
+            "schools": [],
+            "work_time": 0,
+            "edus": [],
+            "jobs": [],
+            "titles": []
+        }
+        # 获取名字，先过滤所有空白字符，防止名字中间有空格
+        remove_blanks_text, index_mapper = self.remove_blanks(text, re.compile(r' '))
+        for name_match in self.name_pattern.finditer(remove_blanks_text):
+            start,end = name_match.span()
+            expand_text, start, end = self.get_expand_span(remove_blanks_text, start, end)
+            entities = self.pipelines['name'](expand_text)
+            for entity in entities:
+                if entity['entity']=='NAME' and self.name_pattern.match(entity['word']) is not None:
+                    obj = {
+                        'start': index_mapper[start+entity['start']],
+                        'end': index_mapper[start+entity['end']-1]+1,
+                        'entity': 'NAME',
+                        'text': entity['word']
+                    }
+                    repeat = False
+                    for o in return_obj['name']:
+                        if obj['start']==o['start'] and obj['end']==o['end']:
+                            repeat = True
+                            break
+                    if not repeat:
+                        obj['origin'] = text[obj['start']:obj['end']]
+                        return_obj['name'].append(obj)
+        # 获取年龄
+        for age_match in self.age_patterns[0].finditer(remove_blanks_text):
+            age = None
+            s,e = -1,-1
+            if age_match.group(1) is not None:
+                age = age_match.group(1)
+                s,e = age_match.span(1)
+            elif age_match.group(2) is not None:
+                age = age_match.group(2)
+                s,e = age_match.span(1)
+            if age is not None:    
+                return_obj['age'].append({
+                    'start': index_mapper[s],
+                    'end': index_mapper[e-1]+1,
+                    'text': str(age),
+                    'entity': 'AGE',
+                    'origin': text[index_mapper[s]:index_mapper[e-1]+1]
+                })
+        for age_match in self.age_patterns[1].finditer(remove_blanks_text):
+            age = None
+            s,e = -1,-1
+            year = age_match.group(2)
+            if year is not None:
+                year = int(year)
+                month = age_match.group(4)
+                if month is not None:
+                    month = int(month)
+                else:
+                    month = 1
+                day = age_match.group(6)
+                if day is not None:
+                    day = int(day)
+                else:
+                    day = 1
+                age = date.today().year - year
+                if date.today().month<month or (date.today().month==month and date.today().day<day):
+                    age -= 1
+            if age is not None:
+                s,e = age_match.span(1)
+                return_obj['age'].append({
+                    'start': index_mapper[s],
+                    'end': index_mapper[e-1]+1,
+                    'text': str(age),
+                    'entity': 'AGE',
+                    'origin': text[index_mapper[s]:index_mapper[e-1]+1]
+                })
+        # 获取学校
+        for school_match in self.school_pattern.finditer(remove_blanks_text):
+            start,end = school_match.span()
+            expand_text, start, end = self.get_expand_span(remove_blanks_text, start, end)
+            entities = self.pipelines['common'](expand_text)
+            for entity in entities:
+                if entity['entity']=="ORG" and self.school_pattern.search(entity['word']) is not None:
+                    obj = {
+                        'start': index_mapper[start+entity['start']],
+                        'end': index_mapper[start+entity['end']-1]+1,
+                        'entity': 'SCHOOL'
+                    }
+                    for school in self.schools:
+                        if school in entity['word']:
+                            obj['text'] = school
+                            obj["level"] = self.schools[school]
+                            break
+                    repeat = False
+                    for o in return_obj['schools']:
+                        if obj['start']==o['start'] and obj['end']==o['end']:
+                            repeat = True
+                            break
+                    if not repeat:
+                        obj['origin'] = text[obj['start']:obj['end']]
+                        return_obj['schools'].append(obj)
+        # 正则找学校
+        for school_match in re.finditer(r"|".join(self.schools.keys()), remove_blanks_text):
+            start,end = school_match.span()
+            obj = {
+                'start': index_mapper[start],
+                'end': index_mapper[end-1]+1,
+                'entity': 'SCHOOL',
+                'text': school_match.group(),
+            }
+            repeat = False
+            for o in return_obj['schools']:
+                if obj['start']==o['start'] and obj['end']==o['end']:
+                    repeat = True
+                    break
+            if not repeat:
+                obj['origin'] = text[obj['start']:obj['end']]
+                obj['level'] = self.schools[obj['text']]
+                return_obj['schools'].append(obj)
+        # 获取学历
+        for i,pattern in enumerate(self.edu_patterns):
+            for edu_match in pattern.finditer(remove_blanks_text):
+                start,end = edu_match.span()
+                expand_text, start, end = self.get_expand_span(remove_blanks_text, start, end)
+                entities = self.pipelines['common'](expand_text)
+                for entity in entities:
+                    if entity['entity']=='EDU' and pattern.search(entity['word']) is not None:
+                        obj = {
+                            'start': index_mapper[start+entity['start']],
+                            'end': index_mapper[start+entity['end']-1]+1,
+                            'text': self.edu_index[i],
+                            'entity': 'EDU',
+                        }
+                        repeat = False
+                        for o in return_obj['edus']:
+                            if obj['start']==o['start'] and obj['end']==o['end']:
+                                repeat = True
+                                break
+                        if not repeat:
+                            obj['origin'] = text[obj['start']:obj['end']]
+                            return_obj['edus'].append(obj)
+        # 如果有工作经历
+        if self.works_key_pattern.search(remove_blanks_text) is not None:
+            for job_time_match in self.job_time_patterns.finditer(remove_blanks_text):
+                origin_start,origin_end = job_time_match.span()
+                # convert_to_date
+                fr = self.to_date(job_time_match.group(1))
+                fs,fe = job_time_match.span(1)
+                to = self.to_date(job_time_match.group(3))
+                ts,te = job_time_match.span(3)
+                expand_text, start, end = self.get_expand_span(remove_blanks_text, origin_start, origin_end, max_expand_length=100)
+                entities = self.pipelines['common'](expand_text)
+                objs = []
+                for entity in entities:
+                    if entity['entity']=="ORG":
+                        obj = {
+                            'start': index_mapper[start+entity['start']],
+                            'end': index_mapper[start+entity['end']-1]+1,
+                            'entity': 'COMPANY',
+                            'text': entity['word'],
+                            'dis': min(
+                                abs(origin_start-start-entity['end']+1),
+                                abs(origin_end-start-entity['start'])
+                            ),
+                        }
+                        obj['origin'] = text[obj['start']:obj['end']]
+                        objs.append(obj)
+                objs.sort(key=lambda x:x['dis'])
+                if len(objs)>0 and self.school_pattern.search(objs[0]['text']) is None:
+                    del objs[0]['dis']
+                    from_date = {
+                        'start': index_mapper[fs],
+                        'end': index_mapper[fe-1]+1,
+                        'text': fr.isoformat(),
+                        'entity': 'DATE',
+                        'origin': text[index_mapper[fs]:index_mapper[fe-1]+1]
+                    }
+                    to_date = {
+                        'start': index_mapper[ts],
+                        'end': index_mapper[te-1]+1,
+                        'text': to.isoformat(),
+                        'entity': 'DATE',
+                        'origin': text[index_mapper[ts]:index_mapper[te-1]+1]
+                    }
+                    jobs = [objs[0],from_date,to_date]
+                    return_obj['jobs'].append(jobs)
+            return_obj["jobs"].sort(key=lambda x:date.fromisoformat(x[1]['text']))
+            # 计算工作时间
+            last_end = None
+            work_month = 0
+            for i in range(0,len(return_obj["jobs"])):
+                start = date.fromisoformat(return_obj["jobs"][i][1]['text'])
+                end = date.fromisoformat(return_obj["jobs"][i][2]['text'])
+                if last_end is not None and start<last_end:
+                    start = last_end
+                diff_y = end.year-start.year
+                diff_m = end.month-start.month
+                work_month += diff_y * 12 + diff_m
+                last_end = end
+            return_obj['work_time'] = math.ceil(work_month/12)
+        # 获取手机号码
+        for phone_match in self.phone_pattern.finditer(text):
+            start,end = phone_match.span()
+            return_obj['phone'].append({
+                'start': start,
+                'end': end,
+                'entity': 'PHONE',
+                'origin': text[start:end],
+                'text': re.sub('\s','',text[start:end])
+            })
+        for email_match in self.email_pattern.finditer(text):
+            start,end = email_match.span()
+            return_obj['email'].append({
+                'start': start,
+                'end': end,
+                'entity': 'EMAIL',
+                'origin': text[start:end],
+                'text': re.sub('\s','',text[start:end])
+            })
+        for gender_match in self.gender_pattern.finditer(text):
+            start,end = gender_match.span(2)
+            return_obj['gender'].append({
+                'start': start,
+                'end': end,
+                'entity': 'GENDER',
+                'word': text[start:end],
+                'text': text[start:end]
+            })
+        for block in self.split_to_blocks(remove_blanks_text):
+            entities = self.pipelines["common"](block["text"])
+            for entity in entities:
+                if entity['entity']=='TITLE':
+                    obj = {
+                        'start': index_mapper[block['start']+entity['start']],
+                        'end': index_mapper[block['start']+entity['end']-1]+1,
+                        'text': entity['word'],
+                        'entity': 'TITLE',
+                    }
+                    obj['origin'] = text[obj['start']:obj['end']]
+                    repeat = False
+                    for o in return_obj['titles']:
+                        if obj['start']==o['start'] and obj['end']==o['end']:
+                            repeat = True
+                            break
+                    if not repeat:
+                        return_obj['titles'].append(obj)
+        return return_obj
+    
+    def __call__(self, *args: Any, **kwds: Any) -> Any:
+        return self.process(*args, **kwds)
+

requirements.txt

@@ -0,0 +1,7 @@
+transformers==4.30.1
+gradio==3.36.1
+huggingface-hub==0.15.1
+torch==2.0.1
+pandas==2.0.3
+requests==2.31.0
+xlrd==2.0.1