Spaces:
Build error
Build error
File size: 2,156 Bytes
22738ca |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 |
#!/usr/bin/python3
# -*- coding: utf-8 -*
import sys
import os
import math
import re
from App.bin import constants
from textblob import TextBlob as tb
class TechnologyFinder(object):
def __init__(self, corpus):
self.corpus = corpus
print("Extracting technologies")
def last_cleansing(self, tech):
tech = str(tech)
tech = re.sub(r'\s?\bcomprises\b', '', tech)
return tech
def get_technologies(self):
corpus = self.corpus
technologies = []
def tf(word, blob):
return (float)(blob.noun_phrases.count(word)) / (float)(len(blob.noun_phrases))
def n_containing(word, bloblist):
return sum(1 for blob in bloblist if word in blob.noun_phrases)
def idf(word, bloblist):
return math.log(len(bloblist) / (float)(1 + n_containing(word, bloblist)))
def tfidf(word, blob, bloblist):
return tf(word, blob) * idf(word, bloblist)
stopwords = open(constants.ASSETS+'stopwords', 'r').read().split('\r\n')
bloblist = []
filenamelist = []
for filepath,patent in corpus.items():
filename = os.path.basename(os.path.normpath(filepath))
#name, extension = filename.split('.')
filenamelist.append(filepath)
filteredtext = [t for t in patent if t.lower() not in stopwords]
filteredcontent = ''.join(filteredtext)
blob = tb(filteredcontent.lower())
bloblist.append(blob)
for i, blob in enumerate(bloblist):
filename = []
technologies.append(filename)
scores = {word: tfidf(word, blob, bloblist) for word in blob.noun_phrases}
sorted_words = sorted(scores.items(), key=lambda x: x[1], reverse=True)
for word, score in sorted_words[:6]:
word = self.last_cleansing(word)
print("techologies found")
filename.append(word)
technologies_list = dict(zip(filenamelist, technologies))
return technologies_list
|