Spaces:
Build error
Build error
File size: 2,177 Bytes
22738ca |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
#!/usr/bin/python3
# -*- coding: utf-8 -*
import sys
import os
import math
import xlsxwriter
from textblob import TextBlob as tb
class FindTechnologies(object):
def __init__(self):
print("Starting")
def tf(word, blob):
return (float)(blob.noun_phrases.count(word)) / (float)(len(blob.noun_phrases))
def n_containing(word, bloblist):
return sum(1 for blob in bloblist if word in blob.noun_phrases)
def idf(word, bloblist):
return math.log(len(bloblist) / (float)(1 + n_containing(word, bloblist)))
def tfidf(word, blob, bloblist):
return tf(word, blob) * idf(word, bloblist)
# Create an excel file for validation purpose
def get_technologies(self):
folder_path = "C:/Users/asouili01/Documents/PatSemBeta-v3/Data/input/Gaggenau/"
stopwords = open('C:/Users/asouili01/Documents/PIXSEB/Ressources/stopwords.txt', 'r').read().split('\r\n')
bloblist = []
filenamelist = []
for path, dirs, files in os.walk(folder_path):
for filename in files:
print(filename)
filenamelist.append(filename)
name, extension = filename.split('.')
filepath = folder_path + "/" + filename
filehandler = open(filepath, "r",encoding="utf-8")
content = filehandler.read()
filteredtext = [t for t in content if t.lower() not in stopwords]
filteredcontent = ''.join(filteredtext)
blob = 'blob_' + name.lower()
print (blob)
blob = tb(filteredcontent.lower())
bloblist.append(blob)
print(bloblist)
for i, blob in enumerate(bloblist):
print("Top words in document {}".format(i + 1))
scores = {word: tfidf(word, blob, bloblist) for word in blob.noun_phrases}
sorted_words = sorted(scores.items(), key=lambda x: x[1], reverse=True)
for word, score in sorted_words[:5]:
print("\tWord: {}, TF-IDF: {}".format(word, round(score, 10)))
|