Upload 5 files

.gitignore

@@ -0,0 +1,152 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

README.md

@@ -1,13 +1,24 @@
----
-title: Documentsummary
-emoji: 📈
-colorFrom: purple
-colorTo: gray
-sdk: streamlit
-sdk_version: 1.17.0
-app_file: app.py
-pinned: false
-license: unlicense
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# Text Summarizer
+
+This Project provides you with a brief summary of the given Text.
+The Project allows you to paste or Upload PDF file to summarize it , It also allows you to customize the summarization % of the Final summary!
+
+## Concept
+The Project is based on the concept of Extractive Text Summarization.
+In this concept we read all the sentences given in the TEXT and then spereate each word in them. 
+Then we grade each of the unique words with a value which corelates to the number of times it has occured in the text.
+
+Then we normalize these grades and reassign them to all the words in the sentences. Then we calculate the grades for the whole sentences and rank the sentences on the grades they get.
+Then we summarize the top "n%" of the sentences as output.
+<img src='https://imgs.search.brave.com/m5dNQYnKvcBHxQBBwzS_lUNmEKAEMcA3WvZZ9EYvQSM/rs:fit:1200:1016:1/g:ce/aHR0cHM6Ly9taXJv/Lm1lZGl1bS5jb20v/bWF4LzI5MjAvMSo1/X3Q0RUpsMUl5OUIx/dzVFdFgxWm9nLmpw/ZWc'>
+## Deployment
+
+The Project is Deployed on Hugging face 
+
+```bash
+  https://huggingface.co/spaces/SRDdev/Summarize
+```
+
+
+ 

app.py

@@ -0,0 +1,77 @@
+import spacy
+from spacy.lang.en.stop_words import STOP_WORDS
+from string import punctuation
+from spacy import tokens
+import streamlit as st
+from heapq import nlargest
+import subprocess
+subprocess.run("pip3 install PyPDF2".split())
+subprocess.run("python3 -m spacy download en_core_web_sm".split())
+import PyPDF2
+from utils import (
+    clean_text,
+    fetch_article_text,
+    preprocess_text_for_abstractive_summarization,
+    read_text_from_file,
+)
+#---------------------Pre-Requiste------------------------#
+stopwords = STOP_WORDS
+punctuation = punctuation + '\n'
+
+
+
+if __name__=="__main__":
+    st.title("Text Summarizer 📝")
+    st.subheader("Creator: Shreyas Dixit")
+    
+    n = st.sidebar.slider('Summarization %',10,90,step=10)
+    n = n/100
+    type=st.selectbox('Pick one', ['PDF','Text'])
+    if type=="PDF":
+        #Upload file
+        uploaded_file = st.file_uploader("Choose a file",type=['pdf','txt','docx'])
+        text = read_text_from_file(uploaded_file)
+        # FileName = uploaded_file.name
+        # if uploaded_file is not None:
+        #     pdfFileObj = open("{FileName}", 'rb') 
+        #     pdfReader = PyPDF2.PdfFileReader(pdfFileObj) 
+        #     pageObj = pdfReader.getPage(0) 
+        #     text = (pageObj.extractText()) 
+        #     pdfFileObj.close() 
+    elif type=="Text": 
+        #Text
+        text=st.text_area("Input text !")
+
+    if st.button('Summarize'):
+        nlp = spacy.load('en_core_web_sm')
+        doc = nlp(text)
+        #Word tokenization
+        tokens = [tokens.text for tokens in doc]
+        word_frquency = {}
+        for word in doc:
+            if word.text.lower() not in stopwords:
+                if word.text.lower() not in punctuation:
+                    if word.text not in word_frquency.keys():
+                        word_frquency[word.text] = 1
+                    else:
+                        word_frquency[word.text] += 1
+        #Normalize the values
+        max_word = max(word_frquency.values())
+        for word in word_frquency.keys():
+            word_frquency[word] = word_frquency[word]/max_word
+        #Sentence Tokenization
+        sentence_token = [sent for sent in doc.sents]
+        sentence_score = {}
+        for sent in sentence_token:
+            for word in sent:
+                if word.text.lower() in word_frquency.keys():
+                    if sent not in sentence_score.keys():
+                        sentence_score[sent] = word_frquency[word.text.lower()]
+                    else:
+                        sentence_score[sent] += word_frquency[word.text.lower()]
+        #Creating a Summary
+        select_length = int(len(sentence_token)*n)
+        summary = nlargest(select_length,sentence_score,key = sentence_score.get)
+        summary = [word.text for word in summary]
+        summary = ' '.join(summary)
+        st.markdown(summary)

requirements.txt

@@ -0,0 +1,5 @@
+heapq-max==0.21
+spacy==2.2.4
+spacy-loggers==1.0.2
+streamlit==1.9.0
+PyPDF2

utils.py

@@ -0,0 +1,138 @@
+import re
+import requests
+import docx2txt
+from io import StringIO
+import PyPDF2
+from PyPDF2 import PdfFileReader
+
+from bs4 import BeautifulSoup
+from nltk.tokenize import sent_tokenize
+
+emoji_pattern = re.compile(
+    "["
+    u"\U0001F600-\U0001F64F"  # emoticons
+    u"\U0001F300-\U0001F5FF"  # symbols & pictographs
+    u"\U0001F680-\U0001F6FF"  # transport & map symbols
+    u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
+    u"\U00002702-\U000027B0"
+    u"\U000024C2-\U0001F251"
+    "]+",
+    flags=re.UNICODE,
+)
+
+
+def clean_text(x):
+    # x = x.lower()  # lowercase
+    x = x.encode("ascii", "ignore").decode()  # unicode
+    x = re.sub(r"https*\S+", " ", x)  # url
+    x = re.sub(r"@\S+", " ", x)  # mentions
+    x = re.sub(r"#\S+", " ", x)  # hastags
+    # x = x.replace("'", "")  # remove ticks
+    # x = re.sub("[%s]" % re.escape(string.punctuation), " ", x)  # punctuation
+    # x = re.sub(r"\w*\d+\w*", "", x)  # numbers
+    x = re.sub(r"\s{2,}", " ", x)  # over spaces
+    x = emoji_pattern.sub(r"", x)  # emojis
+    x = re.sub("[^.,!?A-Za-z0-9]+", " ", x)  # special charachters except .,!?
+
+    return x
+
+
+def fetch_article_text(url: str):
+
+    r = requests.get(url)
+    soup = BeautifulSoup(r.text, "html.parser")
+    results = soup.find_all(["h1", "p"])
+    text = [result.text for result in results]
+    ARTICLE = " ".join(text)
+    ARTICLE = ARTICLE.replace(".", ".<eos>")
+    ARTICLE = ARTICLE.replace("!", "!<eos>")
+    ARTICLE = ARTICLE.replace("?", "?<eos>")
+    sentences = ARTICLE.split("<eos>")
+    current_chunk = 0
+    chunks = []
+    for sentence in sentences:
+        if len(chunks) == current_chunk + 1:
+            if len(chunks[current_chunk]) + len(sentence.split(" ")) <= 500:
+                chunks[current_chunk].extend(sentence.split(" "))
+            else:
+                current_chunk += 1
+                chunks.append(sentence.split(" "))
+        else:
+            print(current_chunk)
+            chunks.append(sentence.split(" "))
+
+    for chunk_id in range(len(chunks)):
+        chunks[chunk_id] = " ".join(chunks[chunk_id])
+
+    return ARTICLE, chunks
+
+
+def preprocess_text_for_abstractive_summarization(tokenizer, text):
+    sentences = sent_tokenize(text)
+
+    # initialize
+    length = 0
+    chunk = ""
+    chunks = []
+    count = -1
+    for sentence in sentences:
+        count += 1
+        combined_length = (
+            len(tokenizer.tokenize(sentence)) + length
+        )  # add the no. of sentence tokens to the length counter
+
+        if combined_length <= tokenizer.max_len_single_sentence:  # if it doesn't exceed
+            chunk += sentence + " "  # add the sentence to the chunk
+            length = combined_length  # update the length counter
+
+            # if it is the last sentence
+            if count == len(sentences) - 1:
+                chunks.append(chunk.strip())  # save the chunk
+
+        else:
+            chunks.append(chunk.strip())  # save the chunk
+
+            # reset
+            length = 0
+            chunk = ""
+
+            # take care of the overflow sentence
+            chunk += sentence + " "
+            length = len(tokenizer.tokenize(sentence))
+
+    return chunks
+
+
+def read_pdf(file):
+    pdfReader = PdfFileReader(file)
+    count = pdfReader.numPages
+    all_page_text = ""
+    for i in range(count):
+        page = pdfReader.getPage(i)
+        all_page_text += page.extractText()
+
+    return all_page_text
+
+
+def read_text_from_file(file):
+
+    # read text file
+    if file.type == "text/plain":
+        # To convert to a string based IO:
+        stringio = StringIO(file.getvalue().decode("utf-8"))
+
+        # To read file as string:
+        file_content = stringio.read()
+
+    # read pdf file
+    elif file.type == "application/pdf":
+        file_content = read_pdf(file)
+
+    # read docx file
+    elif (
+        file.type
+        == "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+    ):
+        file_content = docx2txt.process(file)
+
+    return file_content