Update app.py

app.py

@@ -1,58 +1,84 @@
 from flask import Flask, request, render_template
 import requests
+import PyPDF2
+import io
+import os
+import googletrans
 import re
-import nltk
-from googletrans import Translator
-
-nltk.download("punkt")
+import pandas as pd
 
 app = Flask(__name__)
 
-def translate_text(text, src='en', dest='en'):
-    translator = Translator(service_urls=['translate.google.com'])
-    return translator.translate(text, dest=dest, src=src).text
-
-def text_processing(text, language='en'):
-    # remove special characters and numbers
-    processed_text = re.sub(r'[^a-zA-Z\s]', '', text)
-    
-    # tokenize the text into words
-    tokens = nltk.word_tokenize(processed_text)
-    
-    # translate the text if it's not in English
-    if language != 'en':
-        translated_text = translate_text(processed_text, dest='en')
-        tokens = nltk.word_tokenize(translated_text)
-    
-    # create the vocabulary of words
-    vocab = set(tokens)
-    
-    # create a dictionary to encode each word as an integer
-    word_to_int = {word: i for i, word in enumerate(vocab)}
-    
-    # encode the words as integers
-    encoded_text = [word_to_int[word] for word in tokens]
-    
-    return encoded_text
-
-@app.route("/", methods=['GET', 'POST'])
+@app.route('/')
 def index():
-    if request.method == 'POST':
-        if 'url' in request.form:
-            url = request.form['url']
-            response = requests.get(url)
-            text = response.text
-            language = translate_text(text, dest='en')
-        elif 'file' in request.files:
-            file = request.files['file']
-            text = file.read().decode('utf-8')
-            language = 'en'
-        
-        encoded_text = text_processing(text, language)
-        
-        return render_template('index.html', encoded_text=encoded_text)
-    
     return render_template('index.html')
 
-if __name__ == "__main__":
-    app.run(host="0.0.0.0",port=7860)
+@app.route('/upload', methods=['POST'])
+def upload():
+    file = request.files.get('file')
+    url = request.form.get('url')
+
+    if file:
+        filename = file.filename
+        file_extension = os.path.splitext(filename)[1]
+        file.save(filename)
+
+        # Check file extension and read the content
+        if file_extension == '.pdf':
+            # Read pdf file
+            pdf_file = PyPDF2.PdfFileReader(file)
+            text = ''
+            for page in range(pdf_file.getNumPages()):
+                text += pdf_file.getPage(page).extractText() + ' '
+        elif file_extension == '.txt':
+            # Read txt file
+            text = file.read().decode('utf-8')
+        else:
+            return 'Invalid file format'
+    elif url:
+        response = requests.get(url)
+        file_extension = os.path.splitext(url)[1]
+
+        # Check file extension and read the content
+        if file_extension == '.pdf':
+            # Read pdf file
+            pdf_file = PyPDF2.PdfFileReader(io.BytesIO(response.content))
+            text = ''
+            for page in range(pdf_file.getNumPages()):
+                text += pdf_file.getPage(page).extractText() + ' '
+        elif file_extension == '.txt':
+            # Read txt file
+            text = response.text
+        else:
+            return 'Invalid file format'
+    else:
+        return 'No file or URL found'
+
+    # Check if the language of the text is English, otherwise translate it
+    try:
+        src_lang = googletrans.LANGUAGES[googletrans.detect(text).lang]
+        if src_lang != 'en':
+            # Initialize the translator
+            translator = googletrans.Translator()
+
+            # Translate the text to English
+            text = translator.translate(text, dest='en').text
+
+            # Display a message indicating the text has been translated
+            print('The text has been translated from {} to English'.format(src_lang))
+    except Exception as e:
+        print('Error:', e)
+
+    # Convert the text to a dataset
+    lines = re.split(r'[.!?]+', text)
+    lines = [line.strip() for line in lines if line.strip() != '']
+    data = {'sentence': lines}
+    df = pd.DataFrame(data)
+
+    # Save the dataset to a CSV file
+    df.to_csv('dataset.csv', index=False)
+
+    return 'Dataset created successfully!'
+
+if __name__ == '__main__':
+    app.run(host="0.0.0.0",port=7860,debug=True)