Upload 7 files

app.py

@@ -0,0 +1,124 @@
+import os
+import pytesseract
+from pdf2image import convert_from_path
+from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer
+from datasets import Dataset
+import torch
+from flask import Flask, request, jsonify, render_template
+from threading import Thread
+
+app = Flask(__name__)
+
+# إعداد المسارات
+input_folder = 'input'
+model_folder = 'model'
+
+# وظيفة لتحويل PDF إلى نص باستخدام Tesseract
+def pdf_to_text(file_path):
+    images = convert_from_path(file_path)
+    text = ''
+    for image in images:
+        text += pytesseract.image_to_string(image, lang='ara')  # assuming Arabic language
+    return text
+
+# وظيفة لتحضير البيانات
+def prepare_data():
+    data = {'text': [], 'label': []}
+    labels = os.listdir(input_folder)
+    for label in labels:
+        label_folder = os.path.join(input_folder, label)
+        for file_name in os.listdir(label_folder):
+            file_path = os.path.join(label_folder, file_name)
+            text = pdf_to_text(file_path)
+            data['text'].append(text)
+            data['label'].append(label)
+    return Dataset.from_dict(data), labels
+
+# دالة لتحميل النموذج والمحول
+def load_model():
+    model_name = "bert-base-multilingual-cased"
+    tokenizer = AutoTokenizer.from_pretrained(model_folder)
+    model = AutoModelForSequenceClassification.from_pretrained(model_folder)
+    return tokenizer, model
+
+# دالة لتدريب النموذج
+def train_model():
+    global tokenizer, model, labels  # تأكد من أن هذه المتغيرات متاحة في جميع أنحاء البرنامج
+
+    dataset, labels = prepare_data()
+    train_test_split = dataset.train_test_split(test_size=0.2)
+    tokenized_datasets = train_test_split.map(lambda x: tokenizer(x['text'], padding="max_length", truncation=True), batched=True)
+
+    training_args = TrainingArguments(
+        output_dir=model_folder,
+        evaluation_strategy="epoch",
+        learning_rate=2e-5,
+        per_device_train_batch_size=16,
+        per_device_eval_batch_size=16,
+        num_train_epochs=3,
+        weight_decay=0.01,
+    )
+
+    model = AutoModelForSequenceClassification.from_pretrained("bert-base-multilingual-cased", num_labels=len(labels))
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=tokenized_datasets['train'],
+        eval_dataset=tokenized_datasets['test'],
+    )
+
+    trainer.train()
+    # حفظ النموذج وجميع الملفات الضرورية
+    model.save_pretrained(model_folder)
+    tokenizer.save_pretrained(model_folder)
+    return "Model trained and saved!"
+
+# دالة لتصنيف الوثائق
+def classify_document(file_path):
+    text = pdf_to_text(file_path)
+    inputs = tokenizer(text, return_tensors="pt", padding="max_length", truncation=True)
+    outputs = model(**inputs)
+    predictions = torch.argmax(outputs.logits, dim=-1)
+    label = labels[predictions.item()]
+    return label, text
+
+# واجهة ويب
+@app.route('/')
+def home():
+    return render_template('index.html')
+
+@app.route('/train', methods=['POST'])
+def train():
+    message = train_model()
+    return jsonify({'message': message})
+
+@app.route('/classify', methods=['POST'])
+def classify():
+    if 'file' not in request.files:
+        return jsonify({'error': 'No file provided'}), 400
+
+    file = request.files['file']
+    if file.filename == '':
+        return jsonify({'error': 'No file selected'}), 400
+
+    file_path = os.path.join('uploads', file.filename)
+    file.save(file_path)
+    
+    label, text = classify_document(file_path)
+    
+    return jsonify({'label': label, 'text': text})
+
+def run_flask():
+    if os.path.exists(model_folder):
+        global tokenizer, model, labels
+        tokenizer, model = load_model()
+        labels = os.listdir(input_folder)
+    else:
+        tokenizer = AutoTokenizer.from_pretrained("bert-base-multilingual-cased")
+        model = None
+        labels = []
+    app.run()
+
+if __name__ == '__main__':
+    Thread(target=run_flask).start()

config.json

@@ -0,0 +1,13 @@
+{
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 512,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "type_vocab_size": 2,
+  "vocab_size": 30522
+}

requirements.txt

@@ -0,0 +1,5 @@
+Flask
+pytesseract
+pdf2image
+transformers==4.26.1
+torch

templates/index.html

@@ -0,0 +1,81 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Document Classifier</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 40px;
+        }
+        h1 {
+            color: #333;
+        }
+        .container {
+            max-width: 600px;
+            margin: auto;
+        }
+        .button {
+            display: inline-block;
+            padding: 10px 20px;
+            font-size: 16px;
+            cursor: pointer;
+            text-align: center;
+            text-decoration: none;
+            outline: none;
+            color: #fff;
+            background-color: #4CAF50;
+            border: none;
+            border-radius: 5px;
+            box-shadow: 0 4px #999;
+        }
+        .button:hover {background-color: #45a049}
+        .button:active {
+            background-color: #3e8e41;
+            box-shadow: 0 2px #666;
+            transform: translateY(2px);
+        }
+        .result {
+            margin-top: 20px;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>Document Classifier</h1>
+        <form id="upload-form" enctype="multipart/form-data">
+            <label for="file">Select a PDF file to classify:</label>
+            <input type="file" id="file" name="file" accept="application/pdf" required>
+            <button type="submit" class="button">Classify Document</button>
+        </form>
+        <button id="train-button" class="button">Train Model</button>
+        <div id="result" class="result"></div>
+    </div>
+
+    <script>
+        document.getElementById('upload-form').onsubmit = async function(event) {
+            event.preventDefault();
+            const fileInput = document.getElementById('file');
+            const formData = new FormData();
+            formData.append('file', fileInput.files[0]);
+
+            const response = await fetch('/classify', {
+                method: 'POST',
+                body: formData
+            });
+
+            const result = await response.json();
+            document.getElementById('result').innerText = 'Label: ' + result.label + '\nText: ' + result.text;
+        };
+
+        document.getElementById('train-button').onclick = async function() {
+            const response = await fetch('/train', {
+                method: 'POST'
+            });
+
+            const result = await response.json();
+            alert(result.message);
+        };
+    </script>
+</body>
+</html>