From Deprecated

.github/workflows/main.yml

@@ -0,0 +1,18 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push --force https://Roaoch:[email protected]/spaces/Roaoch/CyberClassic main

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.9
+
+RUN useradd -m -u 1000 user
+
+WORKDIR /app
+
+COPY ./startings.csv ./startings.csv
+COPY ./src ./src
+COPY ./requirements.txt ./requirements.txt
+COPY ./main.py ./main.py
+
+RUN pip install --upgrade pip
+RUN pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+RUN pip install pandas numpy transformers fastapi unicorn[standard]
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

main.py

@@ -0,0 +1,22 @@
+import warnings
+
+from src.cyberclaasic import CyberClassic
+from fastapi import FastAPI
+
+warnings.simplefilter("ignore", UserWarning)
+
+app = FastAPI()
+
+text_generator = CyberClassic(
+    min_length=30,
+    max_length=50,
+    startings_path='./startings.csv'
+)
+
+@app.get("/")
+def generete():
+    return {"text": str(text_generator.generate())}
+
+@app.get('/answer')
+def answer(promt: str):
+    return {"text": str(text_generator.answer(f'{promt}:\n'))}

src/cyberclaasic.py

@@ -0,0 +1,71 @@
+import uuid
+import torch
+import json
+
+import pandas as pd
+
+from src.discriminator import DiscriminatorModel
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, GPT2LMHeadModel, GenerationConfig
+
+import numpy as np
+
+class CyberClassic(torch.nn.Module):
+    def __init__(
+            self,
+            min_length: int, 
+            max_length: int,
+            startings_path: str
+        ) -> None:
+        super().__init__()
+        self.min_length = min_length
+        self.max_length = max_length
+        self.startings = pd.read_csv(startings_path)
+
+        self.tokenizer = AutoTokenizer.from_pretrained('Roaoch/CyberClassic-Generator')
+        self.generator: GPT2LMHeadModel = AutoModelForCausalLM.from_pretrained('Roaoch/CyberClassic-Generator')
+        self.discriminator = DiscriminatorModel.from_pretrained('Roaoch/CyberClassic-Discriminator')
+
+        self.tokenizer.pad_token = self.tokenizer.eos_token
+        self.generation_config = GenerationConfig(
+            max_new_tokens=max_length,
+            num_beams=6,
+            early_stopping=True,
+            do_sample=True,
+            # top_k=60,
+            # penalty_alpha=0.6,
+            # top_p=0.95,
+            eos_token_id=self.tokenizer.eos_token_id,
+            pad_token=self.tokenizer.pad_token_id
+        )
+
+    def encode(self, input_ids: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:
+        last_hidden_state  = self.generator(input_ids=input_ids, attention_mask=attention_mask, output_hidden_states=True)['hidden_states'][-1]
+        weights_for_non_padding = attention_mask * torch.arange(start=1, end=last_hidden_state.shape[1] + 1).unsqueeze(0)
+        sum_embeddings = torch.sum(last_hidden_state * weights_for_non_padding.unsqueeze(-1), dim=1)
+        num_of_none_padding_tokens = torch.sum(weights_for_non_padding, dim=-1).unsqueeze(-1)
+        return sum_embeddings / num_of_none_padding_tokens
+
+    def generate(self) -> str:
+        starts = self.startings['text'].values[np.random.randint(0, len(self.startings), 4)].tolist()
+        tokens = self.tokenizer(starts, return_tensors='pt', padding=True, truncation=True)
+        generated = self.generator.generate(**tokens, generation_config=self.generation_config)
+
+        input_emb = self.encode(input_ids=generated, attention_mask=torch.full(generated.size(), 1))
+        score = self.discriminator(input_emb)
+        score = torch.abs(score - 0.889)
+        index = int(torch.argmin(score))
+
+        decoded = self.tokenizer.batch_decode(generated, skip_special_tokens=True)
+
+        return decoded[index]
+
+    def answer(self, promt: str) -> str:
+        promt_tokens = self.tokenizer(promt, return_tensors='pt')
+        output = self.generator.generate(
+            **promt_tokens, 
+            generation_config=self.generation_config,
+        )
+
+        decoded = self.tokenizer.batch_decode(output)
+        return decoded[0]

src/discriminator.py

@@ -0,0 +1,28 @@
+import torch
+
+from transformers import PretrainedConfig, PreTrainedModel
+
+class DiscriminatorModelConfig(PretrainedConfig):
+    model_type = 'descriminatormodel'
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+
+class DiscriminatorModel(PreTrainedModel):
+    config_class = DiscriminatorModelConfig
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+        self.model = torch.nn.Sequential(
+            torch.nn.Linear(768, 512),
+            torch.nn.ReLU(),
+            torch.nn.Dropout(0.1),
+            torch.nn.Linear(512, 256),
+            torch.nn.ReLU(),
+            torch.nn.Dropout(0.1),
+            torch.nn.Linear(256, 1),
+            torch.nn.Dropout(0.1),
+            torch.nn.Sigmoid()
+        )
+    def forward(self, input):
+        return self.model(input) 

src/utils/proccess_data.py

@@ -0,0 +1,14 @@
+import pandas as pd
+
+df = pd.read_csv('dataset.csv')['text'].values
+res = [
+    ' '.join(txt.split(' ')[:3])
+    for txt in df
+]
+
+res_df = pd.DataFrame({
+    'text':
+    res
+})
+
+res_df.to_csv('startings.csv', index=False)