Add application file

app.py

@@ -1,7 +1,11 @@
 from flask import Flask, request
 from flask_restful import Resource, Api, reqparse
 from flask_cors import CORS
+from encoder_file import get_encoder
 
+len(encoder.encode(t))
+
+encoder = get_encoder()
 app = Flask(__name__)
 CORS(app)
 api = Api(app)
@@ -29,5 +33,21 @@ class Data(Resource):
 api.add_resource(Status, '/')
 api.add_resource(Data, '/data')
 
+
+@app.route('/get_sample',methods=['POST'])
+def _inference():
+    
+    try:
+        info    = request.json["data"]
+        return jsonify({"result" : encoder.encode(info)})
+
+    except Exception as e:
+        print(e)
+        return jsonify({"result":"-1"}) 
+
+@app.route('/checking',methods=['GET'])
+def check():
+    return jsonify({"status": 'working'})
+
 if __name__ == '__main__':
     app.run(debug=True)

encoder_file.py

@@ -0,0 +1,150 @@
+import json
+from functools import lru_cache
+
+import regex as re
+
+import os
+from datetime import datetime
+from email.utils import formatdate, parsedate_to_datetime
+
+import requests
+from appdirs import user_cache_dir
+
+
+def download(url, destination_file):
+    headers = {}
+
+    path = user_cache_dir("promptify")
+
+    if not os.path.isdir(path):
+        os.makedirs(path)
+
+    destination_file = os.path.join(path, destination_file)
+
+    if os.path.exists(destination_file):
+        mtime = os.path.getmtime(destination_file)
+        headers["if-modified-since"] = formatdate(mtime, usegmt=True)
+
+    response = requests.get(url, headers=headers, stream=True)
+    response.raise_for_status()
+
+    if response.status_code == requests.codes.not_modified:
+        return
+
+    if response.status_code == requests.codes.ok:
+        with open(destination_file, "wb") as f:
+            for chunk in response.iter_content(chunk_size=1048576):
+                f.write(chunk)
+
+        if last_modified := response.headers.get("last-modified"):
+            new_mtime = parsedate_to_datetime(last_modified).timestamp()
+            os.utime(destination_file, times=(datetime.now().timestamp(), new_mtime))
+    return destination_file
+
+bpe_file = {"filename": "vocab.bpe", "link": "https://github.com/syonfox/GPT-3-Encoder/raw/master/vocab.bpe"}
+encoder_file = {"filename": "encoder.json", "link": "https://github.com/syonfox/GPT-3-Encoder/raw/master/encoder.json"}
+
+
+@lru_cache()
+def bytes_to_unicode():
+
+    bs = (
+        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+def get_pairs(word):
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+class Encoder:
+    def __init__(self, encoder, bpe_merges, errors="replace"):
+        self.encoder = encoder
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.errors = errors
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
+        self.cache = {}
+        self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
+
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token)
+
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token
+
+        while True:
+            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                    new_word.extend(word[i:j])
+                    i = j
+                except:
+                    new_word.extend(word[i:])
+                    break
+
+                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
+                    new_word.append(first + second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+
+        word = " ".join(word)
+        self.cache[token] = word
+        return word
+
+    def encode(self, text):
+        bpe_tokens = []
+        for token in re.findall(self.pat, text):
+            token = "".join(self.byte_encoder[b] for b in token.encode("utf-8"))
+
+            bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" "))
+        return bpe_tokens
+
+    def decode(self, tokens):
+        text = "".join([self.decoder[token] for token in tokens])
+        text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
+        return text
+
+
+def get_encoder():
+    encoder_filename = download(encoder_file["link"], encoder_file["filename"])
+    bpe_filename = download(bpe_file["link"], bpe_file["filename"])
+    with open(encoder_filename, "r") as f:
+        encoder = json.load(f)
+    with open(bpe_filename, "r", encoding="utf-8") as f:
+        bpe_data = f.read()
+    bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split("\n")[1:-1]]
+    return Encoder(encoder=encoder, bpe_merges=bpe_merges)