requirements

app.py

@@ -20,7 +20,7 @@ def load_readme(dataset_name: str):
     return response.text
 
 # Function to check if the README content exceeds the token limit
-def check_token_limit(content: str, max_tokens: int = 8192):
+def check_token_limit(content: str, max_tokens: int = 7500):
     tokens = tokenizer.encode(content)
     if len(tokens) > max_tokens:
         truncated_content = tokenizer.decode(tokens[:max_tokens])

requirements.txt

@@ -0,0 +1 @@
+transformers