Update process_documents.py
Browse files- process_documents.py +1 -1
process_documents.py
CHANGED
|
@@ -157,4 +157,4 @@ def get_pdf_semantic_snippets(filtered_snippets, median_font_size):
|
|
| 157 |
|
| 158 |
|
| 159 |
def num_tokens(string):
|
| 160 |
-
return len(tiktoken.get_encoding("cl100k_base", disallowed_special=())
|
|
|
|
| 157 |
|
| 158 |
|
| 159 |
def num_tokens(string):
|
| 160 |
+
return len(tiktoken.get_encoding("cl100k_base").encode(string, disallowed_special=()))
|