import re
import numpy as np
import tiktoken
from langchain.text_splitter import TokenTextSplitter
def strtobool(val):
val = val.lower()
if val in ('yes', 'true', 't', '1'):
return True
elif val in ('no', 'false', 'f', '0'):
return False
else:
raise ValueError(f"Invalid truth value {val}")
def split_camel_case(word):
# This regular expression pattern matches the transition from a lowercase letter to an uppercase letter
pattern = re.compile(r'(?<=[a-z])(?=[A-Z])')
# Replace the matched pattern (the empty string between lowercase and uppercase letters) with a space
split_word = pattern.sub(' ', word)
return split_word
# Function to split tokens into chunks
def chunk_tokens(tokens, max_len):
for i in range(0, len(tokens), max_len):
yield tokens[i:i + max_len]
def update_nested_dict(d, u):
for k, v in u.items():
if isinstance(v, dict):
d[k] = update_nested_dict(d.get(k, {}), v)
else:
d[k] = v
return d
def cleanInputText(textInputLLM):
# Sequentially applying all the replacements and cleaning operations on textInputLLM
# Using regular expressions substitution
textInputLLM = re.sub(r'\(\'\\n\\n', ' ', textInputLLM)
textInputLLM = re.sub(r'\(\"\\n\\n', ' ', textInputLLM)
textInputLLM = re.sub(r'\\n\\n\',\)', ' ', textInputLLM)
textInputLLM = re.sub(r'\\n\\n\",\)', ' ', textInputLLM)
# Applying replacements with while loops since we need repetition until conditions are met
while re.search(r'##\n', textInputLLM):
textInputLLM = re.sub(r"##\n", '. ', textInputLLM)
while '###' in textInputLLM:
textInputLLM = textInputLLM.replace("###", ' ')
while '##' in textInputLLM:
textInputLLM = textInputLLM.replace("##", ' ')
while ' # ' in textInputLLM:
textInputLLM = textInputLLM.replace(" # ", ' ')
while '--' in textInputLLM:
textInputLLM = textInputLLM.replace("--", '-')
while re.search(r'\\\\-', textInputLLM):
textInputLLM = re.sub(r"\\\\-", '.', textInputLLM)
while re.search(r'\*\*\n', textInputLLM):
textInputLLM = re.sub(r"\*\*\n", '. ', textInputLLM)
while re.search(r'\*\*\*', textInputLLM):
textInputLLM = re.sub(r"\*\*\*", ' ', textInputLLM)
while re.search(r'\*\*', textInputLLM):
textInputLLM = re.sub(r"\*\*", ' ', textInputLLM)
while re.search(r' \* ', textInputLLM):
textInputLLM = re.sub(r" \* ", ' ', textInputLLM)
while re.search(r'is a program of the\n\nInternational Society for Infectious Diseases', textInputLLM):
textInputLLM = re.sub(
r'is a program of the\n\nInternational Society for Infectious Diseases',
'is a program of the International Society for Infectious Diseases',
textInputLLM,
flags=re.M
)
# Optionally, if you want to include these commented out operations:
# while re.search(r'\n\n', textInputLLM):
# textInputLLM = re.sub(r'\n\n', '. ', textInputLLM)
# while re.search(r'\n', textInputLLM):
# textInputLLM = re.sub(r'\n', ' ', textInputLLM)
while re.search(r' \*\.', textInputLLM):
textInputLLM = re.sub(r' \*\.', ' .', textInputLLM)
while ' ' in textInputLLM:
textInputLLM = textInputLLM.replace(" ", ' ')
while re.search(r'\.\.', textInputLLM):
textInputLLM = re.sub(r'\.\.', '.', textInputLLM)
while re.search(r'\. \.', textInputLLM):
textInputLLM = re.sub(r'\. \.', '.', textInputLLM)
# Final cleanup replacements
textInputLLM = re.sub(r'\(\"\.', ' ', textInputLLM)
textInputLLM = re.sub(r'\(\'\.', ' ', textInputLLM)
textInputLLM = re.sub(r'\",\)', ' ', textInputLLM)
textInputLLM = re.sub(r'\',\)', ' ', textInputLLM)
# Strip leading/trailing whitespaces
textInputLLM = textInputLLM.strip()
return textInputLLM
def encoding_getter(encoding_type: str):
"""
Returns the appropriate encoding based on the given encoding type (either an encoding string or a model name).
tiktoken supports three encodings used by OpenAI models:
Encoding name OpenAI models
cl100k_base gpt-4, gpt-3.5-turbo, text-embedding-ada-002
p50k_base Codex models, text-davinci-002, text-davinci-003
r50k_base (or gpt2) GPT-3 models like davinci
https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
"""
if "k_base" in encoding_type:
return tiktoken.get_encoding(encoding_type)
else:
try:
my_enc = tiktoken.encoding_for_model(encoding_type)
return my_enc
except Exception as err:
my_enc = tiktoken.get_encoding("cl100k_base") #default for gpt-4, gpt-3.5-turbo
return my_enc
def tokenizer(string: str, encoding_type: str) -> list:
"""
Returns the tokens in a text string using the specified encoding.
"""
encoding = encoding_getter(encoding_type)
tokens = encoding.encode(string)
return tokens
def token_counter(string: str, encoding_type: str) -> int:
"""
Returns the number of tokens in a text string using the specified encoding.
"""
num_tokens = len(tokenizer(string, encoding_type))
return num_tokens
# Function to extract words from a given text
def extract_words(text, putInLower=False):
# Use regex to find all words (sequences of alphanumeric characters)
if putInLower:
return [word.lower() for word in re.findall(r'\b\w+\b', text)]
else:
return [word for word in re.findall(r'\b\w+\b', text)] #re.findall(r'\b\w+\b', text)
# Function to check if all words from 'compound_word' are in the 'word_list'
def all_words_in_list(compound_word, word_list, putInLower=False):
words_to_check = extract_words(compound_word, putInLower=putInLower)
if putInLower:
return all(word.lower() in word_list for word in words_to_check)
else:
return all(word in word_list for word in words_to_check)
def row_to_dict_string(rrrow, columnsDict):
formatted_items = []
for col in rrrow.index:
if col not in columnsDict:
continue
value = rrrow[col]
# Check if the value is an instance of a number (int, float, etc.)
if isinstance(value, (int, float)):
formatted_items.append(f'"{col}": {value}') # Use double quotes for keys
else:
formatted_items.append(
f'"{col}": "{value}"') # Use double quotes for keys and string values
# Join items and enclose them in {}
return '{' + ', '.join(formatted_items) + '}'
#
# def row_to_dict_string(rrrow):
# formatted_items = []
# for col in rrrow.index:
# value = rrrow[col]
# # Check if the value is an instance of a number (int, float, etc.)
# if isinstance(value, (int, float)):
# formatted_items.append(f"'{col}': {value}")
# else:
# formatted_items.append(f"'{col}': '{value}'")
# # Join items and enclose them in {}
# return '{' + ', '.join(formatted_items) + '}'
def rescale_exponential_to_linear(df, column, new_min=0.5, new_max=1.0):
# Get the original exponential scores
original_scores = df[column]
# Normalize the scores to a 0-1 range
min_score = original_scores.min()
max_score = original_scores.max()
normalized_scores = (original_scores - min_score) / (max_score - min_score)
# Rescale the normalized scores to the interval [0.5, 1.0]
linear_scores = new_min + (normalized_scores * (new_max - new_min))
# Assign the linear scores back to the dataframe
df[column] = linear_scores
return df
def rescale_exponential_to_logarithmic(df, column, new_min=0.5, new_max=1.0):
# Ensure all values are positive and greater than zero, because log(0) is undefined
epsilon = 1e-10
df[column] = df[column] + epsilon
# Apply logarithmic transformation
log_transformed_scores = np.log(df[column])
# Normalize the log-transformed scores to a 0-1 range
min_score = log_transformed_scores.min()
max_score = log_transformed_scores.max()
normalized_log_scores = (log_transformed_scores - min_score) / (max_score - min_score)
# Rescale the normalized scores to the interval [0.5, 1.0]
logarithmic_scores = new_min + (normalized_log_scores * (new_max - new_min))
# Assign the logarithmically scaled scores back to the dataframe
df[column] = logarithmic_scores
return df