Spaces:
Sleeping
Sleeping
File size: 4,702 Bytes
e899e0f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 |
import openai
import os
from dotenv import load_dotenv
import yaml
from langchain.embeddings.openai import OpenAIEmbeddings
from pyprojroot import here # for creating top-level directories in project without changing setwd()
import shutil
load_dotenv()
class LoadConfig:
"""
A class for loading configuration settings and managing directories.
This class loads various configuration settings from the 'app_config.yml' file,
including LLM configurations, retrieval configurations, and memory configurations.
It also performs directory-related operations such as creating and removing directories.
...
Attributes:
llm_engine : str
The language model engine specified in the configuration.
llm_system_role : str
The role of the language model system specified in the configuration.
persist_directory : str
The path to the persist directory where data is stored.
custom_persist_directory : str
The path to the custom persist directory.
embedding_model : OpenAIEmbeddings
An instance of the OpenAIEmbeddings class for language model embeddings.
data_directory : str
The path to the data directory.
k : int
The value of 'k' specified in the retrieval configuration.
embedding_model_engine : str
The engine specified in the embedding model configuration.
chunk_size : int
The chunk size specified in the splitter configuration.
chunk_overlap : int
The chunk overlap specified in the splitter configuration.
temperature : float
The temperature specified in the LLM configuration.
qa_pair_count : int
The number of question-answer pairs specified in the memory configuration.
Methods:
create_directory(directory_path):
Create a directory if it does not exist.
remove_directory(directory_path):
Removes the specified directory.
"""
def __init__(self) -> None:
with open(here("configs/app_config.yml")) as cfg:
app_config = yaml.load(cfg, Loader=yaml.FullLoader)
# llm configs
self.gpt_model = app_config["llm_config"]["gpt_model"]
self.llama3_70bmodel = app_config["llm_config"]["llama3_70bmodel"]
self.llm_system_role = app_config["llm_config"]["llm_system_role"]
self.persist_directory = str(here(app_config["directories"]["persist_directory"])) # converting to string for adding in chromadb backend: self._settings.require("persist_directory") + "/chroma.sqlite3"
self.custom_persist_directory = str(here(app_config["directories"]["custom_persist_directory"]))
self.embedding_model = OpenAIEmbeddings()
# Retrieval configs
self.data_directory = app_config["directories"]["data_directory"]
self.k = app_config["retrieval_config"]["k"]
self.num_of_final_doc = app_config["retrieval_config"]["num_of_final_doc"]
self.embedding_model_engine = app_config["embedding_model_config"]["engine"]
self.chunk_size = app_config["splitter_config"]["chunk_size"]
self.chunk_overlap = app_config["splitter_config"]["chunk_overlap"]
self.temperature = app_config["llm_config"]["temperature"]
# Memory
self.qa_pair_count = app_config["memory"]["qa_pair_count"]
# Load OpenAI credentials
#self.load_openai_cfg()
# clean up the upload doc vectordb if it exists
self.create_directory(self.persist_directory)
self.remove_directory(self.custom_persist_directory)
def create_directory(self, directory_path: str):
"""
Create a directory if it does not exist.
Parameters:
directory_path (str): The path of the directory to be created.
"""
if not os.path.exists(directory_path):
os.makedirs(directory_path)
def remove_directory(self, directory_path: str):
"""
Removes the specified directory.
Parameters:
directory_path (str): The path of the directory to be removed.
Raises:
OSError: If an error occurs during the directory removal process.
Returns:
None
"""
if os.path.exists(directory_path):
try:
shutil.rmtree(directory_path)
print(
f"The directory '{directory_path}' has been successfully removed.")
except OSError as e:
print(f"Error: {e}")
else:
print(f"The directory '{directory_path}' does not exist.") |