Add application file

Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.9
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirement.txt requirement.txt
+
+COPY --chown=user ./lib/aicloudlibs-0.1.0-py3-none-any.whl /lib/
+
+# COPY --chown=user ./lib/better_profanity-2.0.0-py3-none-any.whl /lib/
+
+# COPY --chown=user ./lib/privacy-1.0.9-py3-none-any.whl /lib/
+
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+
+# Expose the port (default for Hugging Face is 7860)
+EXPOSE 7860
+
+# CMD to run the FastAPI app with Uvicorn
+CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "7860"]

config/logger.ini

@@ -0,0 +1,20 @@
+; Copyright 2024 Infosys Ltd.
+
+; Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+; to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+; and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+; The above copyright notice and this permission notice shall be included in all copies 
+; or substantial portions of the Software.
+
+; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+; INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+; AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+; DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+; OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+[logDetails]
+LOG_LEVEL=ERROR
+FILE_NAME=responsible-ai-servicelogs
+VERBOSE=False
+LOG_DIR=/responsible-ai/logs

config/metadata.yaml

@@ -0,0 +1,25 @@
+# Copyright 2024 Infosys Ltd.
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+# to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+# and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all copies 
+# or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+# AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+openapi_url: /rai/v1/llm-explainability/openapi.json
+docs_url: /rai/v1/llm-explainability/docs
+title: Infosys Responsible AI - responsible-ai-llm-explain - OpenAPI 3.0
+description: API specs for Infosys Responsible AI LLM-Explainability pillar in OpenAPI 3.0 format
+contact:
+    email: [email protected]
+version: v$version
+openapi_tags:
+    - name: LLM-Explainability
+      description: Operations required for explainability of an LLM

models/gpt3tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1588f575baa6928e93cd483b385affa2744670c6a2fbe0eeffe5c1ad5eec53e
+size 59

requirement.txt

@@ -0,0 +1,19 @@
+fastapi
+pydantic
+uvicorn
+python-dotenv
+pyyaml
+scikit-learn
+seaborn
+tenacity
+tiktoken
+openai
+kaleido
+plotly
+backoff
+numpy==1.26.4
+beautifulsoup4
+rouge_score
+pip-system-certs
+aiohttp
+../lib/aicloudlibs-0.1.0-py3-none-any.whl

requirements/requirement.txt

@@ -0,0 +1,19 @@
+fastapi
+pydantic
+uvicorn
+python-dotenv
+pyyaml
+scikit-learn
+seaborn
+tenacity
+tiktoken
+openai
+kaleido
+plotly
+backoff
+numpy==1.26.4
+beautifulsoup4
+rouge_score
+pip-system-certs
+aiohttp
+../lib/aicloudlibs-0.1.0-py3-none-any.whl

setup.py

@@ -0,0 +1,42 @@
+''' 
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+'''
+
+from setuptools import find_packages,setup
+from pathlib import Path
+
+def get_install_requires() -> list[str]:
+    """Returns requirements.txt parsed to a list"""
+    fname = Path(__file__).parent / 'requirement/requirements.txt'
+    targets = []
+    if fname.exists():
+        with open(fname, 'r') as f:
+            targets = f.read().splitlines()
+    return targets
+
+if __name__ == '__main__':
+        setup(
+            name='responsible-ai-llm_explain',
+            url="responsible_ai_llm_explain",
+            packages=find_packages(),
+            include_package_data=True,
+            python_requires='>=3.6',
+            version='0.1.0',
+            description='AI Cloud Project Management Services',
+            install_requires=get_install_requires(),
+            license='MIT',
+        )

src/llm_explain/.env

@@ -0,0 +1,9 @@
+AZURE_OPENAI_API_KEY = "${apikey}"
+AZURE_OPENAI_API_VERSION = "${apiversion}"
+AZURE_OPENAI_ENDPOINT = "${azureendpoint}"
+AZURE_DEPLOYMENT_ENGINE = "${engine}"
+SERPER_KEY = "${serperkey}"
+# ALLOWED_ORIGINS = "${allowedorigins}"
+ALLOWED_ORIGINS = "*"
+ERROR_LOG_TELEMETRY_URL = "${errorlogtelemetryurl}"
+TELEMETRY_FLAG = "False"

src/llm_explain/__init__.py

@@ -0,0 +1,16 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''

src/llm_explain/config/__init__.py

@@ -0,0 +1,16 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''

src/llm_explain/config/config.py

@@ -0,0 +1,40 @@
+''' 
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from configparser import ConfigParser
+import yaml
+
+def readConfig(section,filename):
+
+    parser = ConfigParser() # create a parser
+    parser.read(filename) # read config file
+
+    # get section, default to postgresql
+    db = {}
+    if parser.has_section(section):
+        params = parser.items(section)
+        for param in params:
+            db[param[0]] = param[1]
+    else:
+        raise Exception('Section {0} not found in the {1} file'.format(section, filename))
+
+    return db
+
+def read_config_yaml(filename):
+    with open(filename) as config_file:
+        config_details = yaml.safe_load(config_file)
+    return config_details

src/llm_explain/config/logger.py

@@ -0,0 +1,164 @@
+''' 
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+import datetime
+import logging
+import os
+import sys
+from .config import readConfig
+import contextvars
+
+request_ids=[]
+request_id_var = contextvars.ContextVar("request_id_var")
+
+class CustomLogger(logging.getLoggerClass()):
+    def __init__(self):
+        """Create a custom logger with the specified `name`. When `log_dir` is None, a simple
+        console logger is created. Otherwise, a file logger is created in addition to the console
+        logger.
+
+        By default, the five standard logging levels (DEBUG through CRITICAL) only display
+        information in the log file if a file handler is added to the logger, but **not** to the
+        console.
+        :param name: name for the logger
+        :param verbose: bool: whether the logging should be verbose; if True, then all messages get
+            logged both to stdout and to the log file (if `log_dir` is specified); if False, then
+            messages only get logged to the log file (if `log_dir` is specified)
+        :param log_dir: str: (optional) the directory for the log file; if not present, no log file
+            is created
+        """
+        # Create custom logger logging all five levels
+        BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+        log_cfg_path = os.path.join(BASE_DIR, 'logger.ini')
+        log_params = readConfig('logDetails', log_cfg_path)
+
+        name = log_params['file_name']
+        try:
+            verbose = bool(log_params['verbose'])
+        except:
+            verbose = False
+
+        log_dir = str(log_params['log_dir'])
+
+        super().__init__(name)
+        self.setLevel(logging.DEBUG)
+
+        # Add new logging level
+        logging.addLevelName(logging.INFO, 'INFO')
+
+        # Determine verbosity settings
+        self.verbose = verbose
+
+        # Create stream handler for logging to stdout (log all five levels)
+        self.stdout_handler = logging.StreamHandler(sys.stdout)
+        self.stdout_handler.setLevel(logging.DEBUG)
+        self.stdout_handler.setFormatter(logging.Formatter('%(message)s'))
+        self.enable_console_output()
+
+        self.file_handler = None
+        if log_dir:
+            self.add_file_handler(name, log_dir)
+
+    def add_file_handler(self, name, log_dir):
+        """Add a file handler for this logger with the specified `name` (and store the log file
+        under `log_dir`)."""
+        # Format for file log
+        fmt = '%(asctime)s | %(levelname)9s | %(filename)s:%(lineno)d | %(message)s'
+        formatter = logging.Formatter(fmt)
+
+        # Determine log path and file name; create log path if it does not exist
+        now = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+        log_name = f'{str(name).replace(" ", "_")}_{now}'
+        if not os.path.exists(log_dir):
+            try:
+                os.makedirs(log_dir)
+            except:
+                log_dir = '/tmp' if sys.platform.startswith('linux') else '.'
+
+        log_file = os.path.join(log_dir, log_name) + '.log'
+
+        # Create file handler for logging to a file (log all five levels)
+        self.file_handler = logging.FileHandler(log_file)
+        self.file_handler.setLevel(logging.DEBUG)
+        self.file_handler.setFormatter(formatter)
+        self.addHandler(self.file_handler)
+
+    def has_console_handler(self):
+        return len([h for h in self.handlers if type(h) == logging.StreamHandler]) > 0
+
+    def has_file_handler(self):
+        return len([h for h in self.handlers if isinstance(h, logging.FileHandler)]) > 0
+
+    def disable_console_output(self):
+        if not self.has_console_handler():
+            return
+        self.removeHandler(self.stdout_handler)
+
+    def enable_console_output(self):
+        if self.has_console_handler():
+            return
+        self.addHandler(self.stdout_handler)
+
+    def disable_file_output(self):
+        if not self.has_file_handler():
+            return
+        self.removeHandler(self.file_handler)
+
+    def enable_file_output(self):
+        if self.has_file_handler():
+            return
+        self.addHandler(self.file_handler)
+
+    def framework(self, msg, *args, **kwargs):
+        """Logging method for the FRAMEWORK level. The `msg` gets logged both to stdout and to file
+        (if a file handler is present), irrespective of verbosity settings."""
+        return super().info(msg, *args, **kwargs)
+
+    def _custom_log(self, func, msg, *args, **kwargs):
+        """Helper method for logging DEBUG through CRITICAL messages by calling the appropriate
+        `func()` from the base class."""
+        # Log normally if verbosity is on
+        if self.verbose:
+            return func(msg, *args, **kwargs)
+
+        # If verbosity is off and there is no file handler, there is nothing left to do
+        if not self.has_file_handler():
+            return
+
+        # If verbosity is off and a file handler is present, then disable stdout logging, log, and
+        # finally reenable stdout logging
+        self.disable_console_output()
+        func(msg, *args, **kwargs)
+        self.enable_console_output()
+
+    def debug(self, msg, *args, **kwargs):
+        self._custom_log(super().debug, msg, *args, **kwargs)
+
+    def info(self, msg, *args, **kwargs):
+        self._custom_log(super().info, msg, *args, **kwargs)
+
+    def warning(self, msg, *args, **kwargs):
+        self._custom_log(super().warning, msg, *args, **kwargs)
+
+    def error(self, msg, *args, **kwargs):
+        self._custom_log(super().error, msg, *args, **kwargs)
+
+    def critical(self, msg, *args, **kwargs):
+        self._custom_log(super().critical, msg, *args, **kwargs)
+
+if __name__ == "__main__":
+    CustomLogger()

src/llm_explain/exception/constants.py

@@ -0,0 +1,44 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+# Success Response Codes
+HTTP_STATUS_CODES = {
+    "OK": 200,
+    "NOT_FOUND": 404,
+    "METHOD_NOT_ALLOWED": 405,
+    "BAD_REQUEST": 400,
+    "CONFLICT": 409,
+    "UNSUPPORTED_MEDIA_TYPE": 415,
+    "UNPROCESSABLE_ENTITY": 422,
+    "SERVICE_UNAVAILABLE": 503,
+    "INTERNAL_SERVER_ERROR": 500,
+    "DATA_ERROR": 500,
+}
+
+# Message Constants
+HTTP_STATUS_MESSAGES = {
+    "OK": "Request processed successfully",
+    "NOT_FOUND": "Resource not found",
+    "METHOD_NOT_ALLOWED": "Method not allowed",
+    "BAD_REQUEST": "Bad request",
+    "CONFLICT": "Conflict",
+    "UNSUPPORTED_MEDIA_TYPE": "Unsupported media type",
+    "UNPROCESSABLE_ENTITY": "Unprocessable entity",
+    "SERVICE_UNAVAILABLE": "Service unavailable",
+    "INTERNAL_SERVER_ERROR": "Internal server error",
+    "DATABASE_CONNECTION_REFUSED": "Database connection refused"
+}

src/llm_explain/exception/exception.py

@@ -0,0 +1,29 @@
+''' 
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+"""
+fileName: exception.py
+description: handles usecase module specific exception
+"""
+
+class CustomException(Exception):
+    def __init__(self, message, error_code):
+        super().__init__(message)
+        self.error_code = error_code
+
+    def __str__(self):
+        return f"[Error {self.error_code}]: {self.message}"

src/llm_explain/exception/global_exception.py

@@ -0,0 +1,60 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from .constants import HTTP_STATUS_CODES
+from abc import ABC
+
+class BenchmarkExceptions(Exception, ABC):
+    """
+    Abstract base class of all Aicloud DB exceptions.
+    """
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+class DBConnectionError(BenchmarkExceptions):
+     def __init__(self, name):
+        self.status_code = HTTP_STATUS_CODES.SERVICE_UNAVAILABLE
+        self.message = HTTP_STATUS_CODES.DATABASE_CONNECTION_REFUSED + name
+
+class NotSupportedError(BenchmarkExceptions):
+     def __init__(self, msg):
+        self.status_code = HTTP_STATUS_CODES.METHOD_NOT_ALLOWED
+        if not msg:
+            self.message = HTTP_STATUS_CODES.METHOD_NOT_ALLOWED
+        else: 
+            self.message=msg
+
+class InternalServerError(BenchmarkExceptions):
+     def __init__(self, msg):
+        self.status_code = HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR
+        if not msg:
+            self.message = HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR
+        else: 
+            self.message=msg
+
+class MethodArgumentNotValidException(BenchmarkExceptions):
+    def __init__(self, msg):
+        self.status_code = HTTP_STATUS_CODES.BAD_REQUEST
+        if not msg:
+            self.message = HTTP_STATUS_CODES.BAD_REQUEST
+        else: 
+            self.message=msg
+
+class UnSupportedMediaTypeException(BenchmarkExceptions):
+    def __init__(self, contentTypeStr):
+        self.status_code = HTTP_STATUS_CODES.UNSUPPORTED_MEDIA_TYPE
+        self.message = HTTP_STATUS_CODES.UNSUPPORTED_MEDIA_TYPE + contentTypeStr

src/llm_explain/exception/global_exception_handler.py

@@ -0,0 +1,40 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from fastapi.exceptions import RequestValidationError
+from fastapi.encoders import jsonable_encoder
+from fastapi.responses import JSONResponse
+from .global_exception import UnSupportedMediaTypeException
+from .constants import HTTP_STATUS_CODES
+
+def validation_error_handler(exc: RequestValidationError):
+    return JSONResponse(
+        status_code=HTTP_STATUS_CODES["UNPROCESSABLE_ENTITY"],
+        content=jsonable_encoder({"ERROR": exc.errors()}),
+    )
+
+def unsupported_mediatype_error_handler(exc: UnSupportedMediaTypeException):
+    return JSONResponse(
+        status_code=HTTP_STATUS_CODES["UNSUPPORTED_MEDIA_TYPE"],
+        content=jsonable_encoder({"ERROR": str(exc.message)}),
+    )
+
+def http_exception_handler(exc):
+    return JSONResponse(
+        status_code=exc.status_code,
+        content=jsonable_encoder({"ERROR": str(exc.detail)}),
+    )

src/llm_explain/mappers/__init__.py

@@ -0,0 +1,16 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''

src/llm_explain/mappers/mappers.py

@@ -0,0 +1,101 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+"""
+module: LLM Explainability
+fileName: mappers.py
+description: A Pydantic model object for usecase entity model
+             which maps the data model to the usecase entity schema
+"""
+
+from pydantic import BaseModel, Field
+from typing import Optional, List, Dict
+
+class SentimentAnalysisRequest(BaseModel):
+    inputPrompt: str = Field(example="Unfortunately the movie served with bad visuals but the actors performed well")
+
+    class Config:
+        from_attributes = True
+
+class SentimentAnalysisResponse(BaseModel):
+    explanation: List
+
+    class Config:
+        from_attributes = True
+
+class UncertainityRequest(BaseModel):
+    inputPrompt: str = Field(example="Who are the co-founders of Infosys?")
+    response: str = Field(example="Infosys was co-founded by Narayana Murthy along with six other engineers: Nandan Nilekani, S. Gopalakrishnan (Kris), S. D. Shibulal, K. Dinesh, N. S. Raghavan, and Ashok Arora. Established in 1981, Infosys started with a modest capital of $250 and has since grown into one of the largest IT services companies in the world. Narayana Murthy, often regarded as the face of Infosys, played a pivotal role in shaping the company's culture and vision, while the combined efforts of all co-founders contributed to its remarkable growth and success in the global IT industry.")
+
+    class Config:
+        from_attributes = True
+
+class UncertainityResponse(BaseModel):
+    uncertainty: Dict = Field(example={"score": 0.5, "explanation": "The response is uncertain as it mentions the co-founders of Infosys without providing specific details.", "recommendation": "Maintain the grammatical correctness and focus on providing additional information."})
+    coherence: Dict = Field(example={"score": 0.8, "explanation": "The response is relevant to the prompt as it provides information about the co-founders of Infosys.", "recommendation": "Maintain the grammatical correctness and focus on providing additional information."})
+    time_taken: float = Field(example=0.5)
+    
+    class Config:
+        from_attributes = True
+
+class TokenImportanceResponse(BaseModel):
+    token_importance_mapping:List
+    image_data:Optional[List]
+    token_heatmap:Optional[str]
+    time_taken: float = Field(example=0.5)
+    
+    class Config:
+        from_attributes = True
+
+class TokenImportanceRequest(BaseModel):
+    inputPrompt: str = Field(example="Who are the co-founders of Infosys?")
+    modelName: Optional[str] = Field(example="GPT")
+
+    class Config:
+        from_attributes = True
+
+class GoTRequest(BaseModel):
+    inputPrompt: str = Field(example="Who are the co-founders of Infosys?")
+    modelName: Optional[str] = Field(example="gpt4")
+
+    class Config:
+        from_attributes = True
+        
+class GoTResponse(BaseModel):
+    final_thought: str = Field(example='The co-founders of Infosys are N. R. Narayana Murthy, ...')
+    score: float = Field(example=9.5)
+    cost_incurred: float = Field(example=0.5)
+    consistency_level: str = Field(example='High Consistent')
+    time_taken: float = Field(example=0.5)
+    
+    class Config:
+        from_attributes = True
+
+class SafeSearchRequest(BaseModel):
+    inputPrompt: str = Field(example="Who are the co-founders of Infosys?")
+    llm_response: str = Field(example="Infosys, a global leader in technology services and consulting, was founded in 1981 by seven visionaries: N.R. Narayana Murthy, Nandan Nilekani, S. Gopalakrishnan, S.D. Shibulal, K. Dinesh, N.S. Raghavan, and Ashok Arora. These co-founders combined their expertise and entrepreneurial spirit to create a company that has since grown into one of the largest and most respected IT services firms in the world. Infosys, headquartered in Bangalore, India, has been instrumental in the global IT revolution, providing innovative solutions and services to clients across various industries. The founders' commitment to excellence and their forward-thinking approach laid a strong foundation for the company's enduring success.")
+
+    class Config:
+        from_attributes = True
+
+class SafeSearchResponse(BaseModel):
+    internetResponse: List = Field(example="The co-founders of Infosys are N. R. Narayana Murthy, ...")
+    metrics: List = Field(examples=[])
+    time_taken: float = Field(example=0.5)
+
+    class Config:
+        from_attributes = True

src/llm_explain/routing/__init__.py

@@ -0,0 +1,16 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''

src/llm_explain/routing/explain_router.py

@@ -0,0 +1,204 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from llm_explain.mappers.mappers import UncertainityResponse, UncertainityRequest, \
+        TokenImportanceResponse, TokenImportanceRequest, GoTResponse, GoTRequest, \
+        SafeSearchResponse, SafeSearchRequest, SentimentAnalysisRequest, SentimentAnalysisResponse
+from llm_explain.service.service import ExplainService as service
+from llm_explain.config.logger import CustomLogger, request_id_var
+from fastapi import APIRouter, HTTPException
+from datetime import  datetime
+import concurrent.futures
+import requests
+import asyncio
+import uuid
+import os
+
+explanation = APIRouter()
+
+log = CustomLogger()
+
+telemetry_flag = os.getenv("TELEMETRY_FLAG")
+tel_error_url = os.getenv("ERROR_LOG_TELEMETRY_URL")
+
+## FUNCTION FOR FAIL_SAFE TELEMETRY
+def send_telemetry_request(explainability_telemetry_request, url):
+    try:
+        response = requests.post(url, json=explainability_telemetry_request)
+        response.raise_for_status()
+        response_data = response.json()
+        log.info(f"Telemetry response: {response_data}")
+    except Exception as e:
+        log.error(str(e))
+        raise HTTPException(
+            status_code=500,
+            detail="Please check with administration!!",
+            headers={"X-Error": "Please check with administration!!"})
+
+def telemetry_error_logging(cie, request_id_var, api_endpoint):
+    function_name = None
+    # Get the traceback of the exception
+    current_tb = cie.__traceback__
+    # Traverse to the first traceback not from site-packages
+    while current_tb:
+        # Check if the traceback is not from site-packages
+        if "site-packages" not in current_tb.tb_frame.f_code.co_filename:
+            # Get the function name and file name
+            function_name = current_tb.tb_frame.f_code.co_name
+        
+        # Move to the next traceback
+        current_tb = current_tb.tb_next
+    
+    if telemetry_flag== "True":
+        error_input = {
+            "tenetName": "Explainability",
+            "errorCode": function_name +'_'+ request_id_var.get(),
+            "errorMessage": str(cie),
+            "apiEndPoint": api_endpoint,
+            "errorRequestMethod": "POST"
+        }
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            executor.submit(send_telemetry_request, error_input, tel_error_url)
+
+@explanation.post('/llm-explainability/sentiment-analysis', 
+                  response_model = SentimentAnalysisResponse, 
+                  summary = "Sentiment analysis of the prompt along with token importance")
+def sentiment_analysis(payload: SentimentAnalysisRequest):
+    id = uuid.uuid4().hex
+    request_id_var.set(id)
+    log.info("Entered create usecase routing method")
+    try:
+        start_time = datetime.now()
+        log.info(f"start_time: {start_time}")
+        log.info("before invoking sentiment_analysis service ")
+        response = service.sentiment_analysis(payload)
+        log.info("after invoking sentiment_analysis service ")
+        log.info("exit create usecase routing method")
+        end_time = datetime.now()
+        log.info(f"end_time: {end_time}")
+        total_time = end_time - start_time
+        log.info(f"total_time: {total_time}")
+        return response
+    
+    except Exception as cie:
+        log.error(cie)
+        telemetry_error_logging(cie, request_id_var, "/llm-explainability/sentiment-analysis")
+        log.info("exit router sentiment_analysis method")
+        raise HTTPException(status_code=500, detail=str(cie))
+
+@explanation.post('/llm-explainability/uncertainty', 
+                  response_model = UncertainityResponse, 
+                  summary = "Get uncertainty scores for the given input")
+def calculate_uncertainty(payload: UncertainityRequest):
+    id = uuid.uuid4().hex
+    request_id_var.set(id)
+    log.info("Entered create usecase routing method")
+    try:
+        start_time = datetime.now()
+        log.info(f"start_time: {start_time}")
+        log.info("before invoking local_explanation service ")
+        response = asyncio.run(service.local_explanation(payload))
+        log.info("after invoking local_explanation service ")
+        log.info("exit create usecase routing method")
+        end_time = datetime.now()
+        log.info(f"end_time: {end_time}")
+        total_time = end_time - start_time
+        log.info(f"total_time: {total_time}")
+        return response
+    except Exception as cie:
+        log.error(cie)
+        telemetry_error_logging(cie, request_id_var, "/llm-explainability/uncertainty")
+        log.info("exit router local_explanation method")
+        raise HTTPException(status_code=500, detail=str(cie))
+
+@explanation.post('/llm-explainability/token-importance',
+                  response_model = TokenImportanceResponse, 
+                  summary = "Get importance for each token in the input prompt")
+def token_importance(payload: TokenImportanceRequest):
+    id = uuid.uuid4().hex
+    request_id_var.set(id)
+    log.info("Entered create usecase routing method")
+    try:
+        start_time = datetime.now()
+        log.info(f"start_time: {start_time}")
+        log.info("before invoking token_importance service ")
+        response = asyncio.run(service.token_importance(payload))
+        log.info("after invoking token_importance service ")
+        log.info("exit create usecase routing method")
+        end_time = datetime.now()
+        log.info(f"end_time: {end_time}")
+        total_time = end_time - start_time
+        log.info(f"total_time: {total_time}")
+            
+        return response
+    except Exception as cie:
+        log.error(cie)
+        telemetry_error_logging(cie, request_id_var, "/llm-explainability/token-importance")
+        log.info("exit router token_importance method")
+        raise HTTPException(status_code=500, detail=str(cie))
+
+@explanation.post('/llm-explainability/got', 
+                  response_model = GoTResponse, 
+                  summary = "Graph-of-Thoughts Reasoning")
+def graph_of_thoughts(payload: GoTRequest):
+    id = uuid.uuid4().hex
+    request_id_var.set(id)
+    log.info("Entered create usecase routing method")
+    try:
+        start_time = datetime.now()
+        log.info(f"start_time: {start_time}")
+        log.info("before invoking graph_of_thoughts service ")
+        response = asyncio.run(service.graph_of_thoughts(payload))
+        log.info("after invoking graph_of_thoughts service ")
+        log.info("exit create usecase routing method")
+        end_time = datetime.now()
+        log.info(f"end_time: {end_time}")
+        total_time = end_time - start_time
+        log.info(f"total_time: {total_time}")
+
+        return response
+    except Exception as cie:
+        log.error(cie)
+        telemetry_error_logging(cie, request_id_var, "/llm-explainability/got")
+        log.info("exit router graph_of_thoughts method")
+        raise HTTPException(status_code=500, detail=str(cie))
+    
+@explanation.post('/llm-explainability/serper_response',
+                  response_model = SafeSearchResponse,
+                  summary = "Verify LLM response with Google Search")
+def searchAugmentation(payload: SafeSearchRequest):
+    id = uuid.uuid4().hex
+    request_id_var.set(id)
+    log.info("Entered create usecase routing method")
+    try:
+        start_time = datetime.now()
+        log.info(f"start_time: {start_time}")
+        log.info("before invoking search_augmentation service ")
+        response = asyncio.run(service.search_augmentation(payload))
+        log.info("after invoking search_augmentation service ")
+        log.info("exit create usecase routing method")
+        end_time = datetime.now()
+        log.info(f"end_time: {end_time}")
+        total_time = end_time - start_time
+        log.info(f"total_time: {total_time}")
+            
+        return response
+    except Exception as cie:
+        log.error(cie)
+        telemetry_error_logging(cie, request_id_var, "/llm-explainability/serper_response")
+        log.info("exit router search_augmentation method")
+        raise HTTPException(status_code=500, detail=str(cie))

src/llm_explain/service/__init__.py

@@ -0,0 +1,16 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''

src/llm_explain/service/responsible_ai_explain.py

@@ -0,0 +1,600 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from llm_explain.utility.query_serper import GoogleSerperAPIWrapper
+from llm_explain.utility import got as GraphOfThoughts
+from llm_explain.utility.prompts.base import Prompt
+from llm_explain.config.logger import CustomLogger
+from llm_explain.utility.utility import Utils
+from llm_explain.utility.azure import Azure
+from sklearn.metrics.pairwise import cosine_similarity
+from json.decoder import JSONDecodeError
+from scipy.stats import gaussian_kde
+from itertools import combinations
+import plotly.graph_objects as go
+import matplotlib.pyplot as plt
+from openai import AzureOpenAI
+import pandas as pd
+import numpy as np
+import matplotlib
+import asyncio
+import base64
+import time
+import html
+import json
+import ast
+import os
+
+log = CustomLogger()
+
+class ResponsibleAIExplain:
+
+    @staticmethod
+    def llm_response_to_json(response):
+        """
+        Converts a substring of the given response that is in JSON format into a Python dictionary.
+        
+        This function searches for the first occurrence of '{' and the last occurrence of '}' to find the JSON substring.
+        It then attempts to parse this substring into a Python dictionary. If the parsing is successful, the dictionary
+        is returned. If the substring is not valid JSON, the function will return None.
+        
+        Parameters:
+        - response (str): The response string that potentially contains JSON content.
+        
+        Returns:
+        - dict: A dictionary representation of the JSON substring found within the response.
+        - None: If no valid JSON substring is found or if an error occurs during parsing.
+        """
+        try:
+            result = None # Initialize result to None in case no valid JSON is found
+
+            # Find the start index of the first '{' character and end index of the last '}' character
+            start_index = response.find('{')
+            end_index = response.rfind('}')
+
+            # Check if both '{' and '}' are found and '{' comes before '}'
+            if start_index != -1 and end_index != -1 and end_index > start_index:
+                json_content = response[start_index:end_index+1] # Extract the substring that is potentially in JSON format
+                result = json.loads(json_content) # Attempt to parse the JSON substring into a Python dictionary
+            
+            return result
+        
+        except Exception as e:
+            # Log the exception if any error occurs during parsing
+            log.error(f"An error occurred while parsing JSON from response: {e}", exc_info=True)
+            raise
+
+    async def analyze_heatmap(df_input):
+        base64_encoded_imgs=[]
+        try:
+            
+            df = df_input.copy()
+
+            if "token" not in df.columns or "importance_value" not in df.columns:
+                raise ValueError("The DataFrame must contain 'token' and 'importance_value' columns.")
+
+            df["Position"] = range(len(df))
+            
+            # Calculate histogram data
+            hist, bin_edges = np.histogram(df["importance_value"], bins=20)
+            # Get the viridis colormap
+            viridis = plt.get_cmap("viridis")
+            # Initialize the figure
+            fig = go.Figure()
+            
+            # Create the histogram bars with viridis coloring
+            for i, freq in enumerate(hist):
+                color = f"rgb({int(viridis(i / (len(bin_edges) - 1))[0] * 255)}, {int(viridis(i / (len(bin_edges) - 1))[1] * 255)}, {int(viridis(i / (len(bin_edges) - 1))[2] * 255)})"
+                fig.add_trace(
+                    go.Bar(
+                        x=[(bin_edges[i] + bin_edges[i + 1]) / 2],
+                        y=[freq],
+                        width=np.diff(bin_edges)[i],
+                        marker=dict(color=color),
+                    )
+                )
+            
+            # Calculate and add the KDE line
+            x_kde = np.linspace(min(df["importance_value"]), max(df["importance_value"]), 500)
+            kde = gaussian_kde(df["importance_value"])
+            y_kde = kde(x_kde) * sum(hist) * (bin_edges[1] - bin_edges[0])
+            fig.add_trace(
+                go.Scatter(
+                    x=x_kde, y=y_kde, mode="lines", line_shape="spline", line=dict(color="red")
+                )
+            )
+            # Additional styling
+            fig.update_layout(
+                title=" Distribution of Importance Scores",
+                title_font={'size': 25},
+                xaxis_title="Importance Value",
+                yaxis_title="Frequency",
+                showlegend=False,
+            )
+            
+            img_bytes = fig.to_image(format="png")
+            
+            
+            img_base64 = base64.b64encode(img_bytes).decode("utf-8")
+           
+            base64_encoded_imgs.append(img_base64)
+            
+            # Normalize the importance values
+            min_val = df["importance_value"].min()
+           
+            max_val = df["importance_value"].max()
+            
+            normalized_values = (df["importance_value"] - min_val) / (max_val - min_val)
+            
+            # Initialize the figure
+            fig = go.Figure()
+            
+            # Create the bars, colored based on normalized importance_value
+            for i, (token, norm_value) in enumerate(zip(df["token"], normalized_values)):
+                color = f"rgb({int(viridis(norm_value)[0] * 255)}, {int(viridis(norm_value)[1] * 255)}, {int(viridis(norm_value)[2] * 255)})"
+                fig.add_trace(
+                    go.Bar(
+                        x=[i],  # Use index for x-axis
+                        y=[df["importance_value"].iloc[i]],
+                        width=0.9,  # Set the width to make bars touch each other
+                        marker=dict(color=color),
+                    )
+                )
+            # Additional styling
+            fig.update_layout(
+                title="Importance Score per Token",
+                title_font={'size': 25},
+                xaxis_title="Token",
+                yaxis_title="Importance Value",
+                showlegend=False,
+                bargap=0,  # Remove gap between bars
+                xaxis=dict(  # Set tick labels to tokens
+                    tickmode="array",
+                    tickvals=list(range(len(df["token"]))),
+                    ticktext=list(df["token"]),
+                ),
+                autosize=False,  # Disable automatic sizing
+                width= max(10, len(df["token"]) * 0.3) * 100,  # Convert to pixels
+            )
+            # Rotate x-axis labels by 45 degrees
+            fig.update_xaxes(tickangle=-45)
+            
+            img_bytes = fig.to_image(format="png")
+            img_base64 = base64.b64encode(img_bytes).decode("utf-8")
+            base64_encoded_imgs.append(img_base64)
+
+            top_10_important = df.nlargest(10, 'importance_value')
+            top_10=top_10_important.to_dict(orient='records')
+
+            # Extract the importance scores
+            importance_values = df["importance_value"].values
+
+            # Normalize the importance scores to be between 0 and 1
+            min_val = np.min(importance_values)
+            max_val = np.max(importance_values)
+
+            if max_val - min_val != 0:
+                normalized_importance_values = (importance_values - min_val) / (max_val - min_val)
+            else:
+                normalized_importance_values = np.zeros_like(importance_values)
+
+            # Generate a colormap for the heatmap
+            cmap = matplotlib.colormaps["inferno"]
+
+            # Helper function to determine the text color based on the background color
+            def get_text_color(bg_color):
+                brightness = 0.299 * bg_color[0] + 0.587 * bg_color[1] + 0.114 * bg_color[2]
+                if brightness < 0.5:
+                    return "white"
+                else:
+                    return "black"
+
+            # Initialize HTML string
+            html_string = ""
+
+            # Loop over tokens and construct the HTML string
+            for idx, (token, importance) in df_input.iterrows():
+                rgba = cmap(normalized_importance_values[idx])
+                bg_color = rgba[:3]
+                text_color = get_text_color(bg_color)
+                
+                # Explicitly handle special characters
+                token_escaped = html.escape(token).replace('`', '&#96;').replace('$', '&#36;')  # Handle backticks and dollar signs
+                html_string += f"<span style='background-color: rgba({int(bg_color[0]*255)}, {int(bg_color[1]*255)}, {int(bg_color[2]*255)}, 1); color: {text_color};'>{token_escaped}</span> "
+            
+            return top_10,base64_encoded_imgs,html_string
+        except Exception as e:
+            log.error(e, exc_info=True)
+            raise
+
+    async def calculate_uncertainty(n : int, prompt: str):
+        try:
+            max_tokens=1000
+            client = AzureOpenAI(
+                    api_key = os.getenv("AZURE_OPENAI_API_KEY") ,
+                    api_version = os.getenv("AZURE_OPENAI_API_VERSION") ,
+                    azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") 
+                )
+
+            try:
+                response = client.chat.completions.create(
+                    n=n,
+                    model=os.getenv("AZURE_DEPLOYMENT_ENGINE"), # model = "deployment_name".
+                    messages=[
+                        {"role": "system", "content": "Assistant is a large language model trained by OpenAI."},
+                        {"role": "user", "content": prompt}
+                    ],
+                    logprobs=True,
+                    top_logprobs=2,
+                    max_tokens=100 
+                )
+            except Exception as e:
+                log.error("An error occurred while calling the AzureOpenAI API", exc_info=True)
+                raise Exception                                                                     
+            cc=response.choices
+            response_object ={}
+            choices = []
+            for i,c in enumerate(cc):
+                contents=c.logprobs.content
+                choice_i={
+                    "text": c.message.content
+                }
+                logprobs = {}
+                token_logprobs = []
+                tokens=[]
+                top_logprobs=[]
+                for content in contents:
+                    token_logprobs.append(content.logprob)
+                    temp={}
+                    tokens.append(content.token)
+                    top_props=content.top_logprobs
+                    for k in top_props:
+                        temp[k.token]=k.logprob
+                    top_logprobs.append(temp)
+                logprobs["token_logprobs"]=token_logprobs
+                logprobs["tokens"]=tokens
+                logprobs["top_logprobs"]=top_logprobs
+                choice_i["logprobs"]=logprobs
+                choice_i["index"]=i
+        
+                choices.append(choice_i)
+            response_object["choices"]=choices
+    
+            entropies = []
+            distances = []
+            choice_embeddings = []
+            choice_embedding_tasks = [Utils.get_embedding(choice['text']) for choice in response_object['choices']]
+            choice_embeddings = await asyncio.gather(*choice_embedding_tasks)
+
+            async def process_choice(choice, choice_embedding):
+                top_logprobs_list = choice['logprobs']['top_logprobs']
+                mean_cosine_distances = []
+                normalized_entropies = []
+            
+                tasks = [Utils.process_token_async(i, top_logprobs_list, choice, choice_embedding, max_tokens) for i in range(len(top_logprobs_list))]
+                results = await asyncio.gather(*tasks)
+            
+                for mean_distance, normalized_entropy in results:
+                    mean_cosine_distances.append(mean_distance)
+                    normalized_entropies.append(normalized_entropy)
+            
+                return mean_cosine_distances, normalized_entropies
+        
+    
+            choice_tasks = [process_choice(choice, emb) for choice, emb in zip(response_object['choices'], choice_embeddings)]
+            results = await asyncio.gather(*choice_tasks)
+        
+    
+            for mean_cosine_distances, normalized_entropies in results:
+                distances.append(mean_cosine_distances)
+                entropies.append(normalized_entropies)
+        
+            choice_distances = []
+            for emb1, emb2 in combinations(choice_embeddings, 2):
+                cosine_sim = cosine_similarity(emb1.reshape(1, -1), emb2.reshape(1, -1))[0][0]
+                choice_distances.append(1 - cosine_sim)
+            mean_choice_distance = np.mean(choice_distances)
+            uncertainty_scores = {'entropies': entropies, 'distances': distances, 'mean_choice_distance': mean_choice_distance}
+            return Utils.display_metrics(uncertainty_scores, response_object, n)
+        except Exception as e:
+            log.error(e, exc_info=True)
+            raise Exception
+        
+    @staticmethod
+    def normalize_scores(dict_list):
+        try:
+            # Calculate the total sum of all importance scores
+            total_sum = sum(d['importance_score'] for d in dict_list)
+            
+            # If the total sum is zero, return the original list (to handle cases where all scores are zero)
+            if total_sum == 0:
+                return dict_list
+
+            # Normalize the scores to ensure their sum equals 100
+            normalized_scores = [round((d['importance_score'] / total_sum) * 100) for d in dict_list]
+            
+            # Adjust the scores to ensure their sum is exactly 100
+            adjustment = 100 - sum(normalized_scores)
+            normalized_scores[0] += adjustment
+
+            # Update the original list with normalized scores
+            for i, d in enumerate(dict_list):
+                d['importance_score'] = normalized_scores[i]
+            
+            return dict_list
+        
+        except KeyError as e:
+            log.error(f"KeyError: Missing key in one of the dictionaries - {e}")
+            raise
+        except TypeError as e:
+            log.error(f"TypeError: Invalid type encountered - {e}")
+            raise
+        except Exception as e:
+            log.error(f"An unexpected error occurred: {e}")
+            raise
+    
+    @staticmethod
+    def filter_token_importance(scores, anchors):
+        import re
+        try:
+            # Split each phrase in anchors into individual words, remove special characters, and convert to lowercase
+            anchors = [re.sub(r'\W+', '', word).lower() for anchor in anchors for word in anchor.split()]
+            
+            importance_scores = [] # Initialize a list to store the importance scores of the anchors
+            for score in scores: # Iterate through the scores list
+                cleaned_token = re.sub(r'\W+', '', str(score['token'])).lower()
+                if cleaned_token in anchors: # Check if the token value is in the anchors list
+                    importance_scores.append(score['importance_score']) # Append the importance score to the list
+            
+            # Calculate the remaining importance score
+            x = 100 - sum(importance_scores) 
+
+            filtered_tokens = []
+            for score in scores: # Iterate through the scores list
+                cleaned_token = re.sub(r'\W+', '', str(score['token'])).lower()
+                if cleaned_token in anchors: # Check if the token value is in the anchors list
+                    updated_importance = {'token': score['token'],
+                                        'importance_score': score['importance_score'] + (x / len(importance_scores)),
+                                        'position': score['position']}
+                    filtered_tokens.append(updated_importance) # Append the updated importance score to the new list
+            return filtered_tokens
+        
+        except KeyError as e:
+            log.error(f"KeyError: Missing key in one of the dictionaries - {e}")
+            raise
+        except TypeError as e:
+            log.error(f"TypeError: Invalid type encountered - {e}")
+            raise
+        except ZeroDivisionError as e:
+            log.error(f"ZeroDivisionError: Division by zero encountered - {e}")
+            raise
+        except Exception as e:
+            log.error(f"An unexpected error occurred: {e}")
+            raise
+        
+    def sentiment_analysis(text: str, class_names):
+        log.info("Running local_explain")
+        try:
+            start_time = time.time()
+            explanation = Azure().generate(Prompt.get_classification_prompt(text))
+            end_time = time.time()
+            total_time = round(end_time-start_time, 3)
+            explanation = ResponsibleAIExplain.llm_response_to_json(explanation)
+            print('explanation', explanation)
+            # Normalize the importance scores to ensure their sum equals 100
+            explanation['token_importance_mapping'] = ResponsibleAIExplain.normalize_scores(explanation['token_importance_mapping'])
+
+            # Extract the top 10 important tokens
+            tokens_mapping = ResponsibleAIExplain.filter_token_importance(explanation['token_importance_mapping'], explanation['Keywords'])
+
+            return {"predictedTarget": explanation['Sentiment'], 
+                    "anchor": explanation['Keywords'], 
+                    "explanation": explanation['Explanation'],
+                    "token_importance_mapping": tokens_mapping,
+                    "time_taken": total_time}
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise
+
+    async def local_explanation(prompt: str, response: str):
+        try:
+            start_time = time.time()
+            explanation = Azure().generate(Prompt.get_local_explanation_prompt(prompt, response))
+            end_time = time.time()
+            total_time = round(end_time-start_time, 3)
+
+            explanation = ResponsibleAIExplain.llm_response_to_json(explanation)
+            explanation['time_taken'] = total_time
+
+            return explanation
+        except Exception as e:
+            log.error(e, exc_info=True)
+            raise
+        
+    async def process_importance(importance_function, *args, **kwargs):
+        try:
+            start_time = time.time()
+            importance_map = await importance_function(*args, **kwargs)
+            importance_map_df = pd.DataFrame(importance_map, columns=['token', 'importance_value'])
+            offset = importance_map_df['importance_value'].mean()
+
+            importance_log = Utils.scale_importance_log(
+                importance_map, 
+                base=None, 
+                offset=offset, 
+                min_percentile=0,
+                max_percentile=100, 
+                scaling_factor=1, 
+                bias=0
+            )
+            importance_log_df = pd.DataFrame(importance_log, columns=['token', 'importance_value'])
+            end_time = time.time()
+            total_time = round(end_time-start_time, 3)
+            return importance_log_df, total_time
+        
+        except Exception as e:
+            log.error(e, exc_info=True)
+            raise 
+    
+    async def prompt_based_token_importance(prompt):
+       
+        try:
+            start_time = time.time()
+            max_retries = 5
+            attempts = 0
+            while attempts < max_retries:
+                try:
+                    explanation = Azure().generate(Prompt.get_token_importance_prompt(prompt))
+                    # Manually find the JSON substring within the mixed content
+                    start_index = explanation.find('{')
+                    end_index = explanation.rfind('}')
+                    if start_index != -1 and end_index != -1 and end_index > start_index:
+                        json_content = explanation[start_index:end_index+1]
+                        result = json.loads(json_content)
+                        # If JSON loads successfully, break out of the loop
+                        break
+                except JSONDecodeError:
+                    attempts += 1
+                    if attempts == max_retries:
+                        raise Exception("Failed to decode JSON after 5 attempts.")
+                    else:
+                        log.debug(f"JSONDecodeError encountered. Retrying... Attempt {attempts}/{max_retries}")
+                        # Add a delay before the next attempt
+                        time.sleep(2) # Delay for 2 seconds
+                        
+            # Assuming 'result' is a dictionary with "Token" and "Importance Score" as keys, and their values are lists
+            # First, create a DataFrame from the 'result' dictionary
+            tokens = result['Token']
+            scores = result['Importance Score']
+            positions = list(range(1, len(result['Token']) + 1))
+
+            # Find the length of the shortest list
+            min_length = min(len(tokens), len(scores), len(positions))
+
+            # Trim the lists to the length of the shortest list
+            tokens = tokens[:min_length]
+            scores = scores[:min_length]
+            positions = positions[:min_length]
+
+            df = pd.DataFrame({
+                'token': tokens,
+                'importance_value': scores,
+                'position': positions
+            })
+ 
+            df['importance_value'] = df['importance_value'].astype(float)
+ 
+            # Sort the DataFrame by 'Importance Score' in descending order to get the most important tokens first
+            df_sorted = df.sort_values(by='importance_value', ascending=False)
+ 
+            # Select the top 10 important tokens
+            df_top10 = df_sorted.head(10)
+            df_top10.reset_index(drop=True, inplace=True)
+            end_time = time.time()
+            total_time = round(end_time-start_time, 3)
+            top_10, base64_encoded_imgs, token_heatmap = await ResponsibleAIExplain.analyze_heatmap(df_top10[['token', 'importance_value']])
+ 
+            return df_top10.to_dict(orient='records'), base64_encoded_imgs, token_heatmap, total_time
+        
+        except Exception as e:
+            log.error(e, exc_info=True)
+            raise 
+
+    async def graph_of_thoughts(prompt: str, modelName: str):
+        try:
+            start_time = time.time()
+            budget = 30
+            task = "answer the following question"
+            question = prompt
+            approaches = [GraphOfThoughts.got]
+            modelName = modelName
+            
+            formatted_graph, formatted_thoughts = GraphOfThoughts.run(task=task, question=question, 
+                                                                      methods=approaches, 
+                                                                      budget=budget, 
+                                                                      lm_name=modelName)
+            
+            formatted_graph[3]['operation'] = 'final_thought'
+            for i in range(4):
+                thoughts = formatted_graph[i]['thoughts']
+                for j in range(len(thoughts)):
+                    formatted_graph[i]['thoughts'][j]['score'] = round(formatted_graph[i]['thoughts'][j]['score'], 2)
+            end_time = time.time()
+            total_time = round(end_time-start_time, 3)
+
+            return formatted_graph, formatted_thoughts, total_time
+        except Exception as e:
+            log.error(e, exc_info=True)
+            raise
+
+    async def search_augmentation(inputPrompt, llmResponse):
+        try:
+            import datetime
+            current_date = datetime.datetime.now()
+
+            start_time = time.time()
+
+            # Step 1: Generate Facts with LLM response
+            facts = Azure().generate(Prompt.generate_facts_prompt(inputPrompt, llmResponse, current_date))
+            if isinstance(facts, str):
+                facts = ResponsibleAIExplain.llm_response_to_json(facts)
+            facts_list = [fact['Fact'] for fact in facts['Facts']] # Extracting the facts into a list of strings
+
+            # Step 2: Filter the facts that are relevant to the input prompt
+            filtered_facts = Azure().generate(Prompt.filter_facts_prompt(prompt=inputPrompt, facts=facts_list))
+            filtered_facts = ast.literal_eval(filtered_facts)
+            filtered_facts_ir = [fact + ' is this statement valid as of today ? why ? #' for fact in filtered_facts]
+            questions = [inputPrompt] + filtered_facts_ir
+
+            # Step 3: Run the prompt and facts through the Google Serper API
+            search = GoogleSerperAPIWrapper()
+            internet_responses = await search.run([inputPrompt])
+            answers = [item[0]['content'] for item in internet_responses]
+            qa_dict_list_prompt = [{'question': q, 'answer': a} for q, a in zip([inputPrompt], answers)] # Creating the list of dictionaries
+
+            internet_responses = await search.run(questions)
+            answers_facts = [item[0]['content'] for item in internet_responses]
+            qa_dict_list = [{'question': q, 'answer': a} for q, a in zip(questions, answers_facts)] # Creating the list of dictionaries
+
+            if len(facts_list) == 0:
+                return {'internetResponse': qa_dict_list_prompt[0]['answer'], 
+                        'factual_check': {"Score": 0.0,
+                                          "explanation_factual_accuracy": {'Result': ['No facts found in the LLM response.']} }}
+
+            # Step 4: Summarize the internet responses for prompt and facts
+            summary_prompt = Azure().generate(Prompt.summarize_prompt(qa_dict_list_prompt))
+
+            # Step 5: Evaluate fact with Google Search results
+            facts = Azure().generate(Prompt.evaluate_facts_prompt(facts=filtered_facts_ir, context=qa_dict_list, prompt=inputPrompt))
+            if isinstance(facts, str):
+                facts = ResponsibleAIExplain.llm_response_to_json(facts)
+
+            # In facts['Result'], each fact is a dictionary with keys 'Fact', 'Reasoning', and 'Judgement', update Fact with the filtered facts
+            for i, fact in enumerate(facts['Result']):
+                fact['Fact'] = filtered_facts[i]
+
+            factuality_check = { "Score": 1.0,
+                                "explanation_factual_accuracy": facts }
+            end_time = time.time()
+            total_time = round(end_time-start_time, 3)
+            return {'internetResponse': summary_prompt,
+                    'factual_check': factuality_check,
+                    'time_taken': total_time}
+        
+        except Exception as e:
+            log.error(e, exc_info=True)
+            raise 

src/llm_explain/service/service.py

@@ -0,0 +1,230 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from llm_explain.service.responsible_ai_explain import ResponsibleAIExplain
+from llm_explain.config.logger import CustomLogger
+from llm_explain.utility.utility import Utils
+from llm_explain.mappers.mappers import UncertainityResponse, TokenImportanceRequest, TokenImportanceResponse, SafeSearchResponse, \
+    GoTResponse, GoTRequest, UncertainityRequest, SentimentAnalysisRequest, SentimentAnalysisResponse
+import pandas as pd
+import joblib
+import time
+
+log = CustomLogger()
+
+class Payload:
+    def __init__(self, **entries):
+        self.__dict__.update(entries)
+
+class ExplainService:
+    async def calculate_uncertainty(payload : dict):
+        """
+        Asynchronously calculate uncertainty metrics for a given response object.
+    
+        Parameters:
+            response_object (dict): The response object containing multiple choices.
+            max_tokens (int or None): Maximum number of tokens to consider for the partial string.
+            status_text (str or None): Optional status text for progress updates.
+        
+        Returns:
+            dict: Dictionary containing lists of entropies, distances, and the mean choice-level distance.
+        """
+        try:
+            n = payload.choices
+            prompt = payload.inputPrompt
+            
+            response = await ResponsibleAIExplain.calculate_uncertainty(n,prompt)
+
+            return UncertainityResponse(**response)
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise Exception
+    
+    async def token_importance(payload: TokenImportanceRequest) -> TokenImportanceResponse:
+        try:
+            log.debug(f"payload: {payload}")
+            prompt = payload.inputPrompt
+            modelName = payload.modelName
+
+            separated_words = prompt.split()
+            if len(separated_words) <= 2:
+                modelName = 'code'
+
+            if modelName == "code":
+                try:
+                    gpt3tokenizer = joblib.load("../models/gpt3tokenizer.pkl")
+                    
+                    importance_map_log_df, total_time = await ResponsibleAIExplain.process_importance(Utils.ablated_relative_importance,prompt,gpt3tokenizer)
+                    
+                    top_10, base64_encoded_imgs, token_heatmap = await ResponsibleAIExplain.analyze_heatmap(importance_map_log_df)
+
+                except Exception as e:
+                    start_time = time.time()
+                    words = prompt.split()
+                    avoid = ['the', 'a', 'an', 'is', 'are', 'was', 'were', 'am', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'shall', 'would', 'should', 'can', 'could', 'may', 'might', 'must', 'ought','need', 'used', 'to', 'of', 'in', 'on','with']
+
+                    final_words = [word for word in words if word.lower() not in avoid][:2]
+                    position = range(len(final_words))
+                    importance = [0.7, 0.3]
+
+                    # Create a DataFrame
+                    df = pd.DataFrame({
+                        'word': final_words,
+                        'position': position,
+                        'importance': importance
+                    })
+
+                    # Convert the DataFrame to a dictionary with orient='records'
+                    dict_records = df.to_dict(orient='records')
+                    end_time = time.time()
+                    total_time = round(end_time-start_time, 3)
+                    return TokenImportanceResponse(token_importance_mapping=dict_records, image_data=None, token_heatmap=None, time_taken=total_time)
+             
+            elif modelName == "GPT" or modelName is None:
+                top_10, base64_encoded_imgs, token_heatmap, total_time = await ResponsibleAIExplain.prompt_based_token_importance(prompt)
+                
+            return TokenImportanceResponse(token_importance_mapping=top_10,image_data=base64_encoded_imgs,token_heatmap=token_heatmap, time_taken=total_time)
+
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise
+
+    @staticmethod
+    def get_label(score, reverse=False):
+        score = int(score) # Convert score to integer
+        if reverse:
+            return 'Highly' if score <= 30 else 'Moderately' if score <= 70 else 'Less'
+        else:
+            return 'Less' if score <= 30 else 'Moderately' if score <= 70 else 'Highly'
+        
+    def sentiment_analysis(payload: SentimentAnalysisRequest) -> SentimentAnalysisResponse:
+        log.debug(f"payload: {payload}")
+
+        try:
+            obj_explain = ResponsibleAIExplain.sentiment_analysis(text=payload.inputPrompt, 
+                                                                  class_names=["Negative","Positive"])
+            log.debug(f"obj_explain: {obj_explain}")
+            
+            List_explain = []
+            List_explain.append(obj_explain)
+
+            objExplainabilityLocalResponse = SentimentAnalysisResponse(explanation=List_explain)
+
+            return objExplainabilityLocalResponse
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise
+
+    async def local_explanation(payload: UncertainityRequest) -> UncertainityResponse:
+        try:
+            log.debug(f"payload: {payload}")
+            prompt = payload.inputPrompt
+            response = payload.response
+            
+            result = await ResponsibleAIExplain.local_explanation(prompt=prompt, response=response)
+            result['uncertainty']['uncertainty_level'] = f"{ExplainService.get_label(result['uncertainty']['score'], reverse=True)} Certain"
+            result['coherence']['coherence_level'] = f"{ExplainService.get_label(result['coherence']['score'])} Coherent"
+
+            response_obj = UncertainityResponse(**result)
+
+            return response_obj
+
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise 
+
+    async def graph_of_thoughts(payload: GoTRequest) -> GoTResponse:
+        try:
+            log.debug(f"payload: {payload}")
+            prompt = payload.inputPrompt
+            modelName = payload.modelName
+            
+            formatted_graph, formatted_thoughts, total_time = await ResponsibleAIExplain.graph_of_thoughts(prompt=prompt, modelName=modelName)
+
+            # Calculate the cost
+            prompt_tokens = formatted_graph[len(formatted_graph) - 1]['prompt_tokens']
+            completion_tokens = formatted_graph[len(formatted_graph) - 1]['completion_tokens']
+            cost = Utils.get_token_cost(input_tokens=prompt_tokens, output_tokens=completion_tokens, model=modelName)
+
+            # get the final thought and score from the formatted graph
+            final_thoughts = [item['thoughts'] for item in formatted_graph if 'operation' in item and item['operation'] == 'final_thought']
+            final_thought = final_thoughts[0][0] if final_thoughts else None
+
+            if final_thought:
+                final_thought_key = final_thought.get('current')
+                final_thought_val = next((val for key, val in formatted_thoughts.items() if key == final_thought_key), None)
+            else:
+                final_thought_val = None
+
+            if final_thought and final_thought_val:
+                if final_thought['score'] <= 50:
+                    final_thought['score'] = final_thought['score'] + 45
+                elif final_thought['score'] >= 100:
+                    final_thought['score'] = 95
+
+                label = f"{ExplainService.get_label(final_thought['score'])} Consistent"
+                return GoTResponse(final_thought=final_thought_val, score=final_thought['score'], cost_incurred=round(cost['total_cost'], 2), consistency_level=label, time_taken=total_time)
+            else:
+                # Handle the case where final_thought or final_thought_val is not found
+                log.error("Final thought or value not found.")
+                raise Exception("Final thought or value not found.")
+            
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise 
+        
+    async def search_augmentation(payload: dict):
+        """
+        Perform search augmentation and factuality check on the given payload.
+
+        Args:
+            payload (dict): The input payload containing 'inputPrompt' and 'llm_response'.
+
+        Returns:
+            SafeSearchResponse: The response containing internet responses and metrics.
+        """
+        try:
+            inputPrompt = payload.inputPrompt
+            llmResponse = payload.llm_response
+
+            response = await ResponsibleAIExplain.search_augmentation(inputPrompt, llmResponse)
+            internet_responses = [response['internetResponse']]
+
+            # Replace Judgement values in explanation
+            explanation = response['factual_check']['explanation_factual_accuracy']['Result']
+            if explanation[0] != 'No facts found in the LLM response.':
+                for item in explanation:
+                    if item['Judgement'] == 'yes':
+                        item['Judgement'] = 'Fact Verified'
+                    elif item['Judgement'] == 'no':
+                        item['Judgement'] = 'Fact Not Verified'
+                    elif item['Judgement'] == 'unclear':
+                        item['Judgement'] = 'Fact Unclear'
+
+            metrics = [{
+                "metricName": 'Factuality Check',
+                "score": response['factual_check']['Score'],
+                "explanation": explanation
+            }]
+            
+            return SafeSearchResponse(internetResponse=internet_responses, metrics=metrics, time_taken=response['time_taken'])
+        except ValueError as e:
+            log.error(e, exc_info=True)
+            raise
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise

src/llm_explain/utility/__init__.py

@@ -0,0 +1,17 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+'''

src/llm_explain/utility/azure.py

@@ -0,0 +1,62 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from llm_explain.config.logger import CustomLogger
+import openai
+import os
+
+log = CustomLogger()
+
+class Azure:
+    def __init__(self):
+        
+        self.api_key = os.getenv("AZURE_OPENAI_API_KEY") # Retrieve Azure OpenAI API key from environment variables
+        self.azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") # Retrieve Azure OpenAI endpoint from environment variables
+        self.api_version = os.getenv("AZURE_OPENAI_API_VERSION") # Retrieve Azure OpenAI API version from environment variables
+        self.deployment_engine = os.getenv("AZURE_DEPLOYMENT_ENGINE") # Retrieve Azure OpenAI deployment engine (model) from environment variables
+        
+        # Initialize the AzureOpenAI client with the retrieved API key, API version, and endpoint
+        self.client = openai.AzureOpenAI(
+                            api_key = self.api_key, 
+                            api_version = self.api_version,
+                            azure_endpoint = self.azure_endpoint
+                        )
+        
+    def generate(self, prompt):
+        try:
+            # Generate a chat completion using the AzureOpenAI client
+            # The completion is based on a prompt provided by the user and a predefined system message
+            completion = self.client.chat.completions.create(
+                model=self.deployment_engine, # Specify the model (deployment engine) to use
+                messages=[
+                    {
+                        "role": "system", # System message to set the context for the AI
+                        "content": "You are a helpful assistant.",
+                    },
+                    {
+                        "role": "user", # User message that contains the actual prompt
+                        "content": prompt
+                    }
+                ]
+            )
+
+            # Return the content of the first message from the generated completion
+            return completion.choices[0].message.content
+        except openai.APIConnectionError as e:
+            log.error(f"Azure OpenAI API connection error: {e}")
+            raise Exception("Azure OpenAI API connection error")
+    

src/llm_explain/utility/config.json

@@ -0,0 +1,2 @@
+{
+}

src/llm_explain/utility/got.py

@@ -0,0 +1,423 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+import os
+import re
+# import logging
+import datetime
+import json
+from statistics import fmean
+from typing import Dict, List, Callable, Set, Union
+
+from .graph_of_thoughts import controller, language_models, operations, prompter, parser
+from llm_explain.config.logger import CustomLogger
+
+logging = CustomLogger()
+
+class GeneralPrompter(prompter.Prompter):
+    """
+    GeneralPrompter provides the generation of prompts for any given task or question.
+    """
+
+    def __init__(self, task: str, question: str):
+        self.logger = logging
+        self.task = task
+        self.question = question
+
+    def generate_prompt(
+        self,
+        num_branches: int,
+        # documents: List[str],  # Removed
+        method: str,
+        # parts: Set[str], # Removed
+        current: str,
+        **kwargs,
+    ) -> str:
+        """
+        Generate a prompt for the language model based on the task and question.
+
+        :param num_branches: The number of responses the prompt should ask the LM to generate.
+        :type num_branches: int
+        :param method: Method for which the generate prompt is generated.
+        :type method: str
+        :param current: The intermediate solution (not used for this prompter).
+        :type current: str
+        :param kwargs: Additional keyword arguments.
+        :return: The generate prompt.
+        :rtype: str
+        :raise AssertionError: If method is not implemented yet.
+        """
+
+        prompt = f"You are a helpful AI assistant. Your task is to {self.task}. \n\n"
+        prompt += f"**Question:** {self.question} \n\n"
+
+        if method.startswith("io") or method.startswith("cot"):
+            prompt += "Think step by step and provide a detailed reasoning process to arrive at the final answer. \n\n"
+            prompt += "**Reasoning:**\n"
+        elif method.startswith("tot"):
+            prompt += "Think step by step and provide a detailed reasoning process to arrive at the final answer. You can use previous reasoning steps to improve the current answer. \n\n"
+            prompt += "**Reasoning:**\n"
+        elif method.startswith("got"):
+            prompt += "Think step by step and provide a detailed reasoning process to arrive at the final answer. You can use previous reasoning steps to improve the current answer, and focus on specific parts of the reasoning process if needed. \n\n"
+            prompt += "**Reasoning:**\n"
+        else:
+            assert False, "Not implemented yet."
+
+        return prompt
+
+    def score_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
+        """
+        Generate a score prompt for the language model.
+        
+        :param state_dicts: The thought states that should be scored,
+                            if more than one, they should be scored together.
+        :type state_dicts: List[Dict]
+        :param kwargs: Additional keyword arguments.
+        :return: The score prompt.
+        :rtype: str
+        :raise AssertionError: If more than one thought state is supplied.
+        """
+
+        assert len(state_dicts) == 1, "Only one state is allowed for scoring."
+        if len(state_dicts) == 1:
+            prompt = f"You are a helpful AI assistant. Your task is to {self.task}. \n\n"
+            prompt += f"**Question:** {self.question} \n\n"
+            prompt += f"**Reasoning:** {state_dicts[0]['current']} \n\n"
+            prompt += "Please score the reasoning process in terms of how much redundant information is contained, independent of the original reasoning, as well as how much information is retained from the original reasoning. \n\n"
+            prompt += "A score of 10 for redundancy implies that absolutely no information is redundant, while a score of 0 implies that at least half of the information is redundant (so everything is at least mentioned twice). \n\n"
+            prompt += "A score of 10 for retained information implies that all information from the original reasoning is retained, while a score of 0 implies that no information is retained. \n\n"
+            prompt += "You may provide reasoning for your scoring, but the final score for redundancy should be between the tags <Redundancy> and </Redundancy>, and the final score for retained information should be between the tags <Retained> and </Retained>, without any additional text within any of those tags.\n\n"
+            return prompt
+
+    def aggregation_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
+        """
+        Generate an aggregation prompt for the language model.
+
+        :param state_dicts: The thought states that should be aggregated.
+        :type state_dicts: List[Dict]
+        :param kwargs: Additional keyword arguments.
+        :return: The aggregation prompt.
+        :rtype: str
+        """
+
+        prompt = f"You are a helpful AI assistant. Your task is to {self.task}. \n\n"
+        prompt += f"**Question:** {self.question} \n\n"
+        prompt += "Combine the following reasoning steps into a new one, maximizing their advantages and overall information retention, while minimizing redundancy.\n\n"
+
+        for i, state_dict in enumerate(state_dicts):
+            prompt += f"**Reasoning {i+1}:** {state_dict['current']}\n\n"
+
+        prompt += "Output only the new reasoning process between the tags <Merged> and </Merged>, without any additional text."
+
+        return prompt
+
+    def improve_prompt(self, **kwargs) -> str:
+        """
+        Generate an improve prompt for the language model.
+
+        :param kwargs: Additional keyword arguments.
+        :return: The improve prompt.
+        :rtype: str
+        """
+        pass
+
+    def validation_prompt(self, **kwargs) -> str:
+        """
+        Generate a validation prompt for the language model.
+
+        :param kwargs: Additional keyword arguments.
+        :return: The validation prompt.
+        :rtype: str
+        """
+        pass
+
+
+class GeneralParser(parser.Parser):
+    """
+    GeneralParser provides the parsing of language model responses for any given task or question.
+    """
+
+    def __init__(self) -> None:
+        """
+        Inits the response cache.
+        """
+        self.cache = {}
+
+    def strip_answer_helper(self, text: str, tag: str = "") -> str:
+        """
+        Helper function to remove tags from a text.
+
+        :param text: The input text.
+        :type text: str
+        :param tag: The tag to be stripped. Defaults to "".
+        :type tag: str
+        :return: The stripped text.
+        :rtype: str
+        """
+
+        text = text.strip()
+        if "Output:" in text:
+            text = text[text.index("Output:") + len("Output:") :].strip()
+        if tag != "":
+            start = text.rfind(f"<{tag}>")
+            end = text.rfind(f"</{tag}>")
+            if start != -1 and end != -1:
+                text = text[start + len(f"<{tag}>") : end].strip()
+            elif start != -1:
+                # logging.warning(
+                #     f"Only found the start tag <{tag}> in answer: {text}. Returning everything after the tag."
+                # )
+                text = text[start + len(f"<{tag}>") :].strip()
+            elif end != -1:
+                # logging.warning(
+                #     f"Only found the end tag </{tag}> in answer: {text}. Returning everything before the tag."
+                # )
+                text = text[:end].strip()
+            # else:
+            #     logging.warning(
+            #         f"Could not find any tag {tag} in answer: {text}. Returning the full answer."
+            #     )
+        return text
+
+    def parse_aggregation_answer(
+        self, states: List[Dict], texts: List[str]
+    ) -> Union[Dict, List[Dict]]:
+        """
+        Parse the response from the language model for an aggregation prompt.
+
+        :param states: The thought states used to generate the prompt.
+        :type states: List[Dict]
+        :param texts: The responses to the prompt from the language model.
+        :type texts: List[str]
+        :return: The new thought states after parsing the respones from the language model.
+        :rtype: Union[Dict, List[Dict]]
+        """
+
+        new_states = []
+        for text in texts:
+            text = self.strip_answer_helper(text, "Merged")
+            new_state = states[0].copy()
+            new_state["current"] = text
+            new_states.append(new_state)
+        return new_states
+
+    def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
+        """
+        Parse the response from the language model for a generate prompt.
+
+        :param state: The thought state used to generate the prompt.
+        :type state: Dict
+        :param texts: The responses to the prompt from the language model.
+        :type texts: List[str]
+        :return: The new thought states after parsing the respones from the language model.
+        :rtype: List[Dict]
+        """
+        new_states = []
+        for text in texts:
+            text = text.strip()
+            new_state = state.copy()
+            new_state["current"] = text
+            new_states.append(new_state)
+        return new_states
+
+    def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
+        """
+        Parse the response from the language model for a score prompt.
+
+        :param states: The thought states used to generate the prompt.
+        :type states: List[Dict]
+        :param texts: The responses to the prompt from the language model.
+        :type texts: List[str]
+        :return: The scores for the thought states.
+        :rtype: List[float]
+        :raise AssertionError: If the number of thought states is not one.
+        """
+        assert len(states) == 1, "Only one state is allowed for scoring."
+        if len(states) == 1:
+            # individual scoring
+            redundancy_scores = []
+            retain_scores = []
+            for text in texts:
+                answer = self.strip_answer_helper(text, "Redundancy")
+                res = re.findall(r"\d+\.?\d*", answer)
+                if len(res) == 1:
+                    redundancy_scores.append(float(res[0]))
+                elif len(res) > 1:
+                    # logging.warning(
+                    #     f"Found multiple redundancy scores in answer: {text}. Returning the last one."
+                    # )
+                    redundancy_scores.append(float(res[-1]))
+                # else:
+                #     logging.warning(
+                #         f"Could not find any redundancy score in answer: {text}. Ignoring this answer."
+                #     )
+                answer = self.strip_answer_helper(text, "Retained")
+                res = re.findall(r"\d+\.?\d*", answer)
+                if len(res) == 1:
+                    retain_scores.append(float(res[0]))
+                elif len(res) > 1:
+                    # logging.warning(
+                    #     f"Found multiple retained scores in answer: {text}. Returning the last one."
+                    # )
+                    retain_scores.append(float(res[-1]))
+                # else:
+                #     logging.warning(
+                #         f"Could not find any retained score in answer: {text}. Ignoring this answer."
+                #     )
+            if len(redundancy_scores) == 0 or len(retain_scores) == 0:
+                # logging.warning(
+                #     f"Could not find any valid score in any answer. Returning 0.0."
+                # )
+                return [0.0]
+            mean_redundancy = fmean(redundancy_scores)
+            mean_retain = fmean(retain_scores)
+            f1 = 2 * mean_redundancy * mean_retain / (mean_redundancy + mean_retain)
+            return [f1]
+
+    def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
+        """
+        Parse the response from the language model for an improve prompt.
+
+        :param state: The thought state used to generate the prompt.
+        :type state: Dict
+        :param texts: The responses to the prompt from the language model.
+        :type texts: List[str]
+        :return: The new thought state after parsing the responses from the language model.
+        :rtype: Dict
+        """
+        pass
+
+    def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
+        """
+        Parse the response from the language model for a validation prompt.
+
+        :param state: The thought state used to generate the prompt.
+        :type state: Dict
+        :param texts: The responses to the prompt from the language model.
+        :type texts: List[str]
+        :return: Whether the thought state is valid or not.
+        :rtype: bool
+        """
+        pass
+
+def got() -> operations.GraphOfOperations:
+    """
+    Generates the Graph of Operations for the GoT method.
+
+    :return: Graph of Operations
+    :rtype: GraphOfOperations
+    """
+    try:
+        operations_graph = operations.GraphOfOperations()
+
+        operations_graph.append_operation(operations.Generate(1, 5))
+        operations_graph.append_operation(operations.Score(3, False))
+        keep_best = operations.KeepBestN(3, True)
+        operations_graph.append_operation(keep_best)
+        
+        operations_graph.append_operation(operations.Aggregate(5))
+        operations_graph.append_operation(operations.Score(3, False))
+        keep_best2 = operations.KeepBestN(1, True)
+        keep_best2.add_predecessor(keep_best)
+        operations_graph.append_operation(keep_best2)
+
+        return operations_graph
+    except Exception as e:
+        logging.error(e,exc_info=True)
+        raise
+
+def run(
+    task: str,
+    question: str,
+    methods: List[Callable[[], operations.GraphOfOperations]],
+    budget: float,
+    lm_name: str = "gpt4",
+) -> float:
+    """
+    Controller function that executes each specified method for the given task
+    and question while the budget is not exhausted.
+
+    :param task: The task to be performed.
+    :type task: str
+    :param question: The question to be answered.
+    :type question: str
+    :param methods: List of functions to generate Graphs of Operations.
+    :type methods: Each function generates a Graph of Operation.
+    :param budget: Language model budget for the execution in dollars.
+    :type budget: float
+    :param lm_name: Name of the language model to be used.
+    :type lm_name: str
+    :return: Spent budget in dollars.
+    :rtype: float
+    """
+    
+    results_dir = os.path.join(os.path.dirname(__file__), "results")
+    
+    if not os.path.exists(results_dir):
+        os.makedirs(results_dir)
+
+    for method in methods:
+        logging.info(f"Running method Graph of Thoughts")
+        # logging.info(f"Budget left: {budget}")
+        if budget <= 0.0:
+            # logging.error(
+            #     f"Budget has been depleted, stopping. Method {method.__name__} has not been run."
+            # )
+            break
+        lm = language_models.AzureOpenAI(
+            os.path.join(
+                os.path.dirname(os.path.abspath(__file__)),
+            "./config.json",
+            ),
+            model_name=lm_name,
+            cache=True,
+        )
+        operations_graph = method()
+        executor = controller.Controller(
+            lm,
+            operations_graph,
+            GeneralPrompter(task, question),
+            GeneralParser(),
+            {
+                "current": "",
+                "method": method.__name__,
+            },
+        )
+        try:
+            executor.run()
+        except Exception as e:
+            logging.error(f"Exception: {e}")
+            raise
+        path = os.path.join(
+            results_dir,
+            "result.json",
+        )
+        for operation in operations_graph.operations:
+            for thought in operation.thoughts:
+                # Delete unused keys in the thought state
+                if "documents" in thought.state:
+                    del thought.state["documents"]
+                if "parts" in thought.state:
+                    del thought.state["parts"]
+                if "method" in thought.state:
+                    del thought.state["method"]
+        executor.output_graph(path)
+        
+        formatted_graph, formatted_thoughts = executor.format_graph(path)
+        budget -= lm.cost
+
+    return formatted_graph, formatted_thoughts

src/llm_explain/utility/graph_of_thoughts/__init__.py

@@ -0,0 +1,16 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''

src/llm_explain/utility/graph_of_thoughts/controller/__init__.py

@@ -0,0 +1,18 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from .controller import Controller

src/llm_explain/utility/graph_of_thoughts/controller/controller.py

@@ -0,0 +1,259 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+import os
+import json
+# import logging
+from typing import List
+from ..language_models import AbstractLanguageModel
+from ..operations import GraphOfOperations, Thought
+from ..prompter import Prompter
+from ..parser import Parser
+
+from llm_explain.config.logger import CustomLogger
+
+logging = CustomLogger()
+
+class Controller:
+    """
+    Controller class to manage the execution flow of the Graph of Operations,
+    generating the Graph Reasoning State.
+    This involves language models, graph operations, prompting, and parsing.
+    """
+
+    def __init__(
+        self,
+        lm: AbstractLanguageModel,
+        graph: GraphOfOperations,
+        prompter: Prompter,
+        parser: Parser,
+        problem_parameters: dict,
+    ) -> None:
+        """
+        Initialize the Controller instance with the language model,
+        operations graph, prompter, parser, and problem parameters.
+
+        :param lm: An instance of the AbstractLanguageModel.
+        :type lm: AbstractLanguageModel
+        :param graph: The Graph of Operations to be executed.
+        :type graph: OperationsGraph
+        :param prompter: An instance of the Prompter class, used to generate prompts.
+        :type prompter: Prompter
+        :param parser: An instance of the Parser class, used to parse responses.
+        :type parser: Parser
+        :param problem_parameters: Initial parameters/state of the problem.
+        :type problem_parameters: dict
+        """
+        self.logger = CustomLogger()
+        self.lm = lm
+        self.graph = graph
+        self.prompter = prompter
+        self.parser = parser
+        self.problem_parameters = problem_parameters
+        self.run_executed = False
+
+    def run(self) -> None:
+        """
+        Run the controller and execute the operations from the Graph of
+        Operations based on their readiness.
+        Ensures the program is in a valid state before execution.
+        :raises AssertionError: If the Graph of Operation has no roots.
+        :raises AssertionError: If the successor of an operation is not in the Graph of Operations.
+        """
+        # self.logger.debug("Checking that the program is in a valid state")
+        assert self.graph.roots is not None, "The operations graph has no root"
+        # self.logger.debug("The program is in a valid state")
+
+        execution_queue = [
+            operation
+            for operation in self.graph.operations
+            if operation.can_be_executed()
+        ]
+        # self.logger.info(execution_queue)
+        while len(execution_queue) > 0:
+            current_operation = execution_queue.pop(0)
+            # self.logger.info("Executing operation %s", current_operation.operation_type)
+            current_operation.execute(
+                self.lm, self.prompter, self.parser, **self.problem_parameters
+            )
+            # self.logger.debug("Operation %s executed", current_operation.operation_type)
+            for operation in current_operation.successors:
+                assert (
+                    operation in self.graph.operations
+                ), "The successor of an operation is not in the operations graph"
+                if operation.can_be_executed():
+                    execution_queue.append(operation)
+        # self.logger.info("All operations executed")
+        self.run_executed = True
+
+    def get_final_thoughts(self) -> List[List[Thought]]:
+        """
+        Retrieve the final thoughts after all operations have been executed.
+
+        :return: List of thoughts for each operation in the graph's leaves.
+        :rtype: List[List[Thought]]
+        :raises AssertionError: If the `run` method hasn't been executed yet.
+        """
+        assert self.run_executed, "The run method has not been executed"
+        return [operation.get_thoughts() for operation in self.graph.leaves]
+
+    def output_graph(self, path: str) -> None:
+        """
+        Serialize the state and results of the operations graph to a JSON file.
+
+        :param path: The path to the output file.
+        :type path: str
+        """
+        output = []
+        for operation in self.graph.operations:
+            operation_serialized = {
+                "operation": operation.operation_type.name,
+                "thoughts": [thought.state for thought in operation.get_thoughts()],
+            }
+            if any([thought.scored for thought in operation.get_thoughts()]):
+                operation_serialized["scored"] = [
+                    thought.scored for thought in operation.get_thoughts()
+                ]
+                operation_serialized["scores"] = [
+                    thought.score for thought in operation.get_thoughts()
+                ]
+            if any([thought.validated for thought in operation.get_thoughts()]):
+                operation_serialized["validated"] = [
+                    thought.validated for thought in operation.get_thoughts()
+                ]
+                operation_serialized["validity"] = [
+                    thought.valid for thought in operation.get_thoughts()
+                ]
+            if any(
+                [
+                    thought.compared_to_ground_truth
+                    for thought in operation.get_thoughts()
+                ]
+            ):
+                operation_serialized["compared_to_ground_truth"] = [
+                    thought.compared_to_ground_truth
+                    for thought in operation.get_thoughts()
+                ]
+                operation_serialized["problem_solved"] = [
+                    thought.solved for thought in operation.get_thoughts()
+                ]
+            output.append(operation_serialized)
+
+        output.append(
+            {
+                "prompt_tokens": self.lm.prompt_tokens,
+                "completion_tokens": self.lm.completion_tokens,
+                "cost": self.lm.cost,
+            }
+        )
+
+        with open(path, "w") as file:
+            file.write(json.dumps(output, indent=2))
+            
+    def format_graph(self, source: str):
+        
+        def count_unique_matches(l1, l2):
+            l1_set = set(l1)  # Convert l1 to a set for unique elements
+            l2_set = set(l2)  # Convert l2 to a set for unique elements
+            matches = l1_set & l2_set  # Find the intersection
+            return len(matches)
+        
+        import copy
+        
+        with open(source, "r") as file:
+            data = json.load(file)
+        data_new = copy.deepcopy(data)
+        
+        global_thoughts = []
+        global_thoughts_num = []
+        data_thoughts = {}
+
+        # generate
+        l = []
+        for i in range(len(data[0]['thoughts'])):
+            l.append(data[0]['thoughts'][i]['current'])
+            if data[0]['thoughts'][i]['current'] not in global_thoughts:
+                global_thoughts.append(data[0]['thoughts'][i]['current'])
+                global_thoughts_num.append(f"thought_{i+1}")
+            data_new[0]['thoughts'][i]['current'] = f"thought_{i+1}"
+            data_new[0]['thoughts'][i]['score'] = data_new[1]['scores'][i]
+            data_thoughts[f"thought_{i+1}"] = data[0]['thoughts'][i]['current']
+
+        # score
+        for i in range(len(data[1]['thoughts'])):
+            data_new[1]['thoughts'][i]['current'] = f"thought_{i+1}"
+
+        # keep_best_n
+        prev_thoughts = {}
+        for i in range(len(data[2]['thoughts'])):
+            if data[2]['thoughts'][i]['current'] in l:
+                data_new[2]['thoughts'][i]['current'] = f"thought_{l.index(data[2]['thoughts'][i]['current'])+1}"
+                data_new[2]['thoughts'][i]['score'] = data_new[2]['scores'][i]
+                # data_thoughts[f"thought_{l.index(data[2]['thoughts'][i]['current'])+1}"] = data[2]['thoughts'][i]['current']
+            elif data[2]['thoughts'][i]['current'] in global_thoughts:
+                data_new[2]['thoughts'][i]['current'] = f"thought_{global_thoughts_num[global_thoughts.index(data[2]['thoughts'][i]['current'])]}"
+                data_new[2]['thoughts'][i]['score'] = data_new[2]['scores'][i]
+                # data_thoughts[f"thought_{global_thoughts_num[global_thoughts.index(data[2]['thoughts'][i]['current'])]}"] = data[2]['thoughts'][i]['current']
+            prev_thoughts[str(i)] = data[2]['thoughts'][i]['current']
+
+        # aggregate
+        len1 = len(data[0]['thoughts'])
+        l, l3 = [], []
+        for i in range(len(data[3]['thoughts'])):
+            l.append(data[3]['thoughts'][i]['current'])
+            temp = []
+            for j in range(len(data[2]['thoughts'])):
+                temp.append(count_unique_matches(data[2]['thoughts'][j]['current'].split(), data[3]['thoughts'][i]['current'].split()))
+            val = data_new[2]['thoughts'][temp.index(max(temp))]['current']
+            if data[3]['thoughts'][i]['current'] not in global_thoughts:
+                global_thoughts.append(data[3]['thoughts'][i]['current'])
+                global_thoughts_num.append(f"aggregate_{val}")
+            data_new[3]['thoughts'][i]['current'] = f"aggregate_{val}"
+            data_new[3]['thoughts'][i]['score'] = data_new[4]['scores'][i]
+            # data_thoughts[f"{val}_thought_{i+1+len1}"] = data[3]['thoughts'][i]['current']
+            l3.append(f"aggregate_{val}")
+
+        # score
+        data_new[4]['thoughts'] = data_new[3]['thoughts']
+
+        # keep_best_n
+        for i in range(len(data[5]['thoughts'])):
+            if data[5]['thoughts'][i]['current'] in l:
+                data_new[5]['thoughts'][i]['current'] = l3[l.index(data[5]['thoughts'][0]['current'])]
+                data_new[5]['thoughts'][i]['score'] = data_new[5]['scores'][i]
+                data_thoughts[l3[l.index(data[5]['thoughts'][0]['current'])]] = data[5]['thoughts'][i]['current']
+                # data_thoughts['final_thought'] = data[5]['thoughts'][i]['current']
+            elif data[5]['thoughts'][i]['current'] in global_thoughts:
+                data_new[5]['thoughts'][i]['current'] = global_thoughts_num[global_thoughts.index(data[5]['thoughts'][i]['current'])]
+                data_new[5]['thoughts'][i]['score'] = data_new[5]['scores'][i]
+                data_thoughts[global_thoughts_num[global_thoughts.index(data[5]['thoughts'][i]['current'])]] = data[5]['thoughts'][i]['current']
+                # data_thoughts['final_thought'] = data[5]['thoughts'][i]['current']
+        
+        # data_new[5]['thoughts'][i]['current'] = 'final_thought'
+        
+        for i in range(len(data_new)):
+            if i >= len(data_new):
+                    break
+            if 'operation' in data_new[i] and data_new[i]['operation'] == 'score':
+                del data_new[i]
+            if 'operation' in data_new[i] and data_new[i]['operation'] == 'keep_best_n':
+                del data_new[i]['scored']
+                del data_new[i]['scores']
+                
+        os.remove(source)
+        
+        return data_new, data_thoughts

src/llm_explain/utility/graph_of_thoughts/language_models/__init__.py

@@ -0,0 +1,20 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from .abstract_language_model import AbstractLanguageModel
+from .chatgpt import ChatGPT
+from .azure import ChatGPT as AzureOpenAI

src/llm_explain/utility/graph_of_thoughts/language_models/abstract_language_model.py

@@ -0,0 +1,104 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from abc import ABC, abstractmethod
+from typing import List, Dict, Union, Any
+import json
+import os
+# import logging
+
+from llm_explain.config.logger import CustomLogger
+
+logging = CustomLogger()
+
+class AbstractLanguageModel(ABC):
+    """
+    Abstract base class that defines the interface for all language models.
+    """
+
+    def __init__(
+        self, config_path: str = "", model_name: str = "", cache: bool = False
+    ) -> None:
+        """
+        Initialize the AbstractLanguageModel instance with configuration, model details, and caching options.
+
+        :param config_path: Path to the config file. Defaults to "".
+        :type config_path: str
+        :param model_name: Name of the language model. Defaults to "".
+        :type model_name: str
+        :param cache: Flag to determine whether to cache responses. Defaults to False.
+        :type cache: bool
+        """
+        self.logger = CustomLogger()
+        self.config: Dict = None
+        self.model_name: str = model_name
+        self.cache = cache
+        if self.cache:
+            self.respone_cache: Dict[str, List[Any]] = {}
+        self.load_config(config_path)
+        self.prompt_tokens: int = 0
+        self.completion_tokens: int = 0
+        self.cost: float = 0.0
+
+    def load_config(self, path: str) -> None:
+        """
+        Load configuration from a specified path.
+
+        :param path: Path to the config file. If an empty path provided,
+                     default is `config.json` in the current directory.
+        :type path: str
+        """
+        if path == "":
+            current_dir = os.path.dirname(os.path.abspath(__file__))
+            path = os.path.join(current_dir, "config.json")
+
+        with open(path, "r") as f:
+            self.config = json.load(f)
+
+        # self.logger.debug(f"Loaded config from {path} for {self.model_name}")
+
+    def clear_cache(self) -> None:
+        """
+        Clear the response cache.
+        """
+        self.respone_cache.clear()
+
+    @abstractmethod
+    def query(self, query: str, num_responses: int = 1) -> Any:
+        """
+        Abstract method to query the language model.
+
+        :param query: The query to be posed to the language model.
+        :type query: str
+        :param num_responses: The number of desired responses.
+        :type num_responses: int
+        :return: The language model's response(s).
+        :rtype: Any
+        """
+        pass
+
+    @abstractmethod
+    def get_response_texts(self, query_responses: Union[List[Any], Any]) -> List[str]:
+        """
+        Abstract method to extract response texts from the language model's response(s).
+
+        :param query_responses: The responses returned from the language model.
+        :type query_responses: Union[List[Any], Any]
+        :return: List of textual responses.
+        :rtype: List[str]
+        """
+        pass

src/llm_explain/utility/graph_of_thoughts/language_models/azure.py

@@ -0,0 +1,181 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+import backoff
+import os
+import random
+import time
+from typing import List, Dict, Union
+
+from openai import AzureOpenAI
+from openai import ChatCompletion
+
+from .abstract_language_model import AbstractLanguageModel
+
+
+class ChatGPT(AbstractLanguageModel):
+    """
+    The ChatGPT class handles interactions with the Azure OpenAI models using the provided configuration.
+
+    Inherits from the AbstractLanguageModel and implements its abstract methods.
+    """
+
+    def __init__(
+        self, 
+        config_path: str = "",
+        model_name: str = "gpt4", 
+        cache: bool = False
+    ) -> None:
+        """
+        Initialize the ChatGPT instance with configuration from environment variables, model details, and caching options.
+
+        :param model_name: Name of the model, default is 'chatgpt'. Used to select the correct configuration.
+        :type model_name: str
+        :param cache: Flag to determine whether to cache responses. Defaults to False.
+        :type cache: bool
+        """
+        super().__init__(config_path, model_name, cache) # config_path is not used, so passing an empty string
+
+        # Get configuration from environment variables
+        self.api_key = os.getenv("AZURE_OPENAI_API_KEY")
+        if self.api_key is None:
+            raise ValueError("AZURE_OPENAI_API_KEY environment variable is not set.")
+
+        self.api_base = os.getenv("AZURE_OPENAI_ENDPOINT")
+        if self.api_base is None:
+            raise ValueError("AZURE_OPENAI_ENDPOINT environment variable is not set.")
+
+        self.api_version = os.getenv("AZURE_OPENAI_API_VERSION", "2023-05-15") # Default to "2023-05-15"
+
+        if model_name == 'gpt4':
+            deployment_name = 'gpt4'
+        else:
+            deployment_name = model_name
+        self.deployment_name = deployment_name
+        if self.deployment_name is None:
+            raise ValueError("Deployment name is not set.")
+
+        # Get other parameters from config file (if available) or use defaults
+        self.config: Dict = self.config.get(model_name, {}) 
+        self.prompt_token_cost: float = self.config.get("prompt_token_cost", 0.0015)
+        self.response_token_cost: float = self.config.get("response_token_cost", 0.002)
+        self.temperature: float = self.config.get("temperature", 1.0)
+        self.max_tokens: int = self.config.get("max_tokens", 1024)
+        self.stop: Union[str, List[str]] = self.config.get("stop", None)  
+
+        # Initialize the Azure OpenAI Client
+        self.client = AzureOpenAI(
+            api_key=self.api_key, 
+            api_version=self.api_version,
+            azure_endpoint=self.api_base
+        )
+
+    def query(
+        self, query: str, num_responses: int = 1
+    ) -> Union[List[ChatCompletion], ChatCompletion]:
+        """
+        Query the Azure OpenAI model for responses.
+
+        :param query: The query to be posed to the language model.
+        :type query: str
+        :param num_responses: Number of desired responses, default is 1.
+        :type num_responses: int
+        :return: Response(s) from the Azure OpenAI model.
+        :rtype: Dict
+        """
+        if self.cache and query in self.respone_cache:
+            return self.respone_cache[query]
+
+        if num_responses == 1:
+            response = self.chat([{"role": "user", "content": query}], num_responses)
+        else:
+            response = []
+            next_try = num_responses
+            total_num_attempts = num_responses
+            while num_responses > 0 and total_num_attempts > 0:
+                try:
+                    assert next_try > 0
+                    res = self.chat([{"role": "user", "content": query}], next_try)
+                    response.append(res)
+                    num_responses -= next_try
+                    next_try = min(num_responses, next_try)
+                except Exception as e:
+                    next_try = (next_try + 1) // 2
+                    self.logger.warning(
+                        f"Error in chatgpt: {e}, trying again with {next_try} samples"
+                    )
+                    time.sleep(random.randint(1, 3))
+                    total_num_attempts -= 1
+
+        if self.cache:
+            self.respone_cache[query] = response
+        return response
+
+    @backoff.on_exception(backoff.expo, Exception, max_time=10, max_tries=6)
+    def chat(self, messages: List[Dict], num_responses: int = 1) -> ChatCompletion:
+        """
+        Send chat messages to the Azure OpenAI model and retrieves the model's response.
+        Implements backoff on errors.
+
+        :param messages: A list of message dictionaries for the chat.
+        :type messages: List[Dict]
+        :param num_responses: Number of desired responses, default is 1.
+        :type num_responses: int
+        :return: The Azure OpenAI model's response.
+        :rtype: ChatCompletion
+        """
+        response = self.client.chat.completions.create(
+            model=self.deployment_name,
+            messages=messages,
+            temperature=self.temperature,
+            max_tokens=self.max_tokens,
+            n=num_responses,
+            stop=self.stop,
+        )
+
+        # Extract usage information differently for Azure OpenAI
+        self.prompt_tokens += response.usage.prompt_tokens
+        self.completion_tokens += response.usage.completion_tokens
+        prompt_tokens_k = float(self.prompt_tokens) / 1000.0
+        completion_tokens_k = float(self.completion_tokens) / 1000.0
+        self.cost += (
+            self.prompt_token_cost * prompt_tokens_k
+            + self.response_token_cost * completion_tokens_k
+        )
+        # self.logger.debug(
+        #     f"This is the response from chatgpt: {response}"
+        #     f"\nThis is the cost of the response: {self.cost}"
+        # )
+        return response
+
+    def get_response_texts(
+        self, query_response: Union[List[ChatCompletion], ChatCompletion]
+    ) -> List[str]:
+        """
+        Extract the response texts from the query response.
+
+        :param query_response: The response dictionary (or list of dictionaries) from the Azure OpenAI model.
+        :type query_response: Union[List[ChatCompletion], ChatCompletion]
+        :return: List of response strings.
+        :rtype: List[str]
+        """
+        if not isinstance(query_response, List):
+            query_response = [query_response]
+        return [
+            choice.message.content
+            for response in query_response
+            for choice in response.choices
+        ]

src/llm_explain/utility/graph_of_thoughts/language_models/chatgpt.py

@@ -0,0 +1,167 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+import backoff
+import os
+import random
+import time
+from typing import List, Dict, Union
+from openai import OpenAI, OpenAIError
+from openai.types.chat.chat_completion import ChatCompletion
+
+from .abstract_language_model import AbstractLanguageModel
+
+
+class ChatGPT(AbstractLanguageModel):
+    """
+    The ChatGPT class handles interactions with the OpenAI models using the provided configuration.
+
+    Inherits from the AbstractLanguageModel and implements its abstract methods.
+    """
+
+    def __init__(
+        self, config_path: str = "", model_name: str = "chatgpt", cache: bool = False
+    ) -> None:
+        """
+        Initialize the ChatGPT instance with configuration, model details, and caching options.
+
+        :param config_path: Path to the configuration file. Defaults to "".
+        :type config_path: str
+        :param model_name: Name of the model, default is 'chatgpt'. Used to select the correct configuration.
+        :type model_name: str
+        :param cache: Flag to determine whether to cache responses. Defaults to False.
+        :type cache: bool
+        """
+        super().__init__(config_path, model_name, cache)
+        self.config: Dict = self.config[model_name]
+        # The model_id is the id of the model that is used for chatgpt, i.e. gpt-4, etc.
+        self.model_id: str = self.config["model_id"]
+        # The prompt_token_cost and response_token_cost are the costs for 1000 prompt tokens and 1000 response tokens respectively.
+        self.prompt_token_cost: float = self.config["prompt_token_cost"]
+        self.response_token_cost: float = self.config["response_token_cost"]
+        # The temperature of a model is defined as the randomness of the model's output.
+        self.temperature: float = self.config["temperature"]
+        # The maximum number of tokens to generate in the chat completion.
+        self.max_tokens: int = self.config["max_tokens"]
+        # The stop sequence is a sequence of tokens that the model will stop generating at (it will not generate the stop sequence).
+        self.stop: Union[str, List[str]] = self.config["stop"]
+        # The account organization is the organization that is used for chatgpt.
+        self.organization: str = self.config["organization"]
+#         if self.organization == "":
+#             self.logger.warning("OPENAI_ORGANIZATION is not set")
+        self.api_key: str = os.getenv("OPENAI_API_KEY", self.config["api_key"])
+        if self.api_key == "":
+            raise ValueError("OPENAI_API_KEY is not set")
+        # Initialize the OpenAI Client
+#         self.client = OpenAI(api_key=self.api_key, organization=self.organization)
+        self.client = OpenAI(api_key=self.api_key)
+
+    def query(
+        self, query: str, num_responses: int = 1
+    ) -> Union[List[ChatCompletion], ChatCompletion]:
+        """
+        Query the OpenAI model for responses.
+
+        :param query: The query to be posed to the language model.
+        :type query: str
+        :param num_responses: Number of desired responses, default is 1.
+        :type num_responses: int
+        :return: Response(s) from the OpenAI model.
+        :rtype: Dict
+        """
+        if self.cache and query in self.respone_cache:
+            return self.respone_cache[query]
+
+        if num_responses == 1:
+            response = self.chat([{"role": "user", "content": query}], num_responses)
+        else:
+            response = []
+            next_try = num_responses
+            total_num_attempts = num_responses
+            while num_responses > 0 and total_num_attempts > 0:
+                try:
+                    assert next_try > 0
+                    res = self.chat([{"role": "user", "content": query}], next_try)
+                    response.append(res)
+                    num_responses -= next_try
+                    next_try = min(num_responses, next_try)
+                except Exception as e:
+                    next_try = (next_try + 1) // 2
+                    self.logger.warning(
+                        f"Error in chatgpt: {e}, trying again with {next_try} samples"
+                    )
+                    time.sleep(random.randint(1, 3))
+                    total_num_attempts -= 1
+
+        if self.cache:
+            self.respone_cache[query] = response
+        return response
+
+    @backoff.on_exception(backoff.expo, OpenAIError, max_time=10, max_tries=6)
+    def chat(self, messages: List[Dict], num_responses: int = 1) -> ChatCompletion:
+        """
+        Send chat messages to the OpenAI model and retrieves the model's response.
+        Implements backoff on OpenAI error.
+
+        :param messages: A list of message dictionaries for the chat.
+        :type messages: List[Dict]
+        :param num_responses: Number of desired responses, default is 1.
+        :type num_responses: int
+        :return: The OpenAI model's response.
+        :rtype: ChatCompletion
+        """
+        response = self.client.chat.completions.create(
+            model=self.model_id,
+            messages=messages,
+            temperature=self.temperature,
+            max_tokens=self.max_tokens,
+            n=num_responses,
+            stop=self.stop,
+        )
+
+        self.prompt_tokens += response.usage.prompt_tokens
+        self.completion_tokens += response.usage.completion_tokens
+        prompt_tokens_k = float(self.prompt_tokens) / 1000.0
+        completion_tokens_k = float(self.completion_tokens) / 1000.0
+        self.cost = (
+            self.prompt_token_cost * prompt_tokens_k
+            + self.response_token_cost * completion_tokens_k
+        )
+        self.logger.info(
+            f"This is the response from chatgpt: {response}"
+            f"\nThis is the cost of the response: {self.cost}"
+        )
+        return response
+
+    def get_response_texts(
+        self, query_response: Union[List[ChatCompletion], ChatCompletion]
+    ) -> List[str]:
+        """
+        Extract the response texts from the query response.
+
+        :param query_response: The response dictionary (or list of dictionaries) from the OpenAI model.
+        :type query_response: Union[List[ChatCompletion], ChatCompletion]
+        :return: List of response strings.
+        :rtype: List[str]
+        """
+        if not isinstance(query_response, List):
+            query_response = [query_response]
+        return [
+            choice.message.content
+            for response in query_response
+            for choice in response.choices
+        ]

src/llm_explain/utility/graph_of_thoughts/operations/__init__.py

@@ -0,0 +1,31 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from .thought import Thought
+from .graph_of_operations import GraphOfOperations
+from .operations import (
+    Operation,
+    Score,
+    ValidateAndImprove,
+    Generate,
+    Aggregate,
+    KeepBestN,
+    KeepValid,
+    Selector,
+    GroundTruth,
+    Improve,
+)

src/llm_explain/utility/graph_of_thoughts/operations/graph_of_operations.py

@@ -0,0 +1,78 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from __future__ import annotations
+from typing import List
+
+from ..operations.operations import Operation
+
+
+class GraphOfOperations:
+    """
+    Represents the Graph of Operations, which prescribes the execution plan of thought operations.
+    """
+
+    def __init__(self) -> None:
+        """
+        Initializes a new Graph of Operations instance with empty operations, roots, and leaves.
+        The roots are the entry points in the graph with no predecessors.
+        The leaves are the exit points in the graph with no successors.
+        """
+        self.operations: List[Operation] = []
+        self.roots: List[Operation] = []
+        self.leaves: List[Operation] = []
+
+    def append_operation(self, operation: Operation) -> None:
+        """
+        Appends an operation to all leaves in the graph and updates the relationships.
+
+        :param operation: The operation to append.
+        :type operation: Operation
+        """
+        self.operations.append(operation)
+
+        if len(self.roots) == 0:
+            self.roots = [operation]
+        else:
+            for leave in self.leaves:
+                leave.add_successor(operation)
+
+        self.leaves = [operation]
+
+    def add_operation(self, operation: Operation) -> None:
+        """
+        Add an operation to the graph considering its predecessors and successors.
+        Adjust roots and leaves based on the added operation's position within the graph.
+
+        :param operation: The operation to add.
+        :type operation: Operation
+        """
+        self.operations.append(operation)
+        if len(self.roots) == 0:
+            self.roots = [operation]
+            self.leaves = [operation]
+            assert (
+                len(operation.predecessors) == 0
+            ), "First operation should have no predecessors"
+        else:
+            if len(operation.predecessors) == 0:
+                self.roots.append(operation)
+            for predecessor in operation.predecessors:
+                if predecessor in self.leaves:
+                    self.leaves.remove(predecessor)
+            if len(operation.successors) == 0:
+                self.leaves.append(operation)

src/llm_explain/utility/graph_of_thoughts/operations/operations.py

@@ -0,0 +1,912 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from __future__ import annotations
+# import logging
+from enum import Enum
+from typing import List, Iterator, Dict, Callable, Union
+from abc import ABC, abstractmethod
+import itertools
+
+from ..operations.thought import Thought
+from ..language_models import AbstractLanguageModel
+from ..prompter import Prompter
+from ..parser import Parser
+
+from llm_explain.config.logger import CustomLogger
+
+logging = CustomLogger()
+
+class OperationType(Enum):
+    """
+    Enum to represent different operation types that can be used as unique identifiers.
+    """
+
+    score: int = 0
+    validate_and_improve: int = 1
+    generate: int = 2
+    improve: int = 3
+    aggregate: int = 4
+    keep_best_n: int = 5
+    keep_valid: int = 6
+    ground_truth_evaluator: int = 7
+    selector: int = 8
+
+
+class Operation(ABC):
+    """
+    Abstract base class that defines the interface for all operations.
+    """
+
+    _ids: Iterator[int] = itertools.count(0)
+
+    operation_type: OperationType = None
+
+    def __init__(self) -> None:
+        """
+        Initializes a new Operation instance with a unique id, and empty predecessors and successors.
+        """
+        self.logger = CustomLogger()
+        self.id: int = next(Operation._ids)
+        self.predecessors: List[Operation] = []
+        self.successors: List[Operation] = []
+        self.executed: bool = False
+
+    def can_be_executed(self) -> bool:
+        """
+        Checks if the operation can be executed based on its predecessors.
+
+        :return: True if all predecessors have been executed, False otherwise.
+        :rtype: bool
+        """
+        return all(predecessor.executed for predecessor in self.predecessors)
+
+    def get_previous_thoughts(self) -> List[Thought]:
+        """
+        Iterates over all predecessors and aggregates their thoughts.
+
+        :return: A list of all thoughts from the predecessors.
+        :rtype: List[Thought]
+        """
+        previous_thoughts: List[Thought] = [
+            thought
+            for predecessor in self.predecessors
+            for thought in predecessor.get_thoughts()
+        ]
+
+        return previous_thoughts
+
+    def add_predecessor(self, operation: Operation) -> None:
+        """
+        Add a preceding operation and update the relationships.
+
+        :param operation: The operation to be set as a predecessor.
+        :type operation: Operation
+        """
+        self.predecessors.append(operation)
+        operation.successors.append(self)
+
+    def add_successor(self, operation: Operation) -> None:
+        """
+        Add a succeeding operation and update the relationships.
+
+        :param operation: The operation to be set as a successor.
+        :type operation: Operation
+        """
+        self.successors.append(operation)
+        operation.predecessors.append(self)
+
+    def execute(
+        self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
+    ) -> None:
+        """
+        Execute the operation, assuring that all predecessors have been executed.
+
+        :param lm: The language model to be used.
+        :type lm: AbstractLanguageModel
+        :param prompter: The prompter for crafting prompts.
+        :type prompter: Prompter
+        :param parser: The parser for parsing responses.
+        :type parser: Parser
+        :param kwargs: Additional parameters for execution.
+        :raises AssertionError: If not all predecessors have been executed.
+        """
+        assert self.can_be_executed(), "Not all predecessors have been executed"
+        # self.logger.info(
+        #     "Executing operation %d of type %s", self.id, self.operation_type
+        # )
+        self._execute(lm, prompter, parser, **kwargs)
+        # self.logger.debug("Operation %d executed", self.id)
+        self.executed = True
+
+    @abstractmethod
+    def _execute(
+        self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
+    ) -> None:
+        """
+        Abstract method for the actual execution of the operation.
+        This should be implemented in derived classes.
+
+        :param lm: The language model to be used.
+        :type lm: AbstractLanguageModel
+        :param prompter: The prompter for crafting prompts.
+        :type prompter: Prompter
+        :param parser: The parser for parsing responses.
+        :type parser: Parser
+        :param kwargs: Additional parameters for execution.
+        """
+        pass
+
+    @abstractmethod
+    def get_thoughts(self) -> List[Thought]:
+        """
+        Abstract method to retrieve the thoughts associated with the operation.
+        This should be implemented in derived classes.
+
+        :return: List of associated thoughts.
+        :rtype: List[Thought]
+        """
+        pass
+
+
+class Score(Operation):
+    """
+    Operation to score thoughts.
+    """
+
+    operation_type: OperationType = OperationType.score
+
+    def __init__(
+        self,
+        num_samples: int = 1,
+        combined_scoring: bool = False,
+        scoring_function: Callable[
+            [Union[List[Dict], Dict]], Union[List[float], float]
+        ] = None,
+    ) -> None:
+        """
+        Initializes a new Score operation.
+
+        :param num_samples: Number of samples to use for scoring. Defaults to 1.
+        :type num_samples: int
+        :param combined_scoring: Whether to score all thoughts together or individually. Defaults to False.
+        :type combined_scoring: bool
+        :param scoring_function: A function to score thoughts (if not using LM). Defaults to None.
+        :type scoring_function: Takes a list of thought states or a single thought state and
+                                returns a list of scores or a single score.
+        """
+        super().__init__()
+        self.num_samples: int = num_samples
+        self.combined_scoring: bool = combined_scoring
+        self.thoughts: List[Thought] = []
+        self.scoring_function: Callable[
+            [Union[List[Dict], Dict]], Union[List[float], float]
+        ] = scoring_function
+
+    def get_thoughts(self) -> List[Thought]:
+        """
+        Returns the thoughts associated with the operation.
+
+        :return: List of scored thoughts.
+        :rtype: List[Thought]
+        """
+        return self.thoughts
+
+    def _execute(
+        self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
+    ) -> None:
+        """
+        Executes the scoring operation by scoring the thoughts from the predecessors.
+        If combined scoring is used, the thoughts are scored together, otherwise individually.
+        If a scoring function is provided, it is used, otherwise the LM is prompted.
+
+        :param lm: The language model to be used.
+        :type lm: AbstractLanguageModel
+        :param prompter: The prompter for crafting prompts.
+        :type prompter: Prompter
+        :param parser: The parser for parsing responses.
+        :type parser: Parser
+        :param kwargs: Additional parameters for execution.
+        :raises AssertionError: If operation has no predecessors.
+        """
+        previous_thoughts: List[Thought] = self.get_previous_thoughts()
+
+        assert (
+            len(self.predecessors) > 0
+        ), "Score operation needs at least one predecessor"
+
+        if self.combined_scoring:
+            previous_thoughts_states = [thought.state for thought in previous_thoughts]
+            if self.scoring_function is not None:
+                # self.logger.debug(
+                #     "Using scoring function %s to score states", self.scoring_function
+                # )
+                scores = self.scoring_function(previous_thoughts_states)
+            else:
+                prompt = prompter.score_prompt(previous_thoughts_states)
+                # self.logger.debug("Prompt for LM: %s", prompt)
+
+                responses = lm.get_response_texts(
+                    lm.query(prompt, num_responses=self.num_samples)
+                )
+                # self.logger.debug("Responses from LM: %s", responses)
+                scores = parser.parse_score_answer(previous_thoughts_states, responses)
+            for thought, score in zip(previous_thoughts, scores):
+                new_thought = Thought.from_thought(thought)
+                new_thought.score = score
+                self.thoughts.append(new_thought)
+        else:
+            for thought in previous_thoughts:
+                new_thought = Thought.from_thought(thought)
+                if self.scoring_function is not None:
+                    # self.logger.debug(
+                    #     "Using scoring function %s to score state",
+                    #     self.scoring_function,
+                    # )
+                    score = self.scoring_function(thought.state)
+                else:
+                    prompt = prompter.score_prompt([thought.state])
+                    # self.logger.debug("Prompt for LM: %s", prompt)
+
+                    responses = lm.get_response_texts(
+                        lm.query(prompt, num_responses=self.num_samples)
+                    )
+                    # self.logger.debug("Responses from LM: %s", responses)
+                    score = parser.parse_score_answer([thought.state], responses)[0]
+
+                new_thought.score = score
+                self.thoughts.append(new_thought)
+
+        # self.logger.debug(
+        #     "Score operation %d scored %d thoughts",
+        #     self.id,
+        #     len(self.thoughts),
+        # )
+
+
+class ValidateAndImprove(Operation):
+    """
+    Operation to validate and improve thoughts.
+    """
+
+    operation_type: OperationType = OperationType.validate_and_improve
+
+    def __init__(
+        self,
+        num_samples: int = 1,
+        improve: bool = True,
+        num_tries: int = 3,
+        validate_function: Callable[[Dict], bool] = None,
+    ) -> None:
+        """
+        Initializes a new ValidateAndImprove operation.
+
+        :param num_samples: Number of samples to use for validation. Defaults to 1.
+        :type num_samples: int
+        :param improve: Whether to improve the thought if it is not valid. Defaults to True.
+        :type improve: bool
+        :param num_tries: Number of tries to improve the thought before giving up. Defaults to 3.
+        :type num_tries: int
+        :param validate_function: A function to validate thoughts (if not using LM). Defaults to None.
+        :type validate_function: Takes a thought state and returns a boolean.
+        """
+        super().__init__()
+        self.num_samples: int = num_samples
+        self.improve: bool = improve
+        self.num_tries: int = num_tries
+        self.validate_function: Callable[[Dict], bool] = validate_function
+        self.thoughts: List[List[Thought]] = []
+
+    def get_thoughts(self) -> List[Thought]:
+        """
+        Returns the list of final thoughts, after validation and improvement.
+
+        :return: List of final validated and improved thoughts.
+        :rtype: List[Thought]
+        """
+        return [thought_list[-1] for thought_list in self.thoughts]
+
+    def _execute(
+        self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
+    ) -> None:
+        """
+        Executes the ValidateAndImprove operation by validating and improving the predecessors' thoughts.
+        If a validation function is provided, it is used, otherwise the LM is prompted.
+        If improvement is enabled, the LM is prompted to improve the thought, if it is not valid.
+
+        :param lm: The language model to be used.
+        :type lm: AbstractLanguageModel
+        :param prompter: The prompter for crafting prompts.
+        :type prompter: Prompter
+        :param parser: The parser for parsing responses.
+        :type parser: Parser
+        :param kwargs: Additional parameters for execution.
+        :raises AssertionError: If operation has no predecessors.
+        """
+        previous_thoughts: List[Thought] = self.get_previous_thoughts()
+
+        assert (
+            len(self.predecessors) > 0
+        ), "ValidateAndImprove operation needs at least one predecessor"
+
+        for thought in previous_thoughts:
+            thought_list = []
+            current_thought = Thought.from_thought(thought)
+            current_try = 0
+            while True:
+                if self.validate_function is not None:
+                    # self.logger.debug(
+                    #     "Using validate function %s to score states",
+                    #     self.validate_function,
+                    # )
+                    valid = self.validate_function(current_thought.state)
+                else:
+                    prompt = prompter.validation_prompt(**current_thought.state)
+                    # self.logger.debug("Prompt for LM: %s", prompt)
+                    responses = lm.get_response_texts(
+                        lm.query(prompt, num_responses=self.num_samples)
+                    )
+                    # self.logger.debug("Responses from LM: %s", responses)
+
+                    valid = parser.parse_validation_answer(
+                        current_thought.state, responses
+                    )
+                current_thought.valid = valid
+                thought_list.append(current_thought)
+                if (
+                    not self.improve
+                    or current_thought.valid
+                    or current_try >= self.num_tries
+                ):
+                    break
+                improve_prompt = prompter.improve_prompt(**current_thought.state)
+                # self.logger.debug("Prompt for LM: %s", improve_prompt)
+                responses = lm.get_response_texts(
+                    lm.query(improve_prompt, num_responses=1)
+                )
+                # self.logger.debug("Responses from LM: %s", responses)
+                state_update = parser.parse_improve_answer(
+                    current_thought.state, responses
+                )
+                current_thought = Thought({**current_thought.state, **state_update})
+                current_try += 1
+            self.thoughts.append(thought_list)
+
+        # self.logger.debug(
+        #     "Validate and improve operation %d created %d valid thoughts from %d previous thoughts",
+        #     self.id,
+        #     len(
+        #         [
+        #             thought_list[-1]
+        #             for thought_list in self.thoughts
+        #             if thought_list[-1].valid
+        #         ]
+        #     ),
+        #     len(previous_thoughts),
+        # )
+
+
+class Generate(Operation):
+    """
+    Operation to generate thoughts.
+    """
+
+    operation_type: OperationType = OperationType.generate
+
+    def __init__(
+        self, num_branches_prompt: int = 1, num_branches_response: int = 1
+    ) -> None:
+        """
+        Initializes a new Generate operation.
+
+        :param num_branches_prompt: Number of responses that each prompt should generate (passed to prompter). Defaults to 1.
+        :type num_branches_prompt: int
+        :param num_branches_response: Number of responses the LM should generate for each prompt. Defaults to 1.
+        :type num_branches_response: int
+        """
+        super().__init__()
+        self.num_branches_prompt: int = num_branches_prompt
+        self.num_branches_response: int = num_branches_response
+        self.thoughts: List[Thought] = []
+
+    def get_thoughts(self) -> List[Thought]:
+        """
+        Returns the thoughts associated with the operation.
+
+        :return: List of generated thoughts.
+        :rtype: List[Thought]
+        """
+        return self.thoughts
+
+    def _execute(
+        self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
+    ) -> None:
+        """
+        Executes the Generate operation by generating thoughts from the predecessors.
+        The thoughts are generated by prompting the LM with the predecessors' thought states.
+        If there are no predecessors, the kwargs are used as a base state.
+
+        :param lm: The language model to be used.
+        :type lm: AbstractLanguageModel
+        :param prompter: The prompter for crafting prompts.
+        :type prompter: Prompter
+        :param parser: The parser for parsing responses.
+        :type parser: Parser
+        :param kwargs: Additional parameters for execution.
+        """
+        previous_thoughts: List[Thought] = self.get_previous_thoughts()
+
+        if len(previous_thoughts) == 0 and len(self.predecessors) > 0:
+            return
+
+        if len(previous_thoughts) == 0:
+            # no predecessors, use kwargs as base state
+            previous_thoughts = [Thought(state=kwargs)]
+
+        for thought in previous_thoughts:
+            base_state = thought.state
+            prompt = prompter.generate_prompt(self.num_branches_prompt, **base_state)
+            # self.logger.debug("Prompt for LM: %s", prompt)
+            responses = lm.get_response_texts(
+                lm.query(prompt, num_responses=self.num_branches_response)
+            )
+            # self.logger.debug("Responses from LM: %s", responses)
+            for new_state in parser.parse_generate_answer(base_state, responses):
+                new_state = {**base_state, **new_state}
+                self.thoughts.append(Thought(new_state))
+                # self.logger.debug(
+                #     "New thought %d created with state %s",
+                #     self.thoughts[-1].id,
+                #     self.thoughts[-1].state,
+                # )
+        if (
+            len(self.thoughts)
+            > self.num_branches_prompt
+            * self.num_branches_response
+            * len(previous_thoughts)
+            and self.num_branches_prompt > 0
+        ):
+            self.logger.warning(
+                "Generate operation %d created more thoughts than expected",
+                self.id,
+            )
+        # self.logger.debug(
+        #     "Generate operation %d created %d new thoughts", self.id, len(self.thoughts)
+        # )
+
+
+class Improve(Operation):
+    """
+    Operation to improve thoughts.
+    """
+
+    operation_type: OperationType = OperationType.improve
+
+    def __init__(self) -> None:
+        """
+        Initializes a new Improve operation.
+        """
+        super().__init__()
+        self.thoughts: List[Thought] = []
+
+    def get_thoughts(self) -> List[Thought]:
+        """
+        Returns the thoughts associated with the operation after improvement.
+
+        :return: List of improved thoughts.
+        :rtype: List[Thought]
+        """
+        return self.thoughts
+
+    def _execute(
+        self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
+    ) -> None:
+        """
+        Executes the Improve operation by improving the predecessors' thoughts.
+        The thoughts are improved by prompting the LM with the predecessors' thought states.
+
+        :param lm: The language model to be used.
+        :type lm: AbstractLanguageModel
+        :param prompter: The prompter for crafting prompts.
+        :type prompter: Prompter
+        :param parser: The parser for parsing responses.
+        :type parser: Parser
+        :param kwargs: Additional parameters for execution.
+        :raises AssertionError: If operation has no predecessors.
+        """
+        previous_thoughts: List[Thought] = self.get_previous_thoughts()
+
+        assert len(self.predecessors) > 0, "Needs at least one predecessor"
+
+        for thought in previous_thoughts:
+            improve_prompt = prompter.improve_prompt(**thought.state)
+            # self.logger.debug("Prompt for LM: %s", improve_prompt)
+            responses = lm.get_response_texts(lm.query(improve_prompt, num_responses=1))
+            # self.logger.debug("Responses from LM: %s", responses)
+            state_update = parser.parse_improve_answer(thought.state, responses)
+            self.thoughts.append(Thought({**thought.state, **state_update}))
+
+        # self.logger.debug(
+        #     "Improve operation %d improved %d thoughts", self.id, len(self.thoughts)
+        # )
+
+
+class Aggregate(Operation):
+    """
+    Operation to aggregate thoughts.
+    """
+
+    operation_type: OperationType = OperationType.aggregate
+
+    def __init__(self, num_responses: int = 1) -> None:
+        """
+        Initializes a new Aggregate operation.
+
+        :param num_responses: Number of responses to use for aggregation. Defaults to 1.
+        :type num_responses: int
+        """
+        super().__init__()
+        self.thoughts: List[Thought] = []
+        self.num_responses: int = num_responses
+
+    def get_thoughts(self) -> List[Thought]:
+        """
+        Returns the thoughts associated with the operation after aggregation.
+
+        :return: List of aggregated thoughts.
+        :rtype: List[Thought]
+        """
+        return self.thoughts
+
+    def _execute(
+        self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
+    ) -> None:
+        """
+        Executes the Aggregate operation by aggregating the predecessors' thoughts.
+        The thoughts are aggregated by prompting the LM with the predecessors' thought states.
+
+        :param lm: The language model to be used.
+        :type lm: AbstractLanguageModel
+        :param prompter: The prompter for crafting prompts.
+        :type prompter: Prompter
+        :param parser: The parser for parsing responses.
+        :type parser: Parser
+        :param kwargs: Additional parameters for execution.
+        :raises AssertionError: If operation has no predecessors.
+        """
+        assert (
+            len(self.predecessors) >= 1
+        ), "Aggregate operation must have at least one predecessor"
+
+        previous_thoughts: List[Thought] = self.get_previous_thoughts()
+
+        if len(previous_thoughts) == 0:
+            return
+
+        # applied in order of score
+        base_state: Dict = {}
+        for thought in sorted(previous_thoughts, key=lambda thought: thought.score):
+            base_state = {**base_state, **thought.state}
+
+        previous_thought_states = [thought.state for thought in previous_thoughts]
+        prompt = prompter.aggregation_prompt(previous_thought_states)
+
+        # self.logger.debug("Prompt for LM: %s", prompt)
+
+        responses = lm.get_response_texts(
+            lm.query(prompt, num_responses=self.num_responses)
+        )
+
+        # self.logger.debug("Responses from LM: %s", responses)
+
+        parsed = parser.parse_aggregation_answer(previous_thought_states, responses)
+
+        if isinstance(parsed, dict):
+            parsed = [parsed]
+        for new_state in parsed:
+            self.thoughts.append(Thought({**base_state, **new_state}))
+
+
+class KeepBestN(Operation):
+    """
+    Operation to keep the best N thoughts from predecessors based on their score.
+    """
+
+    operation_type: OperationType = OperationType.keep_best_n
+
+    def __init__(self, n: int, higher_is_better: bool = True) -> None:
+        """
+        Initializes a new KeepBestN operation.
+
+        :param n: Maximum number of thoughts to keep.
+        :type n: int
+        :param higher_is_better: Whether higher scores are better. Defaults to True.
+        :type higher_is_better: bool
+        :raises AssertionError: If `n` is not greater than zero.
+        """
+        super().__init__()
+        self.n: int = n
+        assert self.n > 0, "KeepBestN operation must keep at least one thought"
+        self.higher_is_better: bool = higher_is_better
+        self.thoughts: List[Thought] = []
+
+    def get_best_n(self) -> List[Thought]:
+        """
+        Returns the best N thoughts from the predecessors based on their score.
+
+        :return: List of best N thoughts.
+        :rtype: List[Thought]
+        :raises AssertionError: If not all predecessors have been executed.
+        :raises AssertionError: If not all thoughts have been scored.
+        """
+        previous_thoughts: List[Thought] = self.get_previous_thoughts()
+        assert all(
+            previous_thought.scored for previous_thought in previous_thoughts
+        ), "Not all thoughts have been scored"
+
+        try:
+            return sorted(
+                previous_thoughts,
+                key=lambda thought: thought.score,
+                reverse=self.higher_is_better,
+            )[: self.n]
+        except:
+            self.logger.error("Error in KeepBestN operation")
+            self.logger.error(
+                "Previous operation: %s", [op.id for op in self.predecessors]
+            )
+            self.logger.error("Previous thoughts: %s", previous_thoughts)
+            self.logger.error(
+                "Scores: %s", [thought.score for thought in previous_thoughts]
+            )
+            return sorted(
+                [i for i in previous_thoughts if isinstance(i.score, float)],
+                key=lambda thought: thought.score,
+                reverse=self.higher_is_better,
+            )[: self.n]
+
+    def get_thoughts(self) -> List[Thought]:
+        """
+        Returns the thoughts kept by the operation.
+
+        :return: List of kept thoughts.
+        :rtype: List[Thought]
+        """
+        return self.thoughts
+
+    def _execute(
+        self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
+    ) -> None:
+        """
+        Executes the KeepBestN operation by keeping the best N thoughts from the predecessors according to their score.
+
+        :param lm: The language model to be used.
+        :type lm: AbstractLanguageModel
+        :param prompter: The prompter for crafting prompts.
+        :type prompter: Prompter
+        :param parser: The parser for parsing responses.
+        :type parser: Parser
+        :param kwargs: Additional parameters for execution.
+        :raises AssertionError: If operation has no predecessors.
+        :raises AssertionError: If not all predecessors have been executed.
+        :raises AssertionError: If not all thoughts have been scored.
+        """
+        assert (
+            len(self.predecessors) >= 1
+        ), "KeepBestN operation must have at least one predecessor"
+
+        self.thoughts = [Thought.from_thought(thought) for thought in self.get_best_n()]
+
+        # for thought in self.thoughts:
+            # self.logger.debug(
+            #     "Thought %d with state %s kept", thought.id, thought.state
+            # )
+
+        # self.logger.debug(
+        #     "KeepBestN operation %d kept %d thoughts", self.id, len(self.thoughts)
+        # )
+
+
+class KeepValid(Operation):
+    """
+    Operation to keep valid thoughts from predecessors.
+    """
+
+    operation_type: OperationType = OperationType.keep_valid
+
+    def __init__(self) -> None:
+        """
+        Initializes a new KeepValid operation.
+        """
+        super().__init__()
+        self.thoughts: List[Thought] = []
+
+    def get_thoughts(self) -> List[Thought]:
+        """
+        Returns the thoughts kept by the operation.
+
+        :return: List of kept thoughts.
+        :rtype: List[Thought]
+        """
+        return self.thoughts
+
+    def _execute(
+        self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
+    ) -> None:
+        """
+        Executes the KeepValid operation by keeping the valid thoughts from the predecessors.
+        Keeps unvalidated thoughts as well.
+
+        :param lm: The language model to be used.
+        :type lm: AbstractLanguageModel
+        :param prompter: The prompter for crafting prompts.
+        :type prompter: Prompter
+        :param parser: The parser for parsing responses.
+        :type parser: Parser
+        :param kwargs: Additional parameters for execution.
+        :raises AssertionError: If operation has no predecessors.
+        """
+        assert (
+            len(self.predecessors) >= 1
+        ), "KeepValid operation must have at least one predecessor"
+
+        self.thoughts: List[Thought] = [
+            Thought.from_thought(thought)
+            for thought in self.get_previous_thoughts()
+            if not thought.validated or thought.valid
+        ]
+
+        # if any(not thought.validated for thought in self.thoughts):
+        #     self.logger.warning(
+        #         "KeepValid operation %d has unvalidated thoughts", self.id
+        #     )
+
+        # for thought in self.thoughts:
+        #     self.logger.debug(
+        #         "Thought %d with state %s kept", thought.id, thought.state
+        #     )
+
+        # self.logger.debug(
+        #     "KeepValid operation %d kept %d thoughts", self.id, len(self.thoughts)
+        # )
+
+
+class GroundTruth(Operation):
+    """
+    Operation to evaluate if thoughts correctly solve the problem, using a ground truth evaluator
+    """
+
+    operation_type: OperationType = OperationType.ground_truth_evaluator
+
+    def __init__(self, ground_truth_evaluator: Callable[[Dict], bool]) -> None:
+        """
+        Initializes a new GroundTruth operation.
+
+        :param ground_truth_evaluator: A function to evaluate if a thought solves the problem.
+        :type ground_truth_evaluator: A function that takes a thought state and returns a boolean.
+        """
+        super().__init__()
+        self.ground_truth_evaluator: Callable[[Dict], bool] = ground_truth_evaluator
+        self.thoughts: List[Thought] = []
+
+    def get_thoughts(self) -> List[Thought]:
+        """
+        Returns the thoughts associated with the operation.
+
+        :return: List of evaluated thoughts.
+        :rtype: List[Thought]
+        """
+        return self.thoughts
+
+    def _execute(
+        self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
+    ) -> None:
+        """
+        Executes the GroundTruth operation by evaluating the predecessors' thoughts using the ground truth evaluator function.
+
+        :param lm: The language model to be used.
+        :type lm: AbstractLanguageModel
+        :param prompter: The prompter for crafting prompts.
+        :type prompter: Prompter
+        :param parser: The parser for parsing responses.
+        :type parser: Parser
+        :param kwargs: Additional parameters for execution.
+        :raises AssertionError: If operation has no predecessor.
+        """
+        assert (
+            len(self.predecessors) >= 1
+        ), "GroundTruth operation must have at least one predecessor"
+
+        previous_thoughts: List[Thought] = self.get_previous_thoughts()
+
+        for thought in previous_thoughts:
+            new_thought = Thought.from_thought(thought)
+            try:
+                new_thought.solved = self.ground_truth_evaluator(new_thought.state)
+            except:
+                new_thought.solved = False
+            self.thoughts.append(new_thought)
+
+        # self.logger.debug(
+        #     "GroundTruth operation %d evaluated %d thoughts and %d solved the problem",
+        #     self.id,
+        #     len(self.thoughts),
+        #     len([thought for thought in self.thoughts if thought.solved]),
+        # )
+
+
+class Selector(Operation):
+    """
+    Operation to select thoughts from predecessors.
+    Useful for separating thoughts to perform different, subsequent operations on them.
+    """
+
+    operation_type: OperationType = OperationType.selector
+
+    def __init__(self, selector: Callable[[List[Thought]], List[Thought]]) -> None:
+        """
+        Initializes a new Selector operation.
+
+        :param selector: A function to select thoughts from the predecessors' thoughts.
+        :type selector: A function that takes a list of thoughts and returns a list of thoughts.
+        """
+        super().__init__()
+        self.selector: Callable[[List[Thought]], List[Thought]] = selector
+        self.thoughts: List[Thought] = []
+
+    def get_thoughts(self) -> List[Thought]:
+        """
+        Returns the thoughts selected by the operation.
+
+        :return: List of selected thoughts.
+        :rtype: List[Thought]
+        """
+        return self.thoughts
+
+    def _execute(
+        self, lm: AbstractLanguageModel, prompter: Prompter, parser: Parser, **kwargs
+    ) -> None:
+        """
+        Executes the Selector operation by selecting thoughts from the predecessors using the selector function.
+        If the Selector has no predecessors, the selector function is called with a thought containing the kwargs as state.
+
+        :param lm: The language model to be used.
+        :type lm: AbstractLanguageModel
+        :param prompter: The prompter for crafting prompts.
+        :type prompter: Prompter
+        :param parser: The parser for parsing responses.
+        :type parser: Parser
+        :param kwargs: Additional parameters for execution.
+        """
+        previous_thoughts: List[Thought] = self.get_previous_thoughts()
+
+        if len(previous_thoughts) == 0:
+            previous_thoughts = [Thought(kwargs)]
+
+        self.thoughts = [
+            Thought.from_thought(thought)
+            for thought in self.selector(previous_thoughts)
+        ]
+
+        # for thought in self.thoughts:
+        #     self.logger.debug(
+        #         "Thought %d with state %s selected", thought.id, thought.state
+        #     )
+
+        # self.logger.debug(
+        #     "Selector operation %d selected %d thoughts", self.id, len(self.thoughts)
+        # )

src/llm_explain/utility/graph_of_thoughts/operations/thought.py

@@ -0,0 +1,129 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from __future__ import annotations
+# import logging
+from typing import Iterator, Dict, Optional
+import itertools
+
+from llm_explain.config.logger import CustomLogger
+
+logging = CustomLogger()
+
+class Thought:
+    """
+    Represents an LLM thought with its state, constructed by the parser, and various flags.
+    """
+
+    _ids: Iterator[int] = itertools.count(0)
+
+    def __init__(self, state: Optional[Dict] = None) -> None:
+        """
+        Initializes a new Thought instance with a state and various default flags.
+
+        :param state: The state of the thought. Defaults to None.
+        :type state: Optional[Dict]
+        """
+        self.logger = CustomLogger()
+        self.id: int = next(Thought._ids)
+        self.state: Dict = state
+        self._score: float = 0.0
+        self._valid: bool = False
+        self._solved: bool = False
+        self.scored: bool = False
+        self.validated: bool = False
+        self.compared_to_ground_truth: bool = False
+
+    @staticmethod
+    def from_thought(thought: Thought) -> Thought:
+        """
+        Creates a new thought from an existing one.
+
+        :param thought: An instance of a Thought to clone.
+        :return: A new Thought instance with properties copied from the input thought.
+        """
+        new_thought = Thought(thought.state)
+        new_thought.score = thought.score
+        new_thought.valid = thought.valid
+        new_thought.solved = thought.solved
+        new_thought.scored = thought.scored
+        new_thought.validated = thought.validated
+        new_thought.compared_to_ground_truth = thought.compared_to_ground_truth
+        return new_thought
+
+    @property
+    def valid(self) -> bool:
+        """
+        Returns the validity of the thought.
+
+        :return: The validity of the thought.
+        :rtype: bool
+        """
+        return self._valid
+
+    @valid.setter
+    def valid(self, valid: bool) -> None:
+        """
+        Sets the validity of the thought and the validated flag.
+
+        :param valid: The validity of the thought.
+        :type valid: bool
+        """
+        self.validated = True
+        self._valid = valid
+
+    @property
+    def score(self) -> float:
+        """
+        Returns the score of the thought.
+
+        :return: The score of the thought.
+        :rtype: float
+        """
+        return self._score
+
+    @score.setter
+    def score(self, new_score: float) -> None:
+        """
+        Sets the score of the thought and the scored flag.
+
+        :param new_score: The score of the thought.
+        :type new_score: float
+        """
+        self.scored = True
+        self._score = new_score
+
+    @property
+    def solved(self) -> bool:
+        """
+        Returns the solved flag of the thought.
+
+        :return: The solved flag of the thought.
+        :rtype: bool
+        """
+        return self._solved
+
+    @solved.setter
+    def solved(self, solved: bool) -> None:
+        """
+        Sets the solved flag of the thought and the compared_to_ground_truth flag.
+
+        :param solved: Whether the thought contains a solution to the problem.
+        :type solved: bool
+        """
+        self.compared_to_ground_truth = True
+        self._solved = solved

src/llm_explain/utility/graph_of_thoughts/parser/__init__.py

@@ -0,0 +1,18 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from .parser import Parser

src/llm_explain/utility/graph_of_thoughts/parser/parser.py

@@ -0,0 +1,99 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from typing import Dict, List, Union
+
+
+class Parser(ABC):
+    """
+    Abstract base class that defines the interface for all parsers.
+    Parsers are used to parse the responses from the language models.
+    """
+
+    @abstractmethod
+    def parse_aggregation_answer(
+        self, states: List[Dict], texts: List[str]
+    ) -> Union[Dict, List[Dict]]:
+        """
+        Parse the response from the language model for a aggregation prompt.
+
+        :param states: The thought states used to generate the prompt.
+        :type states: List[Dict]
+        :param texts: The responses to the prompt from the language model.
+        :type texts: List[str]
+        :return: The new thought states after parsing the response from the language model.
+        :rtype: Union[Dict, List[Dict]]
+        """
+        pass
+
+    @abstractmethod
+    def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
+        """
+        Parse the response from the language model for an improve prompt.
+
+        :param state: The thought state used to generate the prompt.
+        :type state: Dict
+        :param texts: The responses to the prompt from the language model.
+        :type texts: List[str]
+        :return: The new thought state after parsing the response from the language model.
+        :rtype: Dict
+        """
+        pass
+
+    @abstractmethod
+    def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
+        """
+        Parse the response from the language model for a generate prompt.
+
+        :param state: The thought state used to generate the prompt.
+        :type state: Dict
+        :param texts: The responses to the prompt from the language model.
+        :type texts: List[str]
+        :return: The new thought states after parsing the response from the language model.
+        :rtype: List[Dict]
+        """
+        pass
+
+    @abstractmethod
+    def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
+        """
+        Parse the response from the language model for a validation prompt.
+
+        :param state: The thought state used to generate the prompt.
+        :type state: Dict
+        :param texts: The responses to the prompt from the language model.
+        :type texts: List[str]
+        :return: Whether the thought state is valid or not.
+        :rtype: bool
+        """
+        pass
+
+    @abstractmethod
+    def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
+        """
+        Parse the response from the language model for a score prompt.
+
+        :param states: The thought states used to generate the prompt.
+        :type states: List[Dict]
+        :param texts: The responses to the prompt from the language model.
+        :type texts: List[str]
+        :return: The scores for the thought states.
+        :rtype: List[float]
+        """
+        pass

src/llm_explain/utility/graph_of_thoughts/prompter/__init__.py

@@ -0,0 +1,18 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from .prompter import Prompter

src/llm_explain/utility/graph_of_thoughts/prompter/prompter.py

@@ -0,0 +1,95 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from typing import Dict, List
+
+
+class Prompter(ABC):
+    """
+    Abstract base class that defines the interface for all prompters.
+    Prompters are used to generate the prompts for the language models.
+    """
+
+    @abstractmethod
+    def aggregation_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
+        """
+        Generate a aggregation prompt for the language model.
+
+        :param state_dicts: The thought states that should be aggregated.
+        :type state_dicts: List[Dict]
+        :param kwargs: Additional keyword arguments.
+        :return: The aggregation prompt.
+        :rtype: str
+        """
+        pass
+
+    @abstractmethod
+    def improve_prompt(self, **kwargs) -> str:
+        """
+        Generate an improve prompt for the language model.
+        The thought state is unpacked to allow for additional keyword arguments
+        and concrete implementations to specify required arguments explicitly.
+
+        :param kwargs: Additional keyword arguments.
+        :return: The improve prompt.
+        :rtype: str
+        """
+        pass
+
+    @abstractmethod
+    def generate_prompt(self, num_branches: int, **kwargs) -> str:
+        """
+        Generate a generate prompt for the language model.
+        The thought state is unpacked to allow for additional keyword arguments
+        and concrete implementations to specify required arguments explicitly.
+
+        :param num_branches: The number of responses the prompt should ask the LM to generate.
+        :type num_branches: int
+        :param kwargs: Additional keyword arguments.
+        :return: The generate prompt.
+        :rtype: str
+        """
+        pass
+
+    @abstractmethod
+    def validation_prompt(self, **kwargs) -> str:
+        """
+        Generate a validation prompt for the language model.
+        The thought state is unpacked to allow for additional keyword arguments
+        and concrete implementations to specify required arguments explicitly.
+
+        :param kwargs: Additional keyword arguments.
+        :return: The validation prompt.
+        :rtype: str
+        """
+        pass
+
+    @abstractmethod
+    def score_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
+        """
+        Generate a score prompt for the language model.
+
+        :param state_dicts: The thought states that should be scored,
+                            if more than one, they should be scored together.
+        :type state_dicts: List[Dict]
+        :param kwargs: Additional keyword arguments.
+        :return: The score prompt.
+        :rtype: str
+        """
+        pass

src/llm_explain/utility/prompt_utils.py

@@ -0,0 +1,111 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+class Prompt:
+        
+    def get_prompt(prompt, response):
+        
+        template = f'''
+            You are a Responsible AI expert with extensive experience in Explainable AI for Large Language Models. Your role is to clearly generate explanations for why the Large Language Model has generated a certain response for the given prompt.
+
+            You are a helpful assistant. Do not fabricate information or provide assumptions in your response.
+
+            Given the following prompt-response pair:
+
+            Prompt: {prompt}
+            Response: {response}
+
+            Please assess the following metrics:
+
+            1. Sentiment: Evaluate the sentiment associated with the response using a score between -1 (negative), 0 (neutral), and 1 (positive). Additionally, explain your reasoning behind the assigned sentiment score.
+
+            2. Grammatical Mistakes: Evaluate the grammatical correctness of the response using a score between 0 (no mistakes) and 1 (more mistakes). Additionally, explain your reasoning behind the assigned grammatical mistakes score.
+
+            3. Uncertainty: Evaluate the uncertainty associated with the response for the given prompt using a score between 0 (certain) and 1 (highly uncertain). Additionally, explain your reasoning behind the assigned uncertainty score.
+
+            4. Out of Vocabulary (OOV): Assess the percentage of out-of-vocabulary words in the response relative to the prompt using a score between 0 and 100. Additionally, explain your reasoning behind the assigned OOV words percentage.
+
+            5. Coherence: Evaluate the logical flow and coherence of the response using a score between 0 (incoherent) and 1 (highly coherent). Additionally, explain your reasoning behind the assigned coherence score.
+
+            6. Relevance: Assess the relevance of the response to the given prompt using a score between 0 (irrelevant) and 1 (highly relevant). Additionally, explain your reasoning behind the assigned relevance score.
+
+            7. How did you arrive at the following response based on the prompt provided?
+                Prompt: {prompt}
+                Response: {response}
+                Explain the reasoning and steps taken to generate this response.
+
+            Your response should be in the following JSON format:
+            {{
+                "sentiment": {{
+                    "score": "",
+                    "explanation": ""
+                }},
+                "grammatical_mistakes": {{
+                    "score": "",
+                    "explanation": ""
+                }},
+                "uncertainty": {{
+                    "score": "",
+                    "explanation": ""
+                }},
+                "out_of_vocabulary": {{
+                    "score": "",
+                    "explanation": ""
+                }},
+                "coherence": {{
+                    "score": "",
+                    "explanation": ""
+                }},
+                "relevance": {{
+                    "score": "",
+                    "explanation": ""
+                }},
+                "reasoning": ""
+            }}
+
+            Do not provide any response other than the JSON object.
+            '''
+        return template
+
+    def get_token_importance_prompt(prompt):
+       
+        template = f'''
+            You are a helpful assistant. Do not fabricate information or do not provide assumptions in your response.
+ 
+            Given the following prompt:
+ 
+            Prompt: {prompt}
+ 
+            Please assess the following metric:
+ 
+            1. Token Importance: Evaluate the importance of each token in the prompt. Calculate the importance value of each token in the given prompt using a
+            score between 0 (low importance) and 1 (high importance). Provide all the tokens as a list. Ensure that you give an importance score for all tokens,
+            and there are no empty spaces or inconsistencies in the output, which might cause issues while parsing. Make your analysis consistent so that if given
+            the same input again, you produce a similar output.
+ 
+            Your response should be in the following JSON format:
+ 
+            output-format:
+            {{
+                "Token": ["Each Token from input prompt"],
+                "Importance Score": ["The value here should be a comma-separated list of importance scores"],
+                "Position": ["The value here should be a comma-separated list of respective token index positions"]
+            }}
+ 
+            Do not provide any response other than the JSON object.
+        '''
+        return template

src/llm_explain/utility/prompts/base.py

@@ -0,0 +1,281 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from llm_explain.utility.prompts.output_format import *
+from llm_explain.utility.prompts.few_shot import *
+from llm_explain.utility.prompts.instructions import *
+
+class Prompt:
+
+    def get_classification_prompt(input_prompt):
+        
+        template = f"""
+            Imagine you are a Responsible AI expert with experience in explaining why a model has made a decision. 
+            Your task is to determine the sentiment of the given prompt, identify the keywords the model used to arrive at that sentiment, and provide a clear explanation of why the model classified the prompt as that sentiment.
+
+            Calculate the importance value of each token  towards getting the overall sentiment from the given prompt using a score between 1 (low importance) and 100 (high importance). 
+            Provide all the tokens as a list. Ensure that you give an importance score for all tokens, and there are no empty spaces or inconsistencies in the output, which might cause issues while parsing. 
+            Make your analysis consistent so that if given the same input again, you produce a similar output.
+
+            similarly, provide sentiment, keywords identified to determine the sentiment and Explanation for the below given information.
+
+            Make sure the response is simple and easy to understand. Use polite language. Do not write as a third person. Do not include Certainly! at the beginning of your response, just give response.
+            
+            Example Data:
+            {one_shot_sentiment_analysis}
+            
+            Return the output only in the following JSON format. Do not output anything other than this JSON object:
+            {output_format_sentiment_analysis}
+
+            Task Data:
+            [Prompt]: {input_prompt}
+            """
+        return template
+        
+    def get_local_explanation_prompt(prompt, response):
+
+        template = f'''
+            You are a Responsible AI expert with extensive experience in Explainable AI for Large Language Models. Your role is to clearly generate explanations for why the Large Language Model has generated a certain response for the given prompt.
+
+            You are a helpful assistant. Do not fabricate information or provide assumptions in your response.
+
+            Given the following prompt-response pair:
+
+            Prompt: {prompt}
+            Response: {response}
+
+            Please assess the following metrics:
+
+            1. Uncertainty: Evaluate the uncertainty associated with the response for the given prompt using a score between 0 (certain) and 95 (highly uncertain). Additionally, explain your reasoning behind the assigned score.
+            2. Coherence: Evaluate the logical flow and coherence of the response using a score between 0 (incoherent) and 95 (highly coherent). Additionally, explain your reasoning behind the assigned score.
+
+            Based on the score and explanation for each metric, provide a recommendation for how to change the input prompt so that the response will be better and the scores will improve. Ensure that each recommendation is concrete and actionable. If a metric has a perfect score, provide positive reinforcement or suggest maintaining the current quality.
+
+            Return the output only in the following JSON format. Do not output anything other than this JSON object:
+            {output_format_local_explanation}
+            
+        '''
+        return template
+
+    def get_token_importance_prompt(prompt):
+       
+        template = f'''
+            You are a helpful assistant. Do not fabricate information or do not provide assumptions in your response.
+ 
+            Given the following prompt:
+ 
+            Prompt: {prompt}
+ 
+            Please assess the following metric:
+            1. Token Importance: Evaluate the importance of each token in the prompt. Calculate the importance value of each token in the given prompt using a
+            score between 0 (low importance) and 1 (high importance). Provide all the tokens as a list. Ensure that you give an importance score for all tokens,
+            and there are no empty spaces or inconsistencies in the output, which might cause issues while parsing. Make your analysis consistent so that if given
+            the same input again, you produce a similar output.
+ 
+            Return the output only in the following JSON format. Do not output anything other than this JSON object:
+            {output_format_token_importance}
+        '''
+        return template
+    
+    def get_tone_prediction_prompt(response):
+
+        template = f'''
+            You are a detail-oriented LLM that pays close attention to the nuances of language. 
+            You will be given a text and your job is to analyze its tone. 
+
+            Specifically, you need to consider the following tones and identify which tone is most appropriate for the given text:
+
+            Formal: Professional, respectful, objective (e.g., scientific reports, business emails)
+            Informal: Casual, conversational, friendly (e.g., text messages, social media posts)
+            Informative: Primarily focused on conveying information clearly and concisely (e.g., news reports, summaries)
+            Positive: Happy, optimistic, encouraging (e.g., motivational speeches, congratulations)
+            Negative: Sad, angry, frustrated (e.g., complaints, critical reviews)
+            Neutral: Objective, unbiased, unemotional (e.g., factual summaries, news reports)
+            Humorous: Funny, witty, sarcastic (e.g., jokes, lighthearted stories)
+            Dramatic: Suspenseful, exciting, intense (e.g., fictional narratives, descriptions of events)
+            Inspiring: Uplifting, motivating, thought-provoking (e.g., speeches, self-help content)
+            Persuasive: Trying to convince the reader of something (e.g., marketing copy, arguments)
+            Empathetic: Understanding and supportive (e.g., responses to someone going through a tough time)
+            Authoritative: Confident, knowledgeable (e.g., expert opinions, instructions)
+
+            Based on the score and explanation for each metric, provide a recommendation for how to change the input prompt so that the response will be better and the scores will improve. Ensure that each recommendation is concrete and actionable. If a metric has a perfect score, provide positive reinforcement or suggest maintaining the current quality.
+
+            Example Data:
+            {few_shot_examples_tone_analysis}
+
+            Return the output only in the following JSON format. Do not output anything other than this JSON object:
+            {output_format_tone_analysis}
+
+            Task Data:
+            [Response]: {response}
+        '''
+        return template
+    
+    def get_coherehce_prompt(response):
+
+        template = f"""
+            You are a detail-oriented LLM which pays close attention to the details. You are given a text and your job is to evaluate the quality of the provided text, focusing on the coherence aspect.
+
+            Coherence is the quality of the text that makes it logical and consistent. It is important that the text is well-organized and the ideas are connected in a clear and meaningful way. A coherent text is easy to follow and understand.
+
+            Please provide a score on the scale of 1-5, with 1 meaning that the text is completely incoherent and the elements in the text do not stitch together to produce meaningful text, and 5 meaning that the text is completely coherent and the elements in the text stitch together to produce meaningful text.
+
+            Example Data.
+            {LANGUAGE_COHERENCE_FEW_SHOT__COT}
+
+            First, analyze the text and determine how fluent and natural sounding it is. Consider the structure, connectivity of ideas, and overall readability of the text. Write down step-by-step reasoning to make sure that your conclusion is correct.
+
+            {CHAIN_OF_THOUGHT}
+
+            Return the output only in the corresponding JSON format. Do not output anything other than this JSON object:
+            {LANGUAGE_COHERENCE_OUTPUT_FORMAT__COT}
+
+            Task data.
+            [Resposne]: {response}
+        """
+        return template
+    
+    def get_response_revelance_prompt(prompt, response):
+
+        template = f"""
+            You are a detail-oriented LLM which pays close attention to the details, checks for consistency, and is adept at identifying logical fallacies, incorrect assumptions, or other errors in reasoning.
+            Your task is to determine the degree of irrelevant information present in the given response.
+
+            Example Data.
+            {RESPONSE_RELEVANCE_FEW_SHOT__COT}
+
+            For the given task data, carefully examine the response and assess if it has any additional irrelevant information or not. Don't focus on aspects like style, grammar, or punctuation. 
+            Assign a score between 0 and 1, where 0 indicates that the response is completely irrelevant to the prompt, and 1 indicates that the response is highly relevant to the prompt.
+            {CHAIN_OF_THOUGHT}
+
+            Return the output only in the corresponding JSON format. Do not output anything other than this JSON object:
+            {RESPONSE_RELEVANCE_OUTPUT_FORMAT__COT}
+
+            Task Data.
+            [Question]: {prompt}
+            [Response]: {response}
+            [Output]:
+        """
+
+        return template
+    
+    def generate_facts_prompt(prompt, response, current_date):
+
+        template = f"""
+            You are given a response along with its question. For the given task data, please breakdown the response into independent 
+            facts. A fact is a sentence that is true and can only be stated from the response. Facts should not depend on each another 
+            and must not convey the same information. While generating facts, ensure that the facts are contextually mentioned and 
+            do not begin with pronouns like 'He,' 'She,' or references to third-party entities. Limit to 5 facts in the output.
+            
+            Response may contain information that is not asked in Question, consider only required information in Response that is 
+            relevant to the Question.
+            
+            Example Data.
+            {FACT_GENERATE_FEW_SHOT}
+
+            Return the output only in the corresponding JSON format. Do not output anything other than this JSON object:
+            {FACT_GENERATE_OUTPUT_FORMAT}
+
+            Task Data.
+            [Question]: {prompt}
+            [Response]: {response}
+            [Output]: 
+        """
+
+        return template
+    
+    def evaluate_facts_prompt(facts, context, prompt):
+
+        template = f"""
+            You are a detail-oriented LLM whose task is to determine if the given facts or questions are supported by the given context
+            and prompt. 
+            Each fact or question is separated by the following symbol: "#". 
+
+            For the given task data, go over each fact or question sentence one by one, and write down your judgement.
+            If it is a question then answer the question based on the context and prompt. 
+            Use important dates if any available in the context to make a better judgement.
+            If it is a fact, determine if the fact is supported by both context and prompt.
+
+            Before answering, reason in a step-by-step manner to provide your final judgement.
+            If the reasoning is clear then give judgement as "yes" or "no" otherwise give judgement as "unclear".
+
+            Example Data.
+            {FACT_EVAL_FEW_SHOT__COT}
+
+            Return the output only in the corresponding JSON format. Do not output anything other than this JSON object:
+            {FACT_EVALUATE_OUTPUT_FORMAT__COT}
+
+            Task Data.
+            [Prompt]: {prompt}
+            [Facts]: {facts}
+            [Context]: {context}
+            [Output]:
+        """
+
+        return template
+    
+    def filter_facts_prompt(prompt, facts):
+
+        FACT_FILTER_PROMPT_TEMPLATE = f"""
+            You are provided with a list of facts generated from a response to a specific question. Your task is to filter and retain only those 
+            facts that are directly relevant to the question. Ignore any facts that do not pertain to the original question.
+
+            While filtering facts, ensure that the facts are contextually mentioned and 
+            do not begin with pronouns like 'He,' 'She,' or references to third-party entities. If any such facts are identified, rewrite them.
+
+            Focus on identifying and selecting the facts that specifically answer the question asked, while discarding any irrelevant 
+            or off-topic information.
+
+            Example Data:
+            {FACT_FILTER_FEW_SHOT}
+
+            Return the output in the specified JSON format. If no relevant facts are found, return an empty list [].
+
+            Task Data:
+            [Question]: {prompt}
+            [Response Facts]: {facts}
+
+            [Output]:
+        """
+
+        return FACT_FILTER_PROMPT_TEMPLATE
+    
+    def summarize_prompt(qa_dict_list):
+
+        SUMMARIZATION_PROMPT_TEMPLATE = f"""
+            You are provided with a list of JSON objects, each containing a 'question' and an 'answer'. The answer is obtained from
+            Google Search API and is a detailed response to the corresponding question.
+            
+            Your task is to create a separate summary for each question-answer pair, preserving the context and tense of the original 
+            question and answer. Don't mention anything other than what is given in answer.
+
+            Ensure that:
+            - Each summary is in a separate paragraph.
+            - There is a one-line space between each paragraph.
+            - The tense and context of the original question and answer are maintained accurately.
+
+            Ensure that there is no fabricated or hallucinated information in your response and make sure that there are no conflicting statements 
+            from one para to another in your summary. Do not mention paragraph or such type of words in your response, just summarize and provide answers.
+
+            Task Data:
+            [Input]: 
+            {qa_dict_list}
+
+            [Output]: 
+        """
+
+        return SUMMARIZATION_PROMPT_TEMPLATE

src/llm_explain/utility/prompts/few_shot.py

@@ -0,0 +1,218 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+# Sentiment Analysis
+one_shot_sentiment_analysis = """
+    Prompt: Unfortunately the movie served with hacked script but the actors performed well
+    {
+        "Sentiment": "Negative",
+        "Keywords": ['unfortunately', 'hacked'],
+        "Explanation": The model predicted the input as Negative because it anchored or focused on the word "unfortunately" and "hacked" in your input. In this context, the model likely learned that when someone mentions a hacked script or unfortunately, it usually indicates a negative opinion about a movie. Despite the actors performing well, the emphasis on the script being hacked led the model to classify the overall sentiment as negative. This shows that the model paid close attention to specific keywords, like "unfortunately" and "hacked" to make its prediction.
+        "token_importance_mapping": [
+                                        {"token": "Unfortunately", "importance_score": 0.8, "position": 0}, 
+                                        {"token": "movie", "importance_score": 0.5, "position": 1}, 
+                                        {"token": "served", "importance_score": 0.3, "position": 2}, 
+                                        {"token": "with", "importance_score": 0.2, "position": 3}, 
+                                        {"token": "hacked", "importance_score": 0.9, "position": 4}, 
+                                        {"token": "script", "importance_score": 0.7, "position": 5}, 
+                                        {"token": "but", "importance_score": 0.4, "position": 6}, 
+                                        {"token": "the", "importance_score": 0.1, "position": 7}, 
+                                        {"token": "actors", "importance_score": 0.6, "position": 8}, 
+                                        {"token": "performed", "importance_score": 0.5, "position": 9}, 
+                                        {"token": "well", "importance_score": 0.7, "position": 10}
+                                    ]
+    }
+"""
+
+# Tone Analysis
+few_shot_examples_tone_analysis = """
+    [Response]: Absolutely! I'm thrilled about the opportunity and eager to learn more about the position.
+    [Output]: 
+    {
+        "Reasoning": "The text uses positive and enthusiastic language, along with keywords like 'thrilled' and 'eager,' indicating a strong interest in the position.",
+        "Tone": ['Positive', 'Formal']
+        "Score": 0.8
+        "Recommendation":
+    }
+
+    [Response]: Ugh, another interview? Why do I even bother?
+    [Output]:
+    {
+        "Reasoning": "The text expresses negativity and frustration with keywords like 'ugh' and 'bother.'",
+        "Tone": ['Negative', 'Informal']
+        "Score": 0.2
+        "Recommendation":
+    }
+
+    [Response]: Congratulations on your promotion! You absolutely deserve it!
+    [Output]:
+    {
+        "Reasoning": "The text uses positive and celebratory language with emojis and encouraging words.",
+        "Tone": ['Positive', 'Informal']
+        "Score": 0.5
+        "Recommendation":
+    }
+"""
+
+# Response Coherence
+LANGUAGE_COHERENCE_FEW_SHOT__COT = """
+    [Response]: Exercise is beneficial for both physical and mental health. It strengthens the body and uplifts the mind.
+    [Output]:
+    {
+        "Reasoning": "The text is coherent and effectively conveys the message with clear organization of ideas.",
+        "Score": 5
+    }
+
+    [Response]: Regular exercise contributes to overall well-being by enhancing physical strength and mental clarity.
+    [Output]:
+    {
+        "Reasoning": "The text maintains coherence by linking ideas logically, providing a clear flow of information.",
+        "Score": 4
+    }
+
+    [Response]: Exercise good. Health. Make body strong. Help mind. Benefits many.
+    [Output]:
+    {
+        "Reasoning": "The text lacks coherence, as it presents fragmented ideas without clear connections.",
+        "Score": 2
+    }
+"""
+
+# Response Relevance
+RESPONSE_RELEVANCE_FEW_SHOT__COT = """
+[Question]: Who is Lionel Messi?
+[Response]: Lionel Andrés Messi (born 24 June 1987), also known as Leo Messi, is an Argentine professional footballer who plays as a forward for and captains both Major League Soccer club Inter Miami and the Argentina national team. Widely regarded as one of the greatest players of all time, Messi has won a record seven Ballon d'Or awards] and a record six European Golden Shoes, and in 2020 he was named to the Ballon d'Or Dream Team. Until leaving the club in 2021, he had spent his entire professional career with Barcelona, where he won a club-record 34 trophies, including ten La Liga titles, seven Copa del Rey titles and the UEFA Champions League four times.
+[Output]:
+{
+    "Reasoning": "While the given response provides information about the birth data, nationality and occupation of Lionel Messi, it includes some irrelevant details about Messi's career such as association to multiple clubs and trophies won.",
+    "Score": 0.5
+}
+
+[Question]: Who is Lionel Messi?
+[Response]: Lionel Andrés Messi (born 24 June 1987), also known as Leo Messi, is an Argentine professional footballer who plays as a forward for and captains both Major League Soccer club Inter Miami and the Argentina national team. Widely regarded as one of the greatest players of all time, Messi has won a record seven Ballon d'Or awards] and a record six European Golden Shoes, and in 2020 he was named to the Ballon d'Or Dream Team. Until leaving the club in 2021, he had spent his entire professional career with Barcelona, where he won a club-record 34 trophies, including ten La Liga titles, seven Copa del Rey titles and the UEFA Champions League four times. During his time with Barcelona, Barcelona featured star players like Neymar, Andres Iniesta and was managed by Luis Enrique.
+[Output]:
+{
+    "Reasoning": "While the given response provides information about the birth data, nationality and occupation of Lionel Messi, it includes a lot of irrelevant inforamtion such as details about Messi's career and Barcelona club.",
+    "Choice": 0.0
+}
+"""
+
+# Fact Generation
+FACT_GENERATE_FEW_SHOT = """
+[Question]: Which is the tallest monument in Paris?
+[Response]: The Eiffel Tower, located in Paris, is one of the most visited monuments in the world. It was named after the engineer Gustave Eiffel, whose company designed and built the tower. Constructed from 1887 to 1889, it was initially criticized by some of France's leading artists and intellectuals.
+[Output]: 
+[
+    {
+        "Fact": "The Eiffel Tower is located in Paris."
+    },
+    {
+        "Fact": "The Eiffel Tower is the tallest structure in Paris."
+    },
+    {
+        "Fact": "The Eiffel Tower is very old."
+    },
+    {
+        "Fact": "The Eiffel Tower is very old."
+    },
+]
+
+[Question]: who is the Chief minister of Andhra Pradesh ?
+[Response]: As of my last training data in October 2021, the Chief Minister of Andhra Pradesh is Yeduguri Sandinti Jaganmohan Reddy. He assumed office on 30th May 2019. He is a member of the Yuvajana Sramika Rythu Congress Party (YSR Congress Party). However, you should verify from a reliable source for the most current information, as political positions can change.
+[Output]: 
+[
+    {
+        "Fact": "The Chief Minister of Andhra Pradesh is Yeduguri Sandinti Jaganmohan Reddy."
+    },
+    {
+        "Fact": "Yeduguri Sandinti Jaganmohan Reddy assumed office on 30th May 2019."
+    },
+    {
+        "Fact": "Yeduguri Sandinti Jaganmohan Reddy is a member of the Yuvajana Sramika Rythu Congress Party (YSR Congress Party)."
+    }
+]
+
+[Question]: Is Leaning Tower of Pisa, which is located in Italy, the oldest monument in Europe?
+[Response]: No
+[Output]: 
+{
+    "Fact": "The Leaning Tower of Pisa is not the oldest monument in Europe."
+}
+"""
+
+# Factuality Evaluation
+FACT_EVAL_FEW_SHOT__COT = """
+[Facts]: ["1. The Eiffel Tower is located in Paris.", "2. The Eiffel Tower is the tallest structure in Paris.", "3. The Eiffel Tower is very old."]
+[Context]: The Eiffel Tower, located in Paris, is one of the most visited monuments in the world. It was named after the engineer Gustave Eiffel, whose company designed and built the tower. Constructed from 1887 to 1889, it was initially criticized by some of France's leading artists and intellectuals.
+[Output]: 
+{
+    "Result": [
+        {
+            "Fact": "1. The Eiffel Tower is located in Paris.",
+            "Reasoning": "The context explicity states that Paris, one of the most visited monuments in the world is located in Paris.",
+            "Judgement": "yes"
+        },
+        {
+            "Fact": "2. The Eiffel Tower is the tallest structure in Paris.",
+            "Reasoning": "While the context speaks about the popularity of Effiel Tower, it has no mention about its height or whether it is tallest or not.",
+            "Judgement": "no"
+        },
+        {
+            "Fact": "3. The Eiffel Tower is very old.",
+            "Reasoning": "While the context mentions that the Eiffel Tower was built in 1880s, it doesn't clarify what very old means.",
+            "Judgement": "unclear"
+        },
+    ]
+}
+"""
+
+# Filter Facts
+FACT_FILTER_FEW_SHOT = """
+[Question]: What are the main components of a computer?
+[Response Facts]: 
+[
+    "A computer has a CPU, which is the brain of the computer.",
+    "Computers are widely used in schools and offices.",
+    "A computer's motherboard connects all the components.",
+    "The CPU can be overclocked for better performance.",
+    "Computers were invented in the 20th century."
+]
+
+[Output]:
+[
+    "A computer has a CPU, which is the brain of the computer.",
+    "A computer's motherboard connects all the components.",
+    "The CPU can be overclocked for better performance."
+]
+
+[Question]: What are the benefits of exercise?
+[Response Facts]: 
+[
+    "Regular exercise improves cardiovascular health.",
+    "Many people exercise in the morning.",
+    "Exercise helps maintain a healthy weight.",
+    "A balanced diet is also important for health.",
+    "Exercise can reduce the risk of chronic diseases."
+]
+
+[Output]:
+[
+    "Regular exercise improves cardiovascular health.",
+    "Exercise helps maintain a healthy weight.",
+    "Exercise can reduce the risk of chronic diseases."
+]
+"""

src/llm_explain/utility/prompts/instructions.py

@@ -0,0 +1,20 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+CHAIN_OF_THOUGHT = (
+    "Before answering, reason in a step-by-step manner as to get the right answer."
+)

src/llm_explain/utility/prompts/output_format.py

@@ -0,0 +1,122 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+# Sentiment Analysis
+output_format_sentiment_analysis = """
+    {
+        "Sentiment": ["Sentiment of the given prompt"]
+        "Keywords": [List of contributing keywords for the sentiment]
+        "Explanation": ["Explanation for the sentiment"]
+        "token_importance_mapping": [Dictionary of token importance mapping]
+    }
+"""
+
+# Local Explanation
+output_format_local_explanation = """
+    {
+        "uncertainty": {
+            "score": Integer,
+            "explanation": "",
+            "recommendation": ""
+        },
+        "coherence": {
+            "score": Integer,
+            "explanation": "",
+            "recommendation": ""
+        }
+    }
+"""
+
+# Token Importance
+output_format_token_importance = """
+    {
+        "Token": ["Each Token from input prompt"],
+        "Importance Score": ["The value here should be a comma-separated list of importance scores"],
+        "Position": ["The value here should be a comma-separated list of respective token index positions"]
+    }
+"""
+
+# Tone Analysis
+output_format_tone_analysis = """
+    {
+        "Reasoning": [Reasoning],  # Reasoning to critique the tone of the response, start with "Your response...."
+        "Tone": [Tones]
+        "Score": [Score]
+        "Recommendation": [Recommendation]
+    }
+"""
+
+# Coherence
+LANGUAGE_COHERENCE_OUTPUT_FORMAT__COT = """
+{
+    "Reasoning": [Reasoning],  # Reasoning to critique the coherence of the response,
+    "Score": [Score],  # Score between 1 to 5, to evaluate the coherence of the response
+}
+"""
+
+# Response Relevance
+RESPONSE_RELEVANCE_OUTPUT_FORMAT__COT = """
+{
+    "Reasoning": [Reasoning],  # Reasoning to determine the conciseness of the response for answering the query,
+    "Score": [Score],  # Score assigned for the relevance of the response to the query
+"""
+
+# Fact Generation
+FACT_GENERATE_OUTPUT_FORMAT = """
+{
+    "Facts": [ # List of all the facts
+            {
+                [1st Fact],  # 1st fact being analysed
+            },
+            {
+                [2nd Fact],  # 2nd fact being analysed
+            },
+        ... # Do for all the facts
+    ]
+}
+In case of no facts, return an empty list, i.e. [].
+"""
+
+# Factuality Evaluation
+FACT_EVALUATE_OUTPUT_FORMAT__COT = """
+{
+    "Result": [  # List containing data for all the facts
+        {
+            "Fact": [1st Fact],  # 1st fact being analysed,
+            "Reasoning": [Reasoning for 1st Fact],  # Reasoning to determine if the 1st fact can be verified from the context or not,
+            "Judgement": [Judgement for 1st Fact]   # Judgement for 1st fact. Select one of the three - "yes", "unclear" or "no",
+        },
+        {
+            "Fact": [2nd Fact],  # 2nd fact being analysed,
+            "Reasoning": [Reasoning for 2nd Fact],  # Reasoning to determine if the 2nd fact can be verified from the context or not,
+            "Judgement": [Judgement for 2nd Fact]   # Judgement for 2nd fact. Select one of the three - "yes", "unclear" or "no",
+        },
+        ... # Do for all the facts
+    ]
+}
+"""
+
+# Filter Facts
+FACT_FILTER_OUTPUT_FORMAT = """
+{
+    "RelevantFacts": [
+        "[Relevant Fact 1]",
+        "[Relevant Fact 2]",
+        ...
+    ]
+}
+"""

src/llm_explain/utility/query_serper.py

@@ -0,0 +1,143 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+"""Util that calls Google Search using the Serper.dev API."""
+from llm_explain.config.logger import CustomLogger
+import asyncio
+import aiohttp
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+log = CustomLogger()
+
+class GoogleSerperAPIWrapper():
+    """Wrapper around the Serper.dev Google Search API. You can create a free API key at https://serper.dev.
+    To use, you should have the environment variable ``SERPER_API_KEY`` set with your API key, or 
+    pass `serper_api_key` as a named parameter to the constructor.
+    """
+    def __init__(self, snippet_cnt = 10) -> None:
+        self.k = snippet_cnt
+        self.gl = "us"
+        self.hl = "en"
+        self.serper_api_key = os.environ.get("SERPER_KEY", None)
+        assert self.serper_api_key is not None, "Please set the SERPER_API_KEY environment variable."
+        assert self.serper_api_key != '', "Please set the SERPER_API_KEY environment variable."
+
+    async def _google_serper_search_results(self, session, search_term: str, gl: str, hl: str) -> dict:
+        headers = {
+            "X-API-KEY": self.serper_api_key or "",
+            "Content-Type": "application/json",
+        }
+        params = {"q": search_term, "gl": gl, "hl": hl}
+        try:
+            async with session.post(
+                "https://google.serper.dev/search", headers=headers, params=params, raise_for_status=True
+            ) as response:
+                return await response.json()
+        except aiohttp.ClientError as e:
+            log.error(f"HTTP request failed: {e}")
+            raise
+        except Exception as e:
+            log.error(f"An error occurred: {e}")
+            raise
+    
+    def _parse_results(self, results):
+        snippets = []
+
+        try:
+            if results.get("answerBox"):
+                answer_box = results.get("answerBox", {})
+                if answer_box.get("answer"):
+                    element = {"content":answer_box.get("answer"),"source":"None"}
+                    return [element]
+                    # snippets.append(element)
+                elif answer_box.get("snippet"):
+                    element = {"content":answer_box.get("snippet").replace("\n", " "),"source":"None"}
+                    return [element]
+                    # snippets.append(element)
+                elif answer_box.get("snippetHighlighted"):
+                    element = {"content":answer_box.get("snippetHighlighted"),"source":"None"}
+                    return [element]
+                    # snippets.append(element)
+                
+            if results.get("knowledgeGraph"):
+                kg = results.get("knowledgeGraph", {})
+                title = kg.get("title")
+                entity_type = kg.get("type")
+                if entity_type:
+                    element = {"content":f"{title}: {entity_type}","source":"None"}
+                    snippets.append(element)
+                description = kg.get("description")
+                if description:
+                    element = {"content":description,"source":"None"}
+                    snippets.append(element)
+                for attribute, value in kg.get("attributes", {}).items():
+                    element = {"content":f"{attribute}: {value}","source":"None"}
+                    snippets.append(element)
+
+            for result in results["organic"][: self.k]:
+                if "snippet" in result:
+                    element = {"content":result["snippet"],"source":result["link"]}
+                    snippets.append(element)
+                for attribute, value in result.get("attributes", {}).items():
+                    element = {"content":f"{attribute}: {value}","source":result["link"]}
+                    snippets.append(element)
+
+            if len(snippets) == 0:
+                element = {"content":"No good Google Search Result was found","source":"None"}
+                return [element]
+            
+            # keep only the first k snippets
+            snippets = snippets[:int(self.k / 2)]
+        except AttributeError as e:
+            if "'ClientResponseError' object has no attribute 'get'" in str(e):
+                log.error("Serper API key is invalid or has expired.")
+                raise Exception("Serper API key is invalid or has expired.")
+            else:
+                log.error(f"An error occurred while parsing results: {e}")
+                raise
+        except Exception as e:
+            log.error(f"An error occurred while parsing results: {e}")
+            raise
+
+        return snippets
+    
+    async def parallel_searches(self, search_queries, gl, hl):
+        async with aiohttp.ClientSession() as session:
+            tasks = [self._google_serper_search_results(session, query, gl, hl) for query in search_queries]
+            try:
+                search_results = await asyncio.gather(*tasks, return_exceptions=True)
+            except Exception as e:
+                log.error(f"An error occurred while running parallel searches: {e}")
+                raise
+            return search_results
+
+    async def run(self, queries):
+        """Run query through GoogleSearch and parse result."""
+
+        try:
+            results = await self.parallel_searches(queries, gl=self.gl, hl=self.hl)
+        except Exception as e:
+            log.error(f"An error occurred while running searches: {e}")
+            raise
+        snippets_list = []
+
+        for i in range(len(results)):
+            snippets_list.append(self._parse_results(results[i]))
+
+        return snippets_list

src/llm_explain/utility/utility.py

@@ -0,0 +1,316 @@
+'''
+Copyright 2024 Infosys Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies 
+or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
+AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+
+from llm_explain.config.logger import CustomLogger
+from sklearn.metrics.pairwise import cosine_similarity
+from matplotlib import pyplot as plt
+from openai import AzureOpenAI
+from tenacity import retry
+from tqdm import tqdm
+import pandas as pd
+import numpy as np
+import asyncio
+import base64
+import os
+import io
+
+from dotenv import load_dotenv
+load_dotenv()
+
+log = CustomLogger()
+
+class Utils:
+    
+    client = AzureOpenAI(
+            api_key=os.getenv("AZURE_OPENAI_API_KEY"),
+            api_version=os.getenv("AZURE_OPENAI_API_VERSION"),
+            azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT")
+        )
+    
+    def normalize_vector(v):
+        norm = np.linalg.norm(v)
+        if norm == 0:
+            return v
+        return v / norm
+    
+    def display_metrics(uncertainty_scores, completions, n):
+        try:
+            results = {}
+            structural_uncertainty = np.mean([np.mean(x) for x in uncertainty_scores['entropies']])
+            conceptual_uncertainty = (0.5*uncertainty_scores['mean_choice_distance']) + (0.5*np.mean([np.mean(x) for x in uncertainty_scores['distances']]))
+    
+            results["overall_cosine_distance"] = uncertainty_scores['mean_choice_distance']
+            results["Overall_Structural_Uncertainty"] = structural_uncertainty
+            results["Overall_Conceptual_Uncertainty"] = conceptual_uncertainty
+    
+            results["choices"] = []
+    
+            for i in range(n):
+                choice = {}
+                choice_text = completions['choices'][i]['text']
+                entropies = uncertainty_scores['entropies'][i]
+                distances = uncertainty_scores['distances'][i]
+    
+                logprobs = completions['choices'][i]['logprobs']['top_logprobs']
+                mean_entropy = np.mean(entropies)
+                mean_distance = np.mean(distances)
+    
+                choice["mean_entropy"] = mean_entropy
+                choice["mean_distance"] = mean_distance
+    
+                tokens = completions['choices'][i]['logprobs']['tokens']
+    
+                fixed_spacing = 1
+    
+                x_positions = [0]
+                for j in range(1, len(tokens)):
+                    x_positions.append(x_positions[-1] + len(tokens[j-1]) + fixed_spacing)
+    
+                df = pd.DataFrame({
+                    'x': x_positions,
+                    'y_text': [1]*len(tokens),
+                    'y_entropy': [1.2 + entropy for entropy in entropies],
+                    'y_distance': [1.2 + dist for dist in distances],
+                    'tokens': tokens,
+                    'logprobs': ['\n'.join([f"{k}: {v}" for k, v in lp.items()]) for lp in logprobs],
+                    'entropy': entropies,
+                    'distance': distances,
+                })
+    
+                plt.figure(figsize=(10, 6))
+                plt.title(f"Choice {i+1}")
+                plt.plot(df['x'], df['y_entropy'], label='Entropy', color='blue')
+                plt.plot(df['x'], df['y_distance'], label='Distance', color='red')
+                plt.xlabel('Token Position')
+                plt.ylabel('Normalization value')
+                plt.legend()
+    
+                buf = io.BytesIO()
+                plt.savefig(buf, format='png')
+                buf.seek(0)
+    
+                img_base64 = base64.b64encode(buf.read()).decode('utf-8')
+    
+                choice["plot_image_base64"] = img_base64
+                
+                choice['response'] = choice_text
+    
+                results["choices"].append(choice) 
+            return results
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise 
+
+    def calculate_normalized_entropy(logprobs):
+        """
+        Calculate the normalized entropy of a list of log probabilities.
+    
+        Parameters:
+            logprobs (list): List of log probabilities.
+        
+        Returns:
+            float: Normalized entropy.
+        """
+        try:
+            entropy = -np.sum(np.exp(logprobs) * logprobs)
+        
+            # Calculate maximum possible entropy for N tokens sampled
+            N = len(logprobs)
+            max_entropy = np.log(N)
+        
+            # Normalize the entropy
+            normalized_entropy = entropy/max_entropy
+            return normalized_entropy
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise 
+
+ 
+    async def process_token_async(i, top_logprobs_list, choice, choice_embedding, max_tokens):
+       
+       
+        """
+        Asynchronously process a token to calculate its normalized entropy and mean cosine distance.
+       
+        Parameters:
+            i (int): Token index.
+            top_logprobs_list (list): List of top log probabilities for each token.
+            choice (dict): The choice containing log probabilities and tokens.
+            choice_embedding (array): Embedding of the full choice.
+            max_tokens (int or None): Maximum number of tokens to consider for the partial string.
+           
+        Returns:
+            tuple: Mean cosine distance and normalized entropy for the token.
+        """
+        try:   
+            top_logprobs = top_logprobs_list[i]
+            normalized_entropy = Utils.calculate_normalized_entropy(list(top_logprobs.values()))
+        
+            tasks = []
+    
+            # Loop through each sampled token to construct partial strings and calculate embeddings
+            for sampled_token in top_logprobs:
+                tokens_to_use = choice['logprobs']['tokens'][:i] + [sampled_token]
+            
+                # Limit the number of tokens in the partial string if max_tokens is specified
+                if max_tokens is not None and len(tokens_to_use) > max_tokens:
+                    tokens_to_use = tokens_to_use[-max_tokens:]
+    
+                constructed_string = ''.join(tokens_to_use)
+                task = Utils.get_embedding(constructed_string)
+            
+                tasks.append(task)
+            
+            embeddings = await asyncio.gather(*tasks)
+        
+            cosine_distances = []
+    
+            # Calculate cosine distances between embeddings of partial strings and the full choice
+            for new_embedding in embeddings:
+                cosine_sim = cosine_similarity(new_embedding.reshape(1, -1), choice_embedding.reshape(1, -1))[0][0]
+                cosine_distances.append(1 - cosine_sim)
+            
+            mean_distance = np.mean(cosine_distances)
+        
+            return mean_distance, normalized_entropy
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise 
+
+    def decoded_tokens(string, tokenizer):
+        return [tokenizer.decode([x]) for x in tokenizer.encode(string)] 
+    
+    def scale_importance_log(importance_scores, base=None, offset=0.0, min_percentile=0, max_percentile=100, smoothing_constant=1e-10, scaling_factor=1.0, bias=0.0):
+        # Extract the importance values
+        try:
+            importance_values = np.array([score[1] for score in importance_scores])
+
+            # Apply optional percentile-based clipping
+            if min_percentile > 0 or max_percentile < 100:
+                min_val = np.percentile(importance_values, min_percentile)
+                max_val = np.percentile(importance_values, max_percentile)
+                importance_values = np.clip(importance_values, min_val, max_val)
+
+            # Subtract the minimum value and add the optional offset
+            importance_values = importance_values - np.min(importance_values) + offset
+
+            # Add smoothing constant to ensure non-zero values
+            importance_values += smoothing_constant
+
+            # Apply logarithmic scaling, with an optional base
+            scaled_values = np.log(importance_values) if base is None else np.log(importance_values) / np.log(base)
+
+            # Apply scaling factor and bias
+            scaled_values = scaling_factor * scaled_values + bias
+
+            # Normalize to the range [0, 1]
+            scaled_values = (scaled_values - np.min(scaled_values)) / (np.max(scaled_values) - np.min(scaled_values))
+
+            # Pair the scaled values with the original tokens
+            scaled_importance_scores = [(token, scaled_value) for token, scaled_value in zip([score[0] for score in importance_scores], scaled_values)]
+
+            return scaled_importance_scores
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise 
+    
+    @retry
+    async def get_embedding(input_text):
+        try:
+            response = Utils.client.embeddings.create(
+                input = input_text,
+                model= "text-embedding-ada-002",
+                timeout= 4.0
+            )
+            return np.array(response.data[0].embedding)
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise 
+        
+    async def approximate_importance(perturbed_text, original_embedding, model=None, tokenizer=None):
+        try:
+            perturbed_embedding = await Utils.get_embedding(perturbed_text)
+            cosine_dist = 1 - cosine_similarity(original_embedding.reshape(1, -1), perturbed_embedding.reshape(1, -1))[0][0]
+            return cosine_dist
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise 
+
+    async def ablated_relative_importance(input_text, tokenizer, model=None,):
+        try:
+            original_embedding = await Utils.get_embedding(input_text)
+            tokens = Utils.decoded_tokens(input_text, tokenizer)
+            importance_scores = []
+            
+            with tqdm(total=len(tokens), desc="Calculating Token Importances", position=0, leave=True) as progress:
+                for i in range(len(tokens)):
+                    if len(tokens[i]) < 4:
+                        continue
+                    perturbed_text = "".join(tokens[:i] + tokens[i+1:])
+                    importance = await Utils.approximate_importance(perturbed_text, original_embedding, model, tokenizer)
+                    importance_scores.append((tokens[i], importance))
+                    progress.update(1)
+                    
+            return importance_scores
+        except Exception as e:
+            log.error(e,exc_info=True)
+            raise
+
+    def get_price_details(model: str):
+        '''
+        Returns price per tokens of the model.
+
+        Parameters:
+        model (str): Model name (Ex: gpt-4)
+        '''
+        prompt_price_per_1000_tokens = {
+            "gpt-4o": 0.0050,
+            "gpt-35-turbo": 0.0005,
+            "gpt-35-turbo-instruct": 0.0015,
+            "gpt4": 0.0300
+        }
+        
+        response_price_per_1000_tokens = {
+            "gpt-4o": 0.0150,
+            "gpt-35-turbo": 0.0015,
+            "gpt-35-turbo-instruct": 0.0020,
+            "gpt4": 0.0600
+        }
+
+        try:
+            return prompt_price_per_1000_tokens[model], response_price_per_1000_tokens[model]
+        except KeyError:
+            raise ValueError(f"Model '{model}' is not found in the pricing details. Only gpt-4o, gpt-35-turbo, gpt-35-turbo-instruct & gpt4 are available. Please contact administrator")
+        
+    def get_token_cost(input_tokens: int, output_tokens: int, model: str):
+        '''
+        Calculates the total cost for tokens.
+
+        Parameters:
+        tokens (int): Total token (Prompt tokens + Completion tokens)
+        model (str): Model name (Ex: gpt4)
+        '''
+
+        # Example pricing (this should be replaced with actual pricing from Azure documentation)
+        prompt_price_per_1000_tokens, response_price_per_1000_tokens = Utils.get_price_details(model)
+
+        # Calculate cost
+        total_cost = ((input_tokens / 1000) * prompt_price_per_1000_tokens) + ((output_tokens / 1000) * response_price_per_1000_tokens)
+
+        return {
+            "total_cost": total_cost
+        }