gpt-engineer/gpt_engineer/applications/cli/learning.py

"""
This module provides tools and data structures for supporting a feedback loop in the GPT Engineer application.

The primary intent of this module is to gather feedback from the user on the output of the gpt-engineer tool,
with their consent, and to store this feedback for further analysis and improvement of the tool.

Classes:
----------
Review:
    Represents user's review of the generated code.
Learning:
    Represents the metadata and feedback collected for a session in which gpt-engineer was used.

Functions:
----------
human_review_input() -> Review:
    Interactively gathers feedback from the user regarding the performance of generated code.

check_consent() -> bool:
    Checks if the user has previously given consent to store their data and if not, asks for it.

collect_consent() -> bool:
    Verifies if the user has given consent to store their data or prompts for it.

ask_if_can_store() -> bool:
    Asks the user if it's permissible to store their data for gpt-engineer improvement.

logs_to_string(steps: List[Step], logs: DB) -> str:
    Converts logs of steps into a readable string format.

extract_learning(model: str, temperature: float, steps: List[Step], dbs: DBs, steps_file_hash) -> Learning:
    Extracts feedback and session details to create a Learning instance.

get_session() -> str:
    Retrieves a unique identifier for the current user session.

Constants:
----------
TERM_CHOICES:
    Terminal color choices for user interactive prompts.
"""

import json
import random
import tempfile

from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Optional, Tuple

from dataclasses_json import dataclass_json
from termcolor import colored

from gpt_engineer.core.default.disk_memory import DiskMemory


@dataclass_json
@dataclass
class Review:
    ran: Optional[bool]
    perfect: Optional[bool]
    works: Optional[bool]
    comments: str
    raw: str


@dataclass_json
@dataclass
class Learning:
    prompt: str
    model: str
    temperature: float
    config: str
    logs: str
    session: str
    review: Optional[Review]
    timestamp: str = field(default_factory=lambda: datetime.utcnow().isoformat())
    version: str = "0.3"


TERM_CHOICES = (
    colored("y", "green")
    + "/"
    + colored("n", "red")
    + "/"
    + colored("u", "yellow")
    + "(ncertain): "
)


def human_review_input() -> Optional[Review]:
    """
    Ask the user to review the generated code and return their review.

    Returns
    -------
    Review
        The user's review of the generated code.
    """
    print()
    if not check_collection_consent():
        return None
    print()
    print(
        colored("To help gpt-engineer learn, please answer 3 questions:", "light_green")
    )
    print()

    ran = input("Did the generated code run at all? " + TERM_CHOICES)
    while ran not in ("y", "n", "u"):
        ran = input("Invalid input. Please enter y, n, or u: ")

    perfect = ""
    useful = ""

    if ran == "y":
        perfect = input(
            "Did the generated code do everything you wanted? " + TERM_CHOICES
        )
        while perfect not in ("y", "n", "u"):
            perfect = input("Invalid input. Please enter y, n, or u: ")

        if perfect != "y":
            useful = input("Did the generated code do anything useful? " + TERM_CHOICES)
            while useful not in ("y", "n", "u"):
                useful = input("Invalid input. Please enter y, n, or u: ")

    comments = ""
    if perfect != "y":
        comments = input(
            "If you have time, please explain what was not working "
            + colored("(ok to leave blank)\n", "light_green")
        )

    return Review(
        raw=", ".join([ran, perfect, useful]),
        ran={"y": True, "n": False, "u": None, "": None}[ran],
        works={"y": True, "n": False, "u": None, "": None}[useful],
        perfect={"y": True, "n": False, "u": None, "": None}[perfect],
        comments=comments,
    )


def check_collection_consent() -> bool:
    """
    Check if the user has given consent to store their data.
    If not, ask for their consent.
    """
    path = Path(".gpte_consent")
    if path.exists() and path.read_text() == "true":
        return True
    else:
        return ask_collection_consent()


def ask_collection_consent() -> bool:
    """
    Ask the user for consent to store their data.
    """
    answer = input(
        "Is it ok if we store your prompts to help improve GPT Engineer? (y/n)"
    )
    while answer.lower() not in ("y", "n"):
        answer = input("Invalid input. Please enter y or n: ")

    if answer.lower() == "y":
        path = Path(".gpte_consent")
        path.write_text("true")
        print(colored("Thank you️", "light_green"))
        print()
        print(
            "(If you no longer wish to participate in data collection, delete the file .gpte_consent)"
        )
        return True
    else:
        print(
            colored(
                "No worries! GPT Engineer will not collect your prompts. ❤️",
                "light_green",
            )
        )
        return False


def extract_learning(
    prompt: str,
    model: str,
    temperature: float,
    config: Tuple[str, ...],
    memory: DiskMemory,
    review: Review,
) -> Learning:
    """
    Extract the learning data from the steps and databases.

    Parameters
    ----------
    model : str
        The name of the model used.
    temperature : float
        The temperature used.
    steps : List[Step]
        The list of steps.
    dbs : DBs
        The databases containing the input, logs, memory, and workspace.
    steps_file_hash : str
        The hash of the steps file.

    Returns
    -------
    Learning
        The extracted learning data.
    """
    learning = Learning(
        prompt=prompt,
        model=model,
        temperature=temperature,
        config=json.dumps(config),
        session=get_session(),
        logs=memory.to_json(),
        review=review,
    )
    return learning


def get_session() -> str:
    """
    Returns a unique user id for the current user project (session).

    Returns
    -------
    str
        The unique user id.
    """
    path = Path(tempfile.gettempdir()) / "gpt_engineer_user_id.txt"

    try:
        if path.exists():
            user_id = path.read_text()
        else:
            # random uuid:
            user_id = str(random.randint(0, 2**32))
            path.write_text(user_id)
        return user_id
    except IOError:
        return "ephemeral_" + str(random.randint(0, 2**32))