app.py

import streamlit as st
import os
from dotenv import load_dotenv
# from langchain.document_loaders import GithubFileLoader
from langchain_community.document_loaders import GithubFileLoader
# from langchain.embeddings import HuggingFaceEmbeddings
from langchain_huggingface import HuggingFaceEmbeddings
from langchain_community.vectorstores import FAISS
from langchain_text_splitters import CharacterTextSplitter
from github import Github
from github import Auth

load_dotenv()

#get the GITHUB_ACCESS_TOKEN from the .env file
GITHUB_ACCESS_TOKEN = os.getenv("GITHUB_ACCESS_TOKEN")
GITHUB_BASE_URL = "https://github.com/"

# initialize Github
auth = Auth.Token(GITHUB_ACCESS_TOKEN)
g = Github(auth=auth)


@st.cache_resource
def get_hugging_face_model():
  model_name = "mchochlov/codebert-base-cd-ft"
  hf = HuggingFaceEmbeddings(model_name=model_name)
  return hf

def get_similar_files(query, db, embeddings):
  docs_and_scores = db.similarity_search_with_score(query)
  return docs_and_scores

def fetch_repos(username):
  print(f"Fetching repositories for user: {username}")
  try:
    user = g.get_user(username)
    print(f"User: {user}")
    return [repo.name for repo in user.get_repos()]
  except Exception as e:
    st.error(f"Error fetching repositories: {e}")
    return []

def get_file_contributors(repo_name, file_path):
    try:
        repo = g.get_repo(f"{USER}/{repo_name}")
        commits = repo.get_commits(path=file_path)
        contributors = {}
        for commit in commits:
            author = commit.author.login if commit.author else "Unknown"
            if author in contributors:
                contributors[author] += 1
            else:
                contributors[author] = 1
        return contributors
    except Exception as e:
        st.error(f"Error fetching contributors: {e}")
        return {}

# Initialize session state for repositories
if "repos" not in st.session_state:
    st.session_state.repos = []

# STREAMLIT INTERFACE
st.title("Find Similar Code")

st.markdown("This app takes a code sample you provide, and finds similar code in a Github repository.")
st.markdown("This functionality could ideally be implemented across multiple repos to allow you to find helpful examples of how to implement the code you are working on writing, or identify other code contributors who could help you resolve your issues")

USER = st.text_input("Enter the Github User", value = "Satttoshi")

fetch_repos_button = st.button("Fetch Repositories")

if fetch_repos_button:
    st.session_state.repos = fetch_repos(USER)


REPO = st.selectbox("Select a Github Repository", options=st.session_state.repos)


FILE_TYPES_TO_LOAD = st.multiselect("Select File Types", [".py", ".ts",".js",".css",".html"], default = [".ts"])

text_input = st.text_area("Enter a Code Example", value =
"""

""", height = 330
)

find_similar_code_button = st.button("Find Similar Code")

if find_similar_code_button:
  print(f"Searching for similar code in {USER}/{REPO}")
  loader = GithubFileLoader(
    repo=f"{USER}/{REPO}",
    access_token=GITHUB_ACCESS_TOKEN,
    github_api_url="https://api.github.com",
    file_filter=lambda file_path: file_path.endswith(
      tuple(FILE_TYPES_TO_LOAD)
    )
  )
  documents = loader.load()
  text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
  docs = text_splitter.split_documents(documents)
  embedding_vector = get_hugging_face_model()
  db = FAISS.from_documents(docs, embedding_vector)
  query = text_input
  results_with_scores = get_similar_files(query, db, embedding_vector)
  results_with_scores = results_with_scores[:5] #limit to 5 results
  for doc, score in results_with_scores:
    #print all metadata info in the doc.metadata dictionary
    # for key, value in doc.metadata.items():
    #     print(f"{key}: {value}")

    path = doc.metadata['path']
    content = doc.page_content
    score = round(float(score), 2)
    contributors = get_file_contributors(REPO, path)
    print(f"Path: {doc.metadata['path']}, Score: {score}, Contributors: {contributors}")
    file_link = f"{GITHUB_BASE_URL}{USER}/{REPO}/blob/main/{path}"
    st.markdown(f"[{path}]({file_link})")
    for contributor, count in contributors.items():
        st.write(f"* Contributor: [{contributor}](https://github.com/{contributor}), Commits: {count}")

else:
  st.info("Please Submit a Code Sample to Find Similar Code")

#https://github.com/heaversm/gdrive-docker/blob/main/gdrive/provider/__init__.py
#https://github.com/heaversm/gdrive-docker/blob/main/gdrive/provider/__init__.py