Added token, git repo download

app.py

@@ -3,30 +3,40 @@
 # Import necessary modules from lib
 from lib.code_reviewer import CodeReviewer, ReviewManager
 import os
+import glob
+
+# Variables for GitHub repository details
+GITHUB_REPO_URL = "https://github.com/vsagar100/ansible_tf_provisioner"
+GITHUB_BRANCH = "main"  # Specify the branch or tag to download
+GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")  # Specify your GitHub token
 
 def main():
     # Directory structure setup
-    # Directory for storing input Ansible files
-    input_directory = "input_files"
+    # Directory for storing downloaded GitHub repository
+    download_directory = "downloaded_repo"
     # Directory for storing output JSON reviews
     output_directory = "output_reviews"
 
     # Ensure the directories exist
-    os.makedirs(input_directory, exist_ok=True)
+    os.makedirs(download_directory, exist_ok=True)
     os.makedirs(output_directory, exist_ok=True)
 
-    # Paths for testing
-    sample_files = [os.path.join(input_directory, "example1.yml"), os.path.join(input_directory, "example2.yml")]
-    output_json_path = os.path.join(output_directory, "code_review_results.json")
-
     # Initialize the code reviewer and review manager
     code_reviewer = CodeReviewer()
     review_manager = ReviewManager(reviewer=code_reviewer)
 
+    # Download GitHub repository
+    review_manager.download_repo(GITHUB_REPO_URL, GITHUB_BRANCH, GITHUB_TOKEN, download_directory)
+
+    # Find all YAML files in the downloaded repository
+    yaml_files = glob.glob(os.path.join(download_directory, "**", "*.yml"), recursive=True)
+    yaml_files += glob.glob(os.path.join(download_directory, "**", "*.yaml"), recursive=True)
+
     # Process files and generate reviews
-    reviews = review_manager.process_files(sample_files)
+    reviews = review_manager.process_files(yaml_files)
 
     # Save reviews to JSON
+    output_json_path = os.path.join(output_directory, "code_review_results.json")
     review_manager.save_reviews_to_json(reviews, output_json_path)
     print(f"Reviews saved to {output_json_path}")
 

lib/code_reviewer.py

@@ -5,6 +5,9 @@ import os
 import json
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer
+import requests
+import zipfile
+import io
 
 # Custom Imports
 from typing import List, Dict
@@ -30,7 +33,7 @@ class CodeReviewer:
         Returns:
             Dict: The code standards in dictionary form.
         """
-        standards_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "standards", "ansible_ode_standards.json")
+        standards_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "standards", "code_standards.json")
         with open(standards_path, 'r') as f:
             return json.load(f)
 
@@ -79,6 +82,25 @@ class ReviewManager:
         """
         self.reviewer = reviewer
 
+    def download_repo(self, repo_url: str, branch: str, token: str, download_path: str):
+        """
+        Downloads a GitHub repository as a ZIP file and extracts it.
+        
+        Args:
+            repo_url (str): The GitHub repository URL.
+            branch (str): The branch or tag to download.
+            token (str): The GitHub personal access token for authentication.
+            download_path (str): The path to extract the downloaded repository.
+        """
+        zip_url = f"{repo_url}/archive/refs/heads/{branch}.zip"
+        headers = {"Authorization": f"token {token}"}
+        response = requests.get(zip_url, headers=headers)
+        if response.status_code == 200:
+            with zipfile.ZipFile(io.BytesIO(response.content)) as zip_ref:
+                zip_ref.extractall(download_path)
+        else:
+            raise Exception(f"Failed to download repository. Status code: {response.status_code}")
+
     def process_files(self, file_paths: List[str]) -> List[Dict[str, str]]:
         """
         Processes multiple files for review.