Added README text GitHub repo data retreival

functions/github.py

@@ -12,6 +12,7 @@ from pathlib import Path
 from datetime import datetime
 
 import requests
+import base64
 
 # pylint: disable=broad-exception-caught
 
@@ -36,7 +37,8 @@ def get_github_repositories(username: str) -> list:
                 "forks": 2,
                 "updated_at": "2024-01-01T00:00:00Z",
                 "html_url": "https://github.com/user/repo",
-                "topics": ["python", "api"]
+                "topics": ["python", "api"],
+                "readme": "# Project Title\n\nProject description..."
             }
         ]
     """
@@ -185,6 +187,14 @@ def _process_repository_data(repos: List[Dict]) -> List[Dict]:
                 "size": repo.get("size", 0)
             }
 
+            # Get README content for the repository
+            repo_url = repo.get("html_url", "")
+            if repo_url:
+                readme_content = get_repository_readme(repo_url)
+                processed_repo["readme"] = readme_content
+            else:
+                processed_repo["readme"] = ""
+
             processed.append(processed_repo)
 
         except Exception as e:
@@ -192,3 +202,89 @@ def _process_repository_data(repos: List[Dict]) -> List[Dict]:
             continue
 
     return processed
+
+
+def get_repository_readme(repo_url: str) -> str:
+    """
+    Get the fulltext content of a repository's README file.
+    
+    Args:
+        repo_url (str): GitHub repository URL (e.g., "https://github.com/owner/repo")
+    
+    Returns:
+        str: README file content as text, or empty string if not found/error
+    
+    Example:
+        >>> readme_content = get_repository_readme("https://github.com/owner/repo")
+        >>> print(readme_content[:100])
+        # My Project
+        
+        This is a sample project that does...
+    """
+    
+    logger = logging.getLogger(f'{__name__}.get_repository_readme')
+    
+    try:
+        # Extract owner and repo name from URL
+        if not repo_url.startswith("https://github.com/"):
+            logger.error("Invalid GitHub URL format: %s", repo_url)
+            return ""
+            
+        # Remove trailing slash and split
+        repo_url = repo_url.rstrip("/")
+        parts = repo_url.replace("https://github.com/", "").split("/")
+        
+        if len(parts) != 2:
+            logger.error("Invalid GitHub URL format, expected owner/repo: %s", repo_url)
+            return ""
+            
+        owner, repo = parts
+        
+        logger.info("Fetching README for repository: %s/%s", owner, repo)
+        
+        # GitHub API endpoint for README
+        api_url = f"https://api.github.com/repos/{owner}/{repo}/readme"
+        
+        headers = {
+            "Accept": "application/vnd.github.v3+json",
+            "User-Agent": "Resumate-App/1.0"
+        }
+        
+        response = requests.get(api_url, headers=headers, timeout=10)
+        
+        if response.status_code == 404:
+            logger.info("No README file found for repository: %s/%s", owner, repo)
+            return ""
+            
+        if response.status_code != 200:
+            logger.error("GitHub API error fetching README: %s", response.status_code)
+            return ""
+            
+        readme_data = response.json()
+        
+        # README content is base64 encoded
+        if "content" not in readme_data:
+            logger.error("README API response missing content field")
+            return ""
+            
+        # Decode base64 content
+        encoded_content = readme_data["content"]
+        # Remove any whitespace/newlines from base64 string
+        encoded_content = encoded_content.replace("\n", "").replace(" ", "")
+        
+        try:
+            decoded_content = base64.b64decode(encoded_content).decode('utf-8')
+            logger.info("Successfully retrieved README content (%d characters)", len(decoded_content))
+            return decoded_content
+            
+        except Exception as decode_error:
+            logger.error("Error decoding README content: %s", str(decode_error))
+            return ""
+            
+    except requests.RequestException as e:
+        logger.error("Network error fetching README: %s", str(e))
+        return ""
+        
+    except Exception as e:
+        logger.error("Error retrieving README: %s", str(e))
+        return ""

tests/test_github.py

@@ -5,6 +5,7 @@ Unit tests for the github module.
 import unittest
 from unittest.mock import patch, MagicMock
 import requests
+import base64
 from functions import github
 
 # pylint: disable=protected-access
@@ -44,7 +45,8 @@ class TestGetGitHubRepositories(unittest.TestCase):
                 "created_at": "2024-01-01T00:00:00Z",
                 "html_url": "https://github.com/user/test-repo",
                 "topics": ["python", "test"],
-                "size": 100
+                "size": 100,
+                "readme": "# Test Repository\n\nThis is a test README."
             }
         ]
 
@@ -222,8 +224,11 @@ class TestGetUserRepositories(unittest.TestCase):
 class TestProcessRepositoryData(unittest.TestCase):
     """Test cases for the _process_repository_data function."""
 
-    def test_basic_processing(self):
+    @patch('functions.github.get_repository_readme')
+    def test_basic_processing(self, mock_get_readme):
         """Test basic repository data processing."""
+        mock_get_readme.return_value = "# Test Repository\n\nThis is a test README."
+        
         raw_repos = [
             {
                 "name": "test-repo",
@@ -254,24 +259,34 @@ class TestProcessRepositoryData(unittest.TestCase):
         self.assertEqual(processed_repo["html_url"], "https://github.com/user/test-repo")
         self.assertEqual(processed_repo["topics"], ["python", "test"])
         self.assertEqual(processed_repo["size"], 100)
+        self.assertEqual(processed_repo["readme"], "# Test Repository\n\nThis is a test README.")
+        
+        # Verify README was fetched
+        mock_get_readme.assert_called_once_with("https://github.com/user/test-repo")
 
-    def test_fork_filtering(self):
+    @patch('functions.github.get_repository_readme')
+    def test_fork_filtering(self, mock_get_readme):
         """Test filtering of unmodified forks."""
+        mock_get_readme.return_value = "# Repository README"
+        
         raw_repos = [
             {
                 "name": "original-repo",
                 "fork": False,
-                "stargazers_count": 5
+                "stargazers_count": 5,
+                "html_url": "https://github.com/user/original-repo"
             },
             {
                 "name": "unmodified-fork",
                 "fork": True,
-                "stargazers_count": 0
+                "stargazers_count": 0,
+                "html_url": "https://github.com/user/unmodified-fork"
             },
             {
                 "name": "modified-fork",
                 "fork": True,
-                "stargazers_count": 3
+                "stargazers_count": 3,
+                "html_url": "https://github.com/user/modified-fork"
             }
         ]
 
@@ -283,9 +298,15 @@ class TestProcessRepositoryData(unittest.TestCase):
         self.assertIn("original-repo", repo_names)
         self.assertIn("modified-fork", repo_names)
         self.assertNotIn("unmodified-fork", repo_names)
+        
+        # Verify README was fetched for included repos only
+        self.assertEqual(mock_get_readme.call_count, 2)
 
-    def test_missing_fields(self):
+    @patch('functions.github.get_repository_readme')
+    def test_missing_fields(self, mock_get_readme):
         """Test handling of missing fields in repository data."""
+        mock_get_readme.return_value = ""
+        
         raw_repos = [
             {
                 "name": "minimal-repo"
@@ -307,20 +328,29 @@ class TestProcessRepositoryData(unittest.TestCase):
         self.assertEqual(processed_repo["html_url"], "")
         self.assertEqual(processed_repo["topics"], [])
         self.assertEqual(processed_repo["size"], 0)
+        self.assertEqual(processed_repo["readme"], "")
+        
+        # Verify README function was NOT called since there's no URL
+        mock_get_readme.assert_not_called()
 
-    def test_processing_error_handling(self):
+    @patch('functions.github.get_repository_readme')
+    def test_processing_error_handling(self, mock_get_readme):
         """Test handling of processing errors for individual repos."""
+        mock_get_readme.return_value = "README content"
+        
         # Create a repo dict that will cause an error during processing
         raw_repos = [
             {
                 "name": "good-repo",
-                "stargazers_count": 5
+                "stargazers_count": 5,
+                "html_url": "https://github.com/user/good-repo"
             },
             # This will cause an AttributeError when trying to call .get() on None
             None,
             {
-                "name": "another-good-repo",
-                "stargazers_count": 3
+                "name": "another-good-repo", 
+                "stargazers_count": 3,
+                "html_url": "https://github.com/user/another-good-repo"
             }
         ]
 
@@ -332,11 +362,34 @@ class TestProcessRepositoryData(unittest.TestCase):
             with self.assertRaises(AttributeError):
                 github._process_repository_data(raw_repos)
 
-    def test_empty_repository_list(self):
+    @patch('functions.github.get_repository_readme')
+    def test_empty_repository_list(self, mock_get_readme):
         """Test processing of empty repository list."""
         result = github._process_repository_data([])
 
         self.assertEqual(result, [])
+        # Verify no README calls were made
+        mock_get_readme.assert_not_called()
+
+    @patch('functions.github.get_repository_readme')
+    def test_readme_retrieval_error_handling(self, mock_get_readme):
+        """Test handling when README retrieval fails."""
+        # Simulate README function returning empty string (error case)
+        mock_get_readme.return_value = ""
+        
+        raw_repos = [
+            {
+                "name": "test-repo",
+                "html_url": "https://github.com/user/test-repo",
+                "stargazers_count": 5
+            }
+        ]
+
+        result = github._process_repository_data(raw_repos)
+
+        self.assertEqual(len(result), 1)
+        self.assertEqual(result[0]["readme"], "")
+        mock_get_readme.assert_called_once_with("https://github.com/user/test-repo")
 
     def test_all_forks_filtered(self):
         """Test when all repositories are unmodified forks."""
@@ -358,5 +411,167 @@ class TestProcessRepositoryData(unittest.TestCase):
         self.assertEqual(result, [])
 
 
+class TestGetRepositoryReadme(unittest.TestCase):
+    """Test cases for the get_repository_readme function."""
+
+    @patch('requests.get')
+    def test_successful_readme_retrieval(self, mock_get):
+        """Test successful README file retrieval."""
+        readme_content = "# Test Repository\n\nThis is a test README file."
+        encoded_content = base64.b64encode(readme_content.encode('utf-8')).decode('ascii')
+        
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "content": encoded_content,
+            "encoding": "base64"
+        }
+        mock_get.return_value = mock_response
+
+        result = github.get_repository_readme("https://github.com/owner/repo")
+
+        self.assertEqual(result, readme_content)
+        mock_get.assert_called_once()
+        call_args = mock_get.call_args
+        self.assertIn("https://api.github.com/repos/owner/repo/readme", call_args[0][0])
+        self.assertEqual(call_args[1]["headers"]["User-Agent"], "Resumate-App/1.0")
+
+    @patch('requests.get')
+    def test_readme_not_found(self, mock_get):
+        """Test handling when README file doesn't exist."""
+        mock_response = MagicMock()
+        mock_response.status_code = 404
+        mock_get.return_value = mock_response
+
+        result = github.get_repository_readme("https://github.com/owner/repo")
+
+        self.assertEqual(result, "")
+
+    @patch('requests.get')
+    def test_api_error(self, mock_get):
+        """Test handling of API errors."""
+        mock_response = MagicMock()
+        mock_response.status_code = 500
+        mock_get.return_value = mock_response
+
+        result = github.get_repository_readme("https://github.com/owner/repo")
+
+        self.assertEqual(result, "")
+
+    @patch('requests.get')
+    def test_network_error(self, mock_get):
+        """Test handling of network errors."""
+        mock_get.side_effect = requests.RequestException("Connection error")
+
+        result = github.get_repository_readme("https://github.com/owner/repo")
+
+        self.assertEqual(result, "")
+
+    def test_invalid_url_format(self):
+        """Test handling of invalid URL formats."""
+        invalid_urls = [
+            "https://gitlab.com/owner/repo",
+            "https://github.com/owner",
+            "https://github.com/owner/repo/extra/path",
+            "not-a-url",
+            "",
+            "https://github.com/"
+        ]
+
+        for url in invalid_urls:
+            with self.subTest(url=url):
+                result = github.get_repository_readme(url)
+                self.assertEqual(result, "")
+
+    @patch('requests.get')
+    def test_missing_content_field(self, mock_get):
+        """Test handling when API response is missing content field."""
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "encoding": "base64"
+            # Missing "content" field
+        }
+        mock_get.return_value = mock_response
+
+        result = github.get_repository_readme("https://github.com/owner/repo")
+
+        self.assertEqual(result, "")
+
+    @patch('requests.get')
+    def test_invalid_base64_content(self, mock_get):
+        """Test handling of invalid base64 content."""
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "content": "invalid-base64-content!@#$",
+            "encoding": "base64"
+        }
+        mock_get.return_value = mock_response
+
+        result = github.get_repository_readme("https://github.com/owner/repo")
+
+        self.assertEqual(result, "")
+
+    @patch('requests.get')
+    def test_unicode_readme_content(self, mock_get):
+        """Test handling of README with Unicode characters."""
+        readme_content = "# Test 🚀\n\nEmoji and unicode: 中文 русский"
+        encoded_content = base64.b64encode(readme_content.encode('utf-8')).decode('ascii')
+        
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "content": encoded_content,
+            "encoding": "base64"
+        }
+        mock_get.return_value = mock_response
+
+        result = github.get_repository_readme("https://github.com/owner/repo")
+
+        self.assertEqual(result, readme_content)
+
+    @patch('requests.get')
+    def test_large_readme_content(self, mock_get):
+        """Test handling of large README files."""
+        # Create a large README content
+        readme_content = "# Large README\n\n" + "This is a line of content.\n" * 1000
+        encoded_content = base64.b64encode(readme_content.encode('utf-8')).decode('ascii')
+        
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "content": encoded_content,
+            "encoding": "base64"
+        }
+        mock_get.return_value = mock_response
+
+        result = github.get_repository_readme("https://github.com/owner/repo")
+
+        self.assertEqual(result, readme_content)
+        self.assertGreater(len(result), 10000)  # Verify it's actually large
+
+    @patch('requests.get')
+    def test_url_with_trailing_slash(self, mock_get):
+        """Test handling of URLs with trailing slash."""
+        readme_content = "# Test README"
+        encoded_content = base64.b64encode(readme_content.encode('utf-8')).decode('ascii')
+        
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "content": encoded_content,
+            "encoding": "base64"
+        }
+        mock_get.return_value = mock_response
+
+        result = github.get_repository_readme("https://github.com/owner/repo/")
+
+        self.assertEqual(result, readme_content)
+        # Verify the API call used the correct URL without trailing slash
+        call_args = mock_get.call_args
+        self.assertIn("https://api.github.com/repos/owner/repo/readme", call_args[0][0])
+
+
 if __name__ == '__main__':
     unittest.main()