Add tests for Go, CSS, and various programming languages

- Introduced a new test suite for Go language support, validating chunking functionality for simple code, structs, interfaces, and goroutines.
- Added test cases for CSS chunking, covering both media queries and simple styles.
- Established a base test class for code chunking tests, allowing for consistent setup and utility methods across different language tests.
- Created individual test files for JavaScript, PHP, Python, Ruby, TypeScript, and CSS, ensuring comprehensive coverage of chunking functionality across multiple languages.

mock_codefiles.json

@@ -15,7 +15,7 @@
   "main.js": "const express = require('express');\nconst routes = require('./routes');\n\n// Create the Express application\nconst app = express();\n\n// Register the routes from the routes.js file\napp.use('/', routes);\n\n// Configuration settings for the app can go here\napp.set('port', process.env.PORT || 3000);\n\n// More complex app initialization steps can be added here\n// For example, database initialization, middleware setups, etc.\n\n// This function can be used to create a database schema\nfunction createDatabase() {\n  // Code to create database schema\n  console.log('Created Database!');\n}\n\n// Optionally, call database creation or other setup functions here\ncreateDatabase();\n\n// Start the server\napp.listen(app.get('port'), () => {\n  console.log(`Server running on port ${app.get('port')}`);\n});",
   "utilities.js": "// Example of utility functions for common tasks\n\n// Function to hash a password\nfunction hashPassword(password) {\n  const salt = uuidv4();\n  return sha256(salt + password) + ':' + salt;\n}\n\n// Function to check a hashed password\nfunction checkPassword(hashedPassword, userPassword) {\n  const [password, salt] = hashedPassword.split(':');\n  return sha256(salt + userPassword) === password;\n}\n\n// Function to validate an email address\nfunction validateEmail(email) {\n  const pattern = /^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$/;\n  return pattern.test(email);\n}\n\n// Function to generate a token expiration date\nfunction generateExpirationDate(days = 1) {\n  const expirationDate = new Date();\n  expirationDate.setDate(expirationDate.getDate() + days);\n  return expirationDate;\n}\n\n// Function to convert a string to a Date object\nfunction stringToDate(dateString, format = 'YYYY-MM-DD HH:mm:ss') {\n  return new Date(dateString);\n}\n\n// Function to convert a Date object to a string\nfunction dateToString(date, format = 'YYYY-MM-DD HH:mm:ss') {\n  return date.toISOString();\n}",
   "services.js": "// Example of service functions for handling data operations\n\n// Function to fetch data from an external API\nasync function fetchExternalData(apiUrl) {\n  try {\n    const response = await fetch(apiUrl);\n    if (response.ok) {\n      return await response.json();\n    } else {\n      return { error: 'Failed to fetch data' };\n    }\n  } catch (error) {\n    return { error: error.message };\n  }\n}\n\n// Function to save user data to the database\nasync function saveUserData(name, email) {\n  try {\n    const newUser = new UserData({ name, email });\n    await newUser.save();\n    return { message: 'User saved successfully' };\n  } catch (error) {\n    return { error: error.message };\n  }\n}\n\n// Function to update user data in the database\nasync function updateUserData(userId, name, email) {\n  try {\n    const user = await UserData.findById(userId);\n    if (!user) {\n      return { error: 'User not found' };\n    }\n    if (name) {\n      user.name = name;\n    }\n    if (email) {\n      user.email = email;\n    }\n    await user.save();\n    return { message: 'User updated successfully' };\n  } catch (error) {\n    return { error: error.message };\n  }\n}\n\n// Function to delete user data from the database\nasync function deleteUserData(userId) {\n  try {\n    const user = await UserData.findById(userId);\n    if (!user) {\n      return { error: 'User not found' };\n    }\n    await user.remove();\n    return { message: 'User deleted successfully' };\n  } catch (error) {\n    return { error: error.message };\n  }\n}",
-  "react_component.js": "import React from 'react';\n\nimport './SearchResults.css';\n\nimport TrackList from '../TrackList/TrackList.js';\n\n constructor(props) {\n        super(props);\n        this.addTopFive = this.addTopFive.bind(this);\n        this.addTopTen = this.addTopTen.bind(this);\n        this.addAll = this.addAll.bind(this);\n    }\n\n    //add the top five tracks to the playlist\n    addTopFive() {\n        this.props.onAdd(this.props.searchResults.slice(0, 5));\n    }\n\n    //add top 10 tracks to the playlist\n    addTopTen() {\n        this.props.onAdd(this.props.searchResults.slice(0, 10));\n    }\n\n    addAll() {\n        this.props.onAdd(this.props.searchResults);\n    }\n    render() {\n    return (\n      <div className=\"SearchResults\">\n        <h2>Results</h2>\n        <TrackList tracks={this.props.searchResults} onAdd={this.props.onAdd} onToggle={this.props.onToggle}  currentTrack={this.props.currentTrack}/>\n      </div>\n    );\n  }\n}\n\nexport default SearchResults;'",
+  "react_component.js": "import React from 'react';\n\nimport './SearchResults.css';\n\nimport TrackList from '../TrackList/TrackList.js';\n\n constructor(props) {\n        super(props);\n        this.addTopFive = this.addTopFive.bind(this);\n        this.addTopTen = this.addTopTen.bind(this);\n        this.addAll = this.addAll.bind(this);\n    }\n\n    //add the top five tracks to the playlist\n    addTopFive() {\n        this.props.onAdd(this.props.searchResults.slice(0, 5));\n    }\n\n    //add top 10 tracks to the playlist\n    addTopTen() {\n        this.props.onAdd(this.props.searchResults.slice(0, 10));\n    }\n\n    addAll() {\n        this.props.onAdd(this.props.searchResults);\n    }\n    render() {\n    return (\n      <div className=\"SearchResults\">\n        <h2>Results</h2>\n        <TrackList tracks={this.props.searchResults} onAdd={this.props.onAdd} onToggle={this.props.onToggle}  currentTrack={this.props.currentTrack}/>\n      </div>\n    );\n  }\n}\n\nexport default SearchResults;",
   "simple_styles.css": "/* Example of CSS styles for a web page */\n\nbody {\n  font-family: Arial, sans-serif;\n  background-color: #f4f4f4;\n  margin: 0;\n  padding: 0;\n}\n\nh1 {\n  color: #333;\n  text-align: center;\n}\n\nbutton {\n  padding: 10px 20px;\n  font-size: 16px;\n  background-color: #007bff;\n  color: #fff;\n  border: none;\n  cursor: pointer;\n}\n\nbutton:hover {\n  background-color: #0056b3;\n}",
   "media_queries.css": "/* Example of CSS styles with media queries for responsive design */\n\nbody {\n  font-family: Arial, sans-serif;\n  background-color: #f4f4f4;\n  margin: 0;\n  padding: 0;\n}\n\nh1 {\n  color: #333;\n  text-align: center;\n}\n\nbutton {\n  padding: 10px 20px;\n  font-size: 16px;\n  background-color: #007bff;\n  color: #fff;\n  border: none;\n  cursor: pointer;\n}\n\nbutton:hover {\n  background-color: #0056b3;\n}\n\n/* Media query for smaller screens */\n@media (max-width: 768px) {\n  button {\n    padding: 8px 16px;\n    font-size: 14px;\n  }\n}",
   "single_syntax_error_example.py": "# This is a sample Python file\n\nprint('Hello, world!'\n\n",
@@ -26,5 +26,9 @@
   "multiple_syntax_errors.css": "body {\n  font-family: Arial, sans-serif;\n  background-color: #f4f4f4;\n  margin: 0;\n  padding: 0;\n}\n\nh1 {\n  color: #333;\n  text-align: center;\n}\n\nbutton {\n  padding: 10px 20px;\n  font-size: 16px;\n  background-color: #007bff;\n  color: #fff;\n  border: none;\n  cursor: pointer;\n  :hover {\n  background-color: #0056b3;\n}\n\n/* Media query for smaller screens */\n@media (max-width: 768px) {\n  button {\n    padding: 8px 16px;\n    font-size: 14px;\n  }\n}",
   "example.ts": "interface User {\n  id: number;\n  name: string;\n  email: string;\n}\n\nclass UserManager {\n  private users: User[] = [];\n\n  addUser(user: User): void {\n    this.users.push(user);\n  }\n\n  getUser(id: number): User | undefined {\n    return this.users.find(user => user.id === id);\n  }\n\n  updateUser(id: number, updatedUser: Partial<User>): void {\n    const userIndex = this.users.findIndex(user => user.id === id);\n    if (userIndex !== -1) {\n      this.users[userIndex] = { ...this.users[userIndex], ...updatedUser };\n    }\n  }\n\n  deleteUser(id: number): void {\n    this.users = this.users.filter(user => user.id !== id);\n  }\n}\n\nconst userManager = new UserManager();\nuserManager.addUser({ id: 1, name: 'John Doe', email: '[email protected]' });\nconsole.log(userManager.getUser(1));",
   "example.rb": "class User\n  attr_accessor :id, :name, :email\n\n  def initialize(id, name, email)\n    @id = id\n    @name = name\n    @email = email\n  end\n\n  def to_s\n    \"User: #{@name} (#{@email})\"\n  end\nend\n\nclass UserManager\n  def initialize\n    @users = []\n  end\n\n  def add_user(user)\n    @users << user\n  end\n\n  def get_user(id)\n    @users.find { |user| user.id == id }\n  end\n\n  def update_user(id, updated_user)\n    user = get_user(id)\n    user.name = updated_user.name if updated_user.name\n    user.email = updated_user.email if updated_user.email\n  end\n\n  def delete_user(id)\n    @users.delete_if { |user| user.id == id }\n  end\nend\n\nuser_manager = UserManager.new\nuser_manager.add_user(User.new(1, 'John Doe', '[email protected]'))\nputs user_manager.get_user(1)",
-  "example.php": "<?php\n\nclass User {\n    public $id;\n    public $name;\n    public $email;\n\n    public function __construct($id, $name, $email) {\n        $this->id = $id;\n        $this->name = $name;\n        $this->email = $email;\n    }\n}\n\nclass UserManager {\n    private $users = [];\n\n    public function addUser($user) {\n        $this->users[] = $user;\n    }\n\n    public function getUser($id) {\n        foreach ($this->users as $user) {\n            if ($user->id === $id) {\n                return $user;\n            }\n        }\n        return null;\n    }\n\n    public function updateUser($id, $updatedUser) {\n        foreach ($this->users as &$user) {\n            if ($user->id === $id) {\n                $user->name = $updatedUser->name ?? $user->name;\n                $user->email = $updatedUser->email ?? $user->email;\n                break;\n            }\n        }\n    }\n\n    public function deleteUser($id) {\n        $this->users = array_filter($this->users, function($user) use ($id) {\n            return $user->id !== $id;\n        });\n    }\n}\n\n$userManager = new UserManager();\n$userManager->addUser(new User(1, 'John Doe', '[email protected]'));\nvar_dump($userManager->getUser(1));\n?>"
+  "example.php": "<?php\n\nclass User {\n    public $id;\n    public $name;\n    public $email;\n\n    public function __construct($id, $name, $email) {\n        $this->id = $id;\n        $this->name = $name;\n        $this->email = $email;\n    }\n}\n\nclass UserManager {\n    private $users = [];\n\n    public function addUser($user) {\n        $this->users[] = $user;\n    }\n\n    public function getUser($id) {\n        foreach ($this->users as $user) {\n            if ($user->id === $id) {\n                return $user;\n            }\n        }\n        return null;\n    }\n\n    public function updateUser($id, $updatedUser) {\n        foreach ($this->users as &$user) {\n            if ($user->id === $id) {\n                $user->name = $updatedUser->name ?? $user->name;\n                $user->email = $updatedUser->email ?? $user->email;\n                break;\n            }\n        }\n    }\n\n    public function deleteUser($id) {\n        $this->users = array_filter($this->users, function($user) use ($id) {\n            return $user->id !== $id;\n        });\n    }\n}\n\n$userManager = new UserManager();\n$userManager->addUser(new User(1, 'John Doe', '[email protected]'));\nvar_dump($userManager->getUser(1));\n?>",
+  "simple.go": "package main\n\nimport \"fmt\"\n\nfunc main() {\n    fmt.Println(\"Hello, Go!\")\n    \n    // Simple variable declaration\n    message := \"Welcome to Go programming\"\n    fmt.Println(message)\n    \n    // Simple conditionals\n    x := 10\n    if x > 5 {\n        fmt.Println(\"x is greater than 5\")\n    } else {\n        fmt.Println(\"x is not greater than 5\")\n    }\n}",
+  "structs.go": "package main\n\nimport \"fmt\"\n\n// Person represents information about a person\ntype Person struct {\n    Name    string\n    Age     int\n    Address string\n}\n\n// Employee extends Person with job-related fields\ntype Employee struct {\n    Person\n    Title    string\n    Salary   float64\n    EmployeeID int\n}\n\nfunc main() {\n    // Create a new person\n    person := Person{\n        Name:    \"John Doe\",\n        Age:     30,\n        Address: \"123 Main St\",\n    }\n    \n    // Create a new employee\n    employee := Employee{\n        Person: Person{\n            Name:    \"Jane Smith\",\n            Age:     28,\n            Address: \"456 Oak Ave\",\n        },\n        Title:    \"Software Engineer\",\n        Salary:   85000.0,\n        EmployeeID: 12345,\n    }\n    \n    fmt.Printf(\"Person: %+v\\n\", person)\n    fmt.Printf(\"Employee: %+v\\n\", employee)\n}",
+  "interfaces.go": "package main\n\nimport (\n    \"fmt\"\n    \"math\"\n)\n\n// Shape interface defines methods for calculating area and perimeter\ntype Shape interface {\n    Area() float64\n    Perimeter() float64\n}\n\n// Circle implements the Shape interface\ntype Circle struct {\n    Radius float64\n}\n\nfunc (c Circle) Area() float64 {\n    return math.Pi * c.Radius * c.Radius\n}\n\nfunc (c Circle) Perimeter() float64 {\n    return 2 * math.Pi * c.Radius\n}\n\n// Rectangle implements the Shape interface\ntype Rectangle struct {\n    Width, Height float64\n}\n\nfunc (r Rectangle) Area() float64 {\n    return r.Width * r.Height\n}\n\nfunc (r Rectangle) Perimeter() float64 {\n    return 2 * (r.Width + r.Height)\n}\n\n// PrintShapeInfo prints information about any shape\nfunc PrintShapeInfo(s Shape) {\n    fmt.Printf(\"Area: %.2f\\n\", s.Area())\n    fmt.Printf(\"Perimeter: %.2f\\n\", s.Perimeter())\n}\n\nfunc main() {\n    circle := Circle{Radius: 5.0}\n    rectangle := Rectangle{Width: 4.0, Height: 6.0}\n    \n    fmt.Println(\"Circle:\")\n    PrintShapeInfo(circle)\n    \n    fmt.Println(\"\\nRectangle:\")\n    PrintShapeInfo(rectangle)\n}",
+  "goroutines.go": "package main\n\nimport (\n    \"fmt\"\n    \"sync\"\n    \"time\"\n)\n\n// Simple goroutine example\nfunc worker(id int, wg *sync.WaitGroup) {\n    defer wg.Done()\n    \n    fmt.Printf(\"Worker %d starting\\n\", id)\n    time.Sleep(time.Second)\n    fmt.Printf(\"Worker %d done\\n\", id)\n}\n\n// Goroutine with channel communication\nfunc sender(ch chan<- int, count int) {\n    for i := 0; i < count; i++ {\n        ch <- i\n        fmt.Printf(\"Sent: %d\\n\", i)\n        time.Sleep(time.Millisecond * 100)\n    }\n    close(ch)\n}\n\nfunc receiver(ch <-chan int, done chan<- bool) {\n    for num := range ch {\n        fmt.Printf(\"Received: %d\\n\", num)\n    }\n    done <- true\n}\n\nfunc main() {\n    // Using wait groups\n    var wg sync.WaitGroup\n    \n    for i := 1; i <= 3; i++ {\n        wg.Add(1)\n        go worker(i, &wg)\n    }\n    \n    // Wait for all goroutines to complete\n    wg.Wait()\n    \n    // Using channels\n    numbers := make(chan int, 5)\n    done := make(chan bool)\n    \n    go sender(numbers, 10)\n    go receiver(numbers, done)\n    \n    // Wait for receiver to process all numbers\n    <-done\n    \n    fmt.Println(\"All operations completed\")\n}"
 }

test_code_chunker.py

@@ -278,5 +278,57 @@ class TestCodeChunkerPHP(unittest.TestCase):
         self.assertIn(php_code, final_code)
         self.assertGreater(len(chunks), 1)  # Ensure the code is actually chunked
 
+# Golang Test Class
+class TestCodeChunkerGolang(unittest.TestCase):
+
+    def setUp(self):
+        self.patcher = patch('utils.count_tokens', side_effect=mock_count_tokens)
+        self.mock_count_tokens = self.patcher.start()
+        self.code_chunker = CodeChunker(file_extension='go')
+        self.mock_codebase = load_json('mock_codefiles.json')
+
+    def tearDown(self):
+        self.patcher.stop()
+
+    def test_chunk_golang_simple_code(self):
+        go_code = self.mock_codebase['simple.go']
+        chunks = self.code_chunker.chunk(go_code, token_limit=20)
+        Chunker.print_chunks(chunks)
+        final_code = Chunker.consolidate_chunks_into_file(chunks)
+        num_lines = Chunker.count_lines(final_code)
+        self.assertEqual(num_lines, len(go_code.split("\n")))
+        self.assertIn(go_code, final_code)
+        self.assertGreater(len(chunks), 1)  # Ensure the code is actually chunked
+
+    def test_chunk_golang_with_structs(self):
+        go_code = self.mock_codebase['structs.go']
+        chunks = self.code_chunker.chunk(go_code, token_limit=20)
+        Chunker.print_chunks(chunks)
+        final_code = Chunker.consolidate_chunks_into_file(chunks)
+        num_lines = Chunker.count_lines(final_code)
+        self.assertEqual(num_lines, len(go_code.split("\n")))
+        self.assertIn(go_code, final_code)
+        self.assertGreater(len(chunks), 1)
+
+    def test_chunk_golang_with_interfaces(self):
+        go_code = self.mock_codebase['interfaces.go']
+        chunks = self.code_chunker.chunk(go_code, token_limit=20)
+        Chunker.print_chunks(chunks)
+        final_code = Chunker.consolidate_chunks_into_file(chunks)
+        num_lines = Chunker.count_lines(final_code)
+        self.assertEqual(num_lines, len(go_code.split("\n")))
+        self.assertIn(go_code, final_code)
+        self.assertGreater(len(chunks), 1)
+
+    def test_chunk_golang_with_goroutines(self):
+        go_code = self.mock_codebase['goroutines.go']
+        chunks = self.code_chunker.chunk(go_code, token_limit=20)
+        Chunker.print_chunks(chunks)
+        final_code = Chunker.consolidate_chunks_into_file(chunks)
+        num_lines = Chunker.count_lines(final_code)
+        self.assertEqual(num_lines, len(go_code.split("\n")))
+        self.assertIn(go_code, final_code)
+        self.assertGreater(len(chunks), 1)
+
 if __name__ == '__main__':
     unittest.main()

tests/__init__.py

@@ -0,0 +1 @@
+# This file makes the tests directory a Python package 

tests/test_base.py

@@ -0,0 +1,42 @@
+import unittest
+from unittest.mock import patch
+import tiktoken
+import sys
+import os
+
+# Add parent directory to path to allow imports
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from Chunker import Chunker, CodeChunker
+from utils import load_json
+
+# Mocking the count_tokens function as it's external and not the focus of these tests
+def mock_count_tokens(string: str, encoding_name='gpt-4') -> int:
+    """Returns the number of tokens in a text string."""
+    encoding = tiktoken.encoding_for_model(encoding_name)
+    num_tokens = len(encoding.encode(string))
+    return num_tokens
+
+class BaseChunkerTest(unittest.TestCase):
+    """Base class for all code chunker tests with common setup and utilities."""
+    
+    def setUp(self):
+        self.patcher = patch('utils.count_tokens', side_effect=mock_count_tokens)
+        self.mock_count_tokens = self.patcher.start()
+        self.mock_codebase = load_json('mock_codefiles.json')
+        
+    def tearDown(self):
+        self.patcher.stop()
+    
+    def run_chunker_test(self, code, token_limit=20):
+        """Helper method to run standard chunker tests."""
+        chunks = self.code_chunker.chunk(code, token_limit=token_limit)
+        Chunker.print_chunks(chunks)
+        final_code = Chunker.consolidate_chunks_into_file(chunks)
+        num_lines = Chunker.count_lines(final_code)
+        
+        # Common assertions
+        self.assertEqual(num_lines, len(code.split("\n")))
+        self.assertIn(code, final_code)
+        
+        return chunks, final_code 

tests/test_css.py

@@ -0,0 +1,19 @@
+import unittest
+from test_base import BaseChunkerTest
+from Chunker import CodeChunker
+
+class TestCodeChunkerCSS(BaseChunkerTest):
+    def setUp(self):
+        super().setUp()
+        self.code_chunker = CodeChunker(file_extension='css')
+
+    def test_chunk_css_with_media_query(self):
+        css_code = self.mock_codebase['media_queries.css']
+        self.run_chunker_test(css_code)
+
+    def test_chunk_css_with_simple_css(self):
+        css_code = self.mock_codebase['simple_styles.css']
+        self.run_chunker_test(css_code)
+
+if __name__ == '__main__':
+    unittest.main() 

tests/test_golang.py

@@ -0,0 +1,31 @@
+import unittest
+from test_base import BaseChunkerTest
+from Chunker import CodeChunker
+
+class TestCodeChunkerGolang(BaseChunkerTest):
+    def setUp(self):
+        super().setUp()
+        self.code_chunker = CodeChunker(file_extension='go')
+
+    def test_chunk_golang_simple_code(self):
+        go_code = self.mock_codebase['simple.go']
+        chunks, _ = self.run_chunker_test(go_code)
+        self.assertGreater(len(chunks), 1)  # Ensure the code is actually chunked
+
+    def test_chunk_golang_with_structs(self):
+        go_code = self.mock_codebase['structs.go']
+        chunks, _ = self.run_chunker_test(go_code)
+        self.assertGreater(len(chunks), 1)
+
+    def test_chunk_golang_with_interfaces(self):
+        go_code = self.mock_codebase['interfaces.go']
+        chunks, _ = self.run_chunker_test(go_code)
+        self.assertGreater(len(chunks), 1)
+
+    def test_chunk_golang_with_goroutines(self):
+        go_code = self.mock_codebase['goroutines.go']
+        chunks, _ = self.run_chunker_test(go_code)
+        self.assertGreater(len(chunks), 1)
+
+if __name__ == '__main__':
+    unittest.main() 

tests/test_javascript.py

@@ -0,0 +1,39 @@
+import unittest
+from test_base import BaseChunkerTest
+from Chunker import CodeChunker
+
+class TestCodeChunkerJavaScript(BaseChunkerTest):
+    def setUp(self):
+        super().setUp()
+        self.code_chunker = CodeChunker(file_extension='js')
+
+    def test_chunk_javascript_simple_code(self):
+        js_code = self.mock_codebase['simple.js']
+        self.run_chunker_test(js_code)
+
+    def test_chunk_javascript_with_routes(self):
+        js_code = self.mock_codebase['routes.js']
+        self.run_chunker_test(js_code)
+
+    def test_chunk_javascript_with_models(self):
+        js_code = self.mock_codebase['models.js']
+        self.run_chunker_test(js_code)
+
+    def test_chunk_javascript_with_main(self):
+        js_code = self.mock_codebase['main.js']
+        self.run_chunker_test(js_code)
+
+    def test_chunk_javascript_with_utilities(self):
+        js_code = self.mock_codebase['utilities.js']
+        self.run_chunker_test(js_code)
+
+    def test_chunk_javascript_with_big_class(self):
+        js_code = self.mock_codebase['big_class.js']
+        self.run_chunker_test(js_code)
+
+    def test_chunk_javascript_with_react_component(self):
+        js_code = self.mock_codebase['react_component.js']
+        self.run_chunker_test(js_code)
+
+if __name__ == '__main__':
+    unittest.main() 

tests/test_php.py

@@ -0,0 +1,16 @@
+import unittest
+from test_base import BaseChunkerTest
+from Chunker import CodeChunker
+
+class TestCodeChunkerPHP(BaseChunkerTest):
+    def setUp(self):
+        super().setUp()
+        self.code_chunker = CodeChunker(file_extension='php')
+
+    def test_chunk_php_code(self):
+        php_code = self.mock_codebase['example.php']
+        chunks, _ = self.run_chunker_test(php_code)
+        self.assertGreater(len(chunks), 1)  # Ensure the code is actually chunked
+
+if __name__ == '__main__':
+    unittest.main() 

tests/test_python.py

@@ -0,0 +1,53 @@
+import unittest
+from test_base import BaseChunkerTest, mock_count_tokens
+from Chunker import CodeChunker
+
+class TestCodeChunkerPython(BaseChunkerTest):
+    def setUp(self):
+        super().setUp()
+        self.code_chunker = CodeChunker(file_extension='py')
+    
+    def test_chunk_simple_code(self):
+        py_code = self.mock_codebase['simple.py']
+        first_chunk_token_limit = mock_count_tokens("import sys")
+        print(f"first_chunk_token_limit = {first_chunk_token_limit}")
+        chunks = self.code_chunker.chunk(py_code, token_limit=25)
+        token_count = self.mock_count_tokens(py_code)
+        print(f"token_count = {token_count}")
+        print(f"original code:\n {py_code}")
+        
+        chunks, full_code = self.run_chunker_test(py_code, token_limit=25)
+        
+        self.assertEqual(len(chunks), 2) # There should be 2 chunks
+        self.assertIn("import sys", chunks[1]) # The first chunk should contain the import statement
+        self.assertIn("print('Hello, world!')", chunks[2]) # The second chunk should contain the print statement
+
+    def test_chunk_code_text_only(self):
+        py_code = self.mock_codebase['text_only.py']
+        chunks, final_code = self.run_chunker_test(py_code)
+        
+        self.assertEqual(len(chunks), 1)
+        self.assertIn("This file is empty and should test the chunker's ability to handle empty files", chunks[1])
+
+    def test_chunk_code_with_routes(self):
+        py_code = self.mock_codebase['routes.py']
+        self.run_chunker_test(py_code)
+
+    def test_chunk_code_with_models(self):
+        py_code = self.mock_codebase['models.py']
+        self.run_chunker_test(py_code)
+
+    def test_chunk_code_with_main(self):
+        py_code = self.mock_codebase['main.py']
+        self.run_chunker_test(py_code)
+
+    def test_chunk_code_with_utilities(self):
+        py_code = self.mock_codebase['utilities.py']
+        self.run_chunker_test(py_code)
+
+    def test_chunk_code_with_big_class(self):
+        py_code = self.mock_codebase['big_class.py']
+        self.run_chunker_test(py_code)
+
+if __name__ == '__main__':
+    unittest.main() 

tests/test_ruby.py

@@ -0,0 +1,16 @@
+import unittest
+from test_base import BaseChunkerTest
+from Chunker import CodeChunker
+
+class TestCodeChunkerRuby(BaseChunkerTest):
+    def setUp(self):
+        super().setUp()
+        self.code_chunker = CodeChunker(file_extension='rb')
+
+    def test_chunk_ruby_code(self):
+        rb_code = self.mock_codebase['example.rb']
+        chunks, _ = self.run_chunker_test(rb_code)
+        self.assertGreater(len(chunks), 1)  # Ensure the code is actually chunked
+
+if __name__ == '__main__':
+    unittest.main() 

tests/test_typescript.py

@@ -0,0 +1,16 @@
+import unittest
+from test_base import BaseChunkerTest
+from Chunker import CodeChunker
+
+class TestCodeChunkerTypeScript(BaseChunkerTest):
+    def setUp(self):
+        super().setUp()
+        self.code_chunker = CodeChunker(file_extension='ts')
+
+    def test_chunk_typescript_code(self):
+        ts_code = self.mock_codebase['example.ts']
+        chunks, _ = self.run_chunker_test(ts_code)
+        self.assertGreater(len(chunks), 1)  # Ensure the code is actually chunked
+
+if __name__ == '__main__':
+    unittest.main()