Updates to UI and backend (#9)

backend/agent/agent.py

@@ -3,7 +3,7 @@
 import openai
 import json
 import os
-from typing import Any, List, Optional, Callable
+from typing import List, Optional, Callable
 from pydantic import BaseModel
 from database.my_codebase import MyCodebase
 
@@ -180,20 +180,33 @@ def add_line_numbers_to_content(self, content: str) -> str:
             lines[i] = f"{i+1} {lines[i]}"
         return "\n".join(lines)
 
-    def process_json(self, args):
+    def process_json(self, args: str) -> str:
+        """
+        Process a JSON string, handling any triple-quoted strings within it.
+
+        Args:
+            args (str): The JSON string to process.
+
+        Returns:
+            str: The processed JSON string.
+        """
         try:
+            # Attempt to load the JSON string
             response = json.loads(args)
             return response
         except json.decoder.JSONDecodeError:
+            # If there's a JSONDecodeError, it may be due to triple-quoted strings
             # Find all occurrences of triple-quoted strings
             triple_quoted_strings = re.findall(r"\"\"\"(.*?)\"\"\"", args, re.DOTALL)
 
             # For each occurrence, replace newlines and triple quotes
             for tqs in triple_quoted_strings:
+                # Replace newlines and double quotes within the triple-quoted string
                 fixed_string = tqs.replace("\n", "\\n").replace('"', '\\"')
+                # Replace the original triple-quoted string with the fixed string
                 response_str = args.replace(tqs, fixed_string)
 
             # Now replace the triple quotes with single quotes
             response_str = args.replace('"""', '"')
 
-            return response_str
+            return json.loads(response_str)

backend/agent/agent_functions/changes.py

@@ -1,12 +1,10 @@
 import os
-import openai
 import difflib as dl
 
 from dotenv import load_dotenv
 from typing import List, Optional, Tuple
 from pydantic import Field, field_validator
 from openai_function_call import OpenAISchema
-import re
 
 load_dotenv()
 DIRECTORY = os.getenv("PROJECT_DIRECTORY")
@@ -16,6 +14,7 @@ class Change(OpenAISchema):
     """
     The correct changes to make to a file.
     Be mindful of formatting and include the proper newline characters.
+    Spaces and indentation in the new code should be relative to the old code.
 
     Args:
         original (str): The full code block to be replaced; formatted as a string.
@@ -43,23 +42,23 @@ def to_dict(self) -> dict:
 
 class Changes(OpenAISchema):
     """
-    A list of changes to make to a file.
-    Think step by step and ensure the changes cover all requests. Changes will be processed similar to a diff
-    of the format:
+    A list of changes similar to the format:
+
     >>>>>> ORIGINAL
     old code
     =========
     new code
     <<<<<< UPDATED
 
-    All you need to provide is the old code and new code. The system will handle the rest.
-    The 'original' field cannot be an empty string.
-    When adding new code, include original code to determine where to put the new code.
-    The 'updated' field can be blank if you want to delete the old code.
-    Always use the relative path from the root of the codebase.
+    Please provide the old code exactly as you read it and the correct new replacement code.
+    Be mindful of formatting, number of spaces, and newline characters.
+    Your code is always valid, correct, and addresses the requests of the human.
+    When adding new code, include original code to determine where to put the new code (The original field should never be an empty string)
+    The 'updated' field can be blank when you need to delete the old code.
+    The file_name is always the relative path from the root of the codebase.
 
     Args:
-        file_name (str): The name of the file to be changed. This needs to be the relative path from the root of the codebase.
+        file_name (str): The relative path from the root of the codebase.
         thought (str): A description of your thought process.
         changes (List[Change]): A list of Change Objects that represent all the changes you want to make to this file.
     """
@@ -76,18 +75,16 @@ def apply_changes(self, changes: List[Change], content: str) -> str:
         Applies the given changes to the content.
         """
         for change in changes:
-            # Use regex to find and replace the old string
-            if content:
-                print("Content exists!!!!!!!!")
-                print(f"Type: {type(content)}")
             new_content = self.replace_part_with_missing_leading_whitespace(
                 whole_lines=content.split("\n"),
                 part_lines=change.original.splitlines(),
                 replace_lines=change.updated.splitlines(),
             )
             if not new_content:
                 print(f"Failed on change: {change.to_dict()}")
-                raise Exception("Failed to apply changes.")
+                raise Exception(
+                    f"Failed to apply changes. Original: {change.original}, Updated: {change.updated}"
+                )
 
             if new_content == content:
                 print("Warning: Expected content not found. No changes made.")
@@ -121,12 +118,17 @@ def execute(self) -> str:
         try:
             with open(file_path, "r") as f:
                 current_contents = f.read()
-                current_contents_with_line_numbers = "\n".join(
-                    [
-                        f"{i+1} {line}"
-                        for i, line in enumerate(current_contents.splitlines())
-                    ]
-                )
+                # current_contents_with_line_numbers = "\n".join(
+                #     [
+                #         f"{i+1} {line}"
+                #         for i, line in enumerate(current_contents.splitlines())
+                #     ]
+                # )
+        except PermissionError:
+            print(
+                f"Error: Permission denied for file {self.file_name} at file_path: {file_path}\nDirectory: {DIRECTORY}"
+            )
+            return "Permission denied. Please check the file permissions."
         except FileNotFoundError:
             print(
                 f"Error: File {self.file_name} not found at file_path: {file_path}\nDirectory: {DIRECTORY}"
@@ -136,7 +138,11 @@ def execute(self) -> str:
             print(e)
             return "Error: File could not be read. Please try again."
 
-        new_text = self.apply_changes(self.changes, current_contents)
+        try:
+            new_text = self.apply_changes(self.changes, current_contents)
+        except Exception as e:
+            print(e)
+            return f"Error: Changes could not be applied. Error: {e}."
 
         # if not new_text:
         # TODO: This could be a point to retry.
@@ -149,8 +155,16 @@ def execute(self) -> str:
             tofile="b",
             n=0,
         )
+        try:
+            self.save(new_text, file_path)
+        except PermissionError:
+            print(
+                f"Error: Permission denied for file {self.file_name} at file_path: {file_path}\nDirectory: {DIRECTORY}"
+            )
+            return (
+                "Permission denied when saving file. Please check the file permissions."
+            )
 
-        self.save(new_text, file_path)
         return "\n\n```diff\n" + "\n".join(str(d) for d in diff) + "\n```\n\n"
 
     def save(self, new_text: str, file_path: str) -> None:

backend/agent/agent_functions/shell_commands.py

@@ -1,23 +1,18 @@
 import os
-import openai
 import pexpect
 
 from dotenv import load_dotenv
 from enum import Enum
 from typing import Optional, List, Generator
 from pydantic import Field, field_validator
 from openai_function_call import OpenAISchema
-from agent.agent_functions.changes import Changes
-from agent.agent_functions.new_file import NewFile
 
 load_dotenv()
 DIRECTORY = os.getenv("PROJECT_DIRECTORY")
 
 
 class CommandType(Enum):
     BASH_COMMAND = "bash"
-    FILE_CHANGE = "file_change"
-    NEW_FILE = "new_file"
 
 
 class CommandResult(OpenAISchema):
@@ -30,41 +25,18 @@ class CommandResults(OpenAISchema):
 
 
 class Command(OpenAISchema):
+    """
+    A command to be executed by the agent. Available command types are:
+    - Bash Command: Execute a bash command in the shell.
+    """
+
     id: int = Field(..., description="Unique id of the command")
     command_type: CommandType = Field(..., description="Type of the command")
     dependent_commands: List[int] = Field(
         default_factory=list,
         description="List of the IDs of commands that need to be completed before this command can be executed.",
     )
     command_line: Optional[str] = Field(None, description="Command to execute")
-    file_change: Optional[Changes] = Field(
-        None, description="File name and changes you would like to request"
-    )
-    new_file: Optional[NewFile] = Field(
-        None,
-        description="""File name (path from root directory), description, and reference
-        files (path from root directory) which provide additional context to the AI.""",
-    )
-
-    @field_validator("file_change")
-    def check_file_change(cls, v, values):
-        if (
-            "command_type" in values
-            and values["command_type"] == CommandType.FILE_CHANGE
-            and v is None
-        ):
-            raise ValueError("file_change is required when command_type is FILE_CHANGE")
-        return v
-
-    @field_validator("new_file")
-    def check_new_file(cls, v, values):
-        if (
-            "command_type" in values
-            and values["command_type"] == CommandType.NEW_FILE
-            and v is None
-        ):
-            raise ValueError("new_file is required when command_type is FILE_CHANGE")
-        return v
 
     @field_validator("command_line")
     def check_command(cls, v, values):
@@ -77,11 +49,6 @@ def check_command(cls, v, values):
         return v
 
     def execute(self, with_results: CommandResults) -> CommandResult:
-        # If a program is set for this command and the command type is PROGRAM, execute the program
-        if self.command_type == CommandType.FILE_CHANGE:
-            self.file_change.save()
-            output = "changes complete"
-
         if self.command_type == CommandType.NEW_FILE:
             self.new_file.save()
             output = "file created"

backend/database/my_codebase.py

@@ -1,6 +1,5 @@
 import os
 import datetime
-import subprocess
 from dotenv import load_dotenv
 import openai
 import numpy as np
@@ -22,26 +21,14 @@
 SUMMARY_MODEL = "gpt-3.5-turbo"
 README_MODEL = "gpt-4"
 SUMMARY_PROMPT = """
-Please summarise the following what the following code is doing.
+Please summarise, in bullet points, what the following code is doing.
 Please be consise and include all the important informastion.\n\n
 CODE:{}
 SUMMARY:
 """
 
 
-def get_git_root(path: str = ".") -> Union[str, None]:
-    try:
-        root = (
-            subprocess.check_output(["git", "rev-parse", "--show-toplevel"], cwd=path)
-            .decode("utf-8")
-            .strip()
-        )
-        return root
-    except Exception as e:
-        print(e)
-        return None
-
-
+# new comment
 class MyCodebase:
     load_dotenv()
     IGNORE_DIRS = os.getenv("IGNORE_DIRS")
@@ -234,7 +221,7 @@ def tree(self) -> str:
         ]
 
         # Insert each file into the tree structure
-        for file_path in file_paths:
+        for file_path in sorted(file_paths):
             parts = file_path.split(os.path.sep)
             # Find the start_from directory in the path and trim up to it
             if start_from in parts:
@@ -263,7 +250,7 @@ def remove_old_files(self) -> None:
         self.cur.execute("SELECT file_path FROM files")
         file_paths = [result[0] for result in self.cur.fetchall()]
         for file_path in file_paths:
-            if not os.path.exists(file_path) and file_path in self.file_dict.keys():
+            if not os.path.exists(file_path):
                 self.cur.execute(
                     sql.SQL(
                         """
@@ -292,4 +279,4 @@ def _is_valid_file(file_name):
             not file_name.startswith(".")
             and not file_name.startswith("_")
             and any(file_name.endswith(ext) for ext in MyCodebase.FILE_EXTENSIONS)
-        )
+        ) or file_name == "Dockerfile"

backend/main.py

@@ -5,8 +5,7 @@
 import tiktoken
 from fastapi import Request
 from fastapi.responses import JSONResponse, StreamingResponse
-from database.my_codebase import get_git_root
-from app_setup import setup_app, app, DIRECTORY
+from app_setup import setup_app, app
 
 
 ENCODER = tiktoken.encoding_for_model("gpt-3.5-turbo")
@@ -23,7 +22,7 @@ def stream():
         for content in AGENT.query(**data):
             if content is not None:
                 accumulated_messages[id] += content
-                yield json.dumps({"id": id, "content": content}) + "\n"
+                yield json.dumps({"id": id, "content": content}) + "@@"
 
         AGENT.memory_manager.add_message("assistant", accumulated_messages[id])
 
@@ -92,7 +91,7 @@ async def get_summaries(reset: bool | None = None):
     cur = CODEBASE.conn.cursor()
     cur.execute("SELECT DISTINCT file_path, summary, token_count FROM files")
     results = cur.fetchall()
-    root_path = get_git_root(CODEBASE.directory)
+    root_path = CODEBASE.directory
     result = [
         {
             "file_path": os.path.relpath(file_path, root_path),
@@ -117,10 +116,7 @@ async def set_summary_files_in_prompt(input: dict):
     if "files" not in input:
         return JSONResponse(status_code=400, content={"error": "missing files"})
 
-    files = [
-        os.path.join(get_git_root(CODEBASE.directory), file)
-        for file in input.get("files")
-    ]
+    files = [os.path.join(CODEBASE.directory, file) for file in input.get("files")]
     summaries = CODEBASE.get_summaries()
     summaries = [f"{k}:\n{v}" for k, v in summaries.items() if k in files]
     additional_system_prompt_summaries = "\n\n".join(summaries)

backend/tests/test_agent_functions.py

@@ -3,8 +3,11 @@
 import json
 import os
 import openai
-from app_setup import DIRECTORY
 from agent.agent_functions.changes import Changes
+from dotenv import load_dotenv
+
+load_dotenv()
+DIRECTORY = os.getenv("PROJECT_DIRECTORY")
 
 temp_file = "backend/tests/test_files/agent_function_test1.py"
 TEST_FILE = "backend/tests/test_files/app_setup_test.py"