diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..175a2b0d8d --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +vsc-dl-x64 export-ignore diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..5751cf9938 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,4 @@ +# (2025-04-04) + + + diff --git a/core/agents/bug_hunter.py b/core/agents/bug_hunter.py index 837ef7bc0a..e6bc0554f5 100644 --- a/core/agents/bug_hunter.py +++ b/core/agents/bug_hunter.py @@ -9,6 +9,15 @@ from core.agents.convo import AgentConvo from core.agents.mixins import ChatWithBreakdownMixin, TestSteps from core.agents.response import AgentResponse from core.config import CHECK_LOGS_AGENT_NAME, magic_words +from core.config.actions import ( + BH_ADDITIONAL_FEEDBACK, + BH_HUMAN_TEST_AGAIN, + BH_IS_BUG_FIXED, + BH_START_BUG_HUNT, + BH_START_USER_TEST, + BH_STARTING_PAIR_PROGRAMMING, + BH_WAIT_BUG_REP_INSTRUCTIONS, +) from core.config.constants import CONVO_ITERATIONS_LIMIT from core.db.models.project_state import IterationStatus from core.llm.parser import JSONParser @@ -45,12 +54,6 @@ class ImportantLogsForDebugging(BaseModel): logs: list[ImportantLog] = Field(description="Important logs that will help the human debug the current bug.") -BH_STARTING_PAIR_PROGRAMMING = "Start pair programming for task #{}" -BH_START_USER_TEST = "Start user testing for task #{}" -BH_WAIT_BUG_REP_INSTRUCTIONS = "Awaiting bug reproduction instructions for task #{}" -BH_START_BUG_HUNT = "Start bug hunt for task #{}" - - class BugHunter(ChatWithBreakdownMixin, BaseAgent): agent_type = "bug-hunter" display_name = "Bug Hunter" @@ -78,6 +81,7 @@ class BugHunter(ChatWithBreakdownMixin, BaseAgent): async def get_bug_reproduction_instructions(self): await self.send_message("Finding a way to reproduce the bug ...") + await self.ui.start_important_stream() llm = self.get_llm() convo = ( AgentConvo(self) @@ -161,7 +165,7 @@ class BugHunter(ChatWithBreakdownMixin, BaseAgent): await self.ui.send_run_command(self.current_state.run_command) await self.ask_question( - "Please test the app again.", + BH_HUMAN_TEST_AGAIN, buttons={"done": "I am done testing"}, buttons_only=True, default="continue", @@ -172,7 +176,7 @@ class BugHunter(ChatWithBreakdownMixin, BaseAgent): if awaiting_user_test: buttons = {"yes": "Yes, the issue is fixed", "no": "No", "start_pair_programming": "Start Pair Programming"} user_feedback = await self.ask_question( - "Is the bug you reported fixed now?", + BH_IS_BUG_FIXED, buttons=buttons, default="yes", buttons_only=True, @@ -200,7 +204,7 @@ class BugHunter(ChatWithBreakdownMixin, BaseAgent): } ) user_feedback = await self.ask_question( - "Please add any additional feedback that could help Pythagora solve this bug", + BH_ADDITIONAL_FEEDBACK, buttons=buttons, default="continue", extra_info="collect_logs", diff --git a/core/agents/code_monkey.py b/core/agents/code_monkey.py index bb7c2771c9..3fa4021f6e 100644 --- a/core/agents/code_monkey.py +++ b/core/agents/code_monkey.py @@ -2,7 +2,7 @@ import asyncio import re from difflib import unified_diff from enum import Enum -from typing import Optional, Union +from typing import Optional from pydantic import BaseModel, Field @@ -11,6 +11,7 @@ from core.agents.convo import AgentConvo from core.agents.mixins import FileDiffMixin from core.agents.response import AgentResponse, ResponseType from core.config import CODE_MONKEY_AGENT_NAME, CODE_REVIEW_AGENT_NAME, DESCRIBE_FILES_AGENT_NAME +from core.config.actions import CM_UPDATE_FILES from core.db.models import File from core.llm.parser import JSONParser, OptionalCodeBlockParser from core.log import get_logger @@ -57,9 +58,6 @@ class FileDescription(BaseModel): ) -CM_UPDATE_FILES = "Updating files" - - class CodeMonkey(FileDiffMixin, BaseAgent): agent_type = "code-monkey" display_name = "Code Monkey" @@ -69,12 +67,7 @@ class CodeMonkey(FileDiffMixin, BaseAgent): return await self.describe_files() else: data = await self.implement_changes() - code_review_done = False - while not code_review_done: - review_response = await self.run_code_review(data) - if isinstance(review_response, AgentResponse): - return review_response - data = await self.implement_changes(review_response) + return await self.accept_changes(data["path"], data["old_content"], data["new_content"]) async def implement_changes(self, data: Optional[dict] = None) -> dict: file_name = self.step["save_file"]["path"] @@ -191,33 +184,6 @@ class CodeMonkey(FileDiffMixin, BaseAgent): # CODE REVIEW # ------------------------------ - async def run_code_review(self, data: Optional[dict]) -> Union[AgentResponse, dict]: - await self.ui.send_file_status(data["path"], "reviewing", source=self.ui_source) - if ( - data is not None - and not data["old_content"] - or data["new_content"] == data["old_content"] - or data["attempt"] >= MAX_CODING_ATTEMPTS - ): - # we always auto-accept new files and unchanged files, or if we've tried too many times - return await self.accept_changes(data["path"], data["old_content"], data["new_content"]) - - approved_content, feedback = await self.review_change( - data["path"], - data["instructions"], - data["old_content"], - data["new_content"], - ) - if feedback: - return { - "new_content": data["new_content"], - "approved_content": approved_content, - "feedback": feedback, - "attempt": data["attempt"], - } - else: - return await self.accept_changes(data["path"], data["old_content"], approved_content) - async def accept_changes(self, file_path: str, old_content: str, new_content: str) -> AgentResponse: await self.ui.send_file_status(file_path, "done", source=self.ui_source) @@ -367,7 +333,7 @@ class CodeMonkey(FileDiffMixin, BaseAgent): """ Get the diff between two files. - This uses Python difflib to produce an unified diff, then splits + This uses Python difflib to produce a unified diff, then splits it into hunks that will be separately reviewed by the reviewer. :param file_name: name of the file being modified diff --git a/core/agents/developer.py b/core/agents/developer.py index 7878d9635f..b2082e4246 100644 --- a/core/agents/developer.py +++ b/core/agents/developer.py @@ -10,6 +10,14 @@ from core.agents.convo import AgentConvo from core.agents.mixins import ChatWithBreakdownMixin, RelevantFilesMixin from core.agents.response import AgentResponse from core.config import PARSE_TASK_AGENT_NAME, TASK_BREAKDOWN_AGENT_NAME +from core.config.actions import ( + DEV_EXECUTE_TASK, + DEV_TASK_BREAKDOWN, + DEV_TASK_REVIEW_FEEDBACK, + DEV_TASK_START, + DEV_TROUBLESHOOT, + DEV_WAIT_TEST, +) from core.db.models.project_state import IterationStatus, TaskStatus from core.db.models.specification import Complexity from core.llm.parser import JSONParser @@ -72,13 +80,6 @@ class TaskSteps(BaseModel): steps: list[Step] -DEV_WAIT_TEST = "Awaiting user test" -DEV_TASK_STARTING = "Starting task #{}" -DEV_TASK_BREAKDOWN = "Task #{} breakdown" -DEV_TROUBLESHOOT = "Troubleshooting #{}" -DEV_TASK_REVIEW_FEEDBACK = "Task review feedback" - - class Developer(ChatWithBreakdownMixin, RelevantFilesMixin, BaseAgent): agent_type = "developer" display_name = "Developer" @@ -214,6 +215,8 @@ class Developer(ChatWithBreakdownMixin, RelevantFilesMixin, BaseAgent): # Check which files are relevant to the current task await self.get_relevant_files_parallel() + current_task_index = self.current_state.tasks.index(current_task) + await self.send_message("Thinking about how to implement this task ...") await self.ui.start_breakdown_stream() @@ -255,6 +258,7 @@ class Developer(ChatWithBreakdownMixin, RelevantFilesMixin, BaseAgent): # There might be state leftovers from previous tasks that we need to clean here self.next_state.modified_files = {} self.set_next_steps(response, source) + self.next_state.action = DEV_TASK_START.format(current_task_index + 1) await telemetry.trace_code_event( "task-start", { @@ -322,12 +326,11 @@ class Developer(ChatWithBreakdownMixin, RelevantFilesMixin, BaseAgent): "task_index": task_index, } ) - self.next_state.action = DEV_TASK_STARTING.format(task_index) await self.send_message(f"Starting task #{task_index} with the description:\n\n" + description) if self.current_state.run_command: await self.ui.send_run_command(self.current_state.run_command) user_response = await self.ask_question( - "Do you want to execute the above task?", + DEV_EXECUTE_TASK, buttons=buttons, default="yes", buttons_only=True, diff --git a/core/agents/executor.py b/core/agents/executor.py index 24bdaa25fc..5af478a1fb 100644 --- a/core/agents/executor.py +++ b/core/agents/executor.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, Field from core.agents.base import BaseAgent from core.agents.convo import AgentConvo from core.agents.response import AgentResponse +from core.config.actions import EX_RUN_COMMAND, EX_SKIP_COMMAND, RUN_COMMAND from core.llm.parser import JSONParser from core.log import get_logger from core.proc.exec_log import ExecLog @@ -32,10 +33,6 @@ class CommandResult(BaseModel): ) -EX_SKIP_COMMAND = 'Skip "{}"' -EX_RUN_COMMAND = 'Run "{}"' - - class Executor(BaseAgent): agent_type = "executor" display_name = "Executor" @@ -82,9 +79,9 @@ class Executor(BaseAgent): timeout = options.get("timeout") if timeout: - q = f"Can I run command: {cmd} with {timeout}s timeout?" + q = f"{RUN_COMMAND} {cmd} with {timeout}s timeout?" else: - q = f"Can I run command: {cmd}?" + q = f"{RUN_COMMAND} {cmd}?" confirm = await self.ask_question( q, diff --git a/core/agents/frontend.py b/core/agents/frontend.py index 5eb7613440..7eaa279661 100644 --- a/core/agents/frontend.py +++ b/core/agents/frontend.py @@ -8,6 +8,14 @@ from core.agents.git import GitMixin from core.agents.mixins import FileDiffMixin from core.agents.response import AgentResponse from core.config import FRONTEND_AGENT_NAME, SWAGGER_EMBEDDINGS_API +from core.config.actions import ( + FE_CHANGE_REQ, + FE_CONTINUE, + FE_DONE_WITH_UI, + FE_ITERATION, + FE_ITERATION_DONE, + FE_START, +) from core.llm.parser import DescriptiveCodeBlockParser from core.log import get_logger from core.telemetry import telemetry @@ -15,12 +23,6 @@ from core.ui.base import ProjectStage log = get_logger(__name__) -FE_INIT = "Frontend init" -FE_START = "Frontend start" -FE_CONTINUE = "Frontend continue" -FE_ITERATION = "Frontend iteration" -FE_ITERATION_DONE = "Frontend iteration done" - class Frontend(FileDiffMixin, GitMixin, BaseAgent): agent_type = "frontend" @@ -125,9 +127,7 @@ class Frontend(FileDiffMixin, GitMixin, BaseAgent): await self.ui.send_project_stage({"stage": ProjectStage.ITERATE_FRONTEND}) answer = await self.ask_question( - "Do you want to change anything or report a bug?" - if frontend_only - else "Do you want to change anything or report a bug? Keep in mind that currently ONLY frontend is implemented.", + "Do you want to change anything or report a bug?" if frontend_only else FE_CHANGE_REQ, buttons={"yes": "I'm done building the UI"} if not frontend_only else None, default="yes", extra_info="restart_app/collect_logs", @@ -136,7 +136,7 @@ class Frontend(FileDiffMixin, GitMixin, BaseAgent): if answer.button == "yes": answer = await self.ask_question( - "Are you sure you're done building the UI and want to start building the backend functionality now?", + FE_DONE_WITH_UI, buttons={ "yes": "Yes, let's build the backend", "no": "No, continue working on the UI", @@ -280,6 +280,12 @@ class Frontend(FileDiffMixin, GitMixin, BaseAgent): command = f"cd client && {command}" if "run start" or "run dev" in command: continue + + # if command is cd client && some_command client/ -> won't work, we need to remove client/ after && + prefix, cmd_part = command.split("&&", 1) + cmd_part = cmd_part.strip().replace("client/", "") + command = f"{prefix} && {cmd_part}" + await self.send_message(f"Running command: `{command}`...") await self.process_manager.run_command(command) else: diff --git a/core/agents/human_input.py b/core/agents/human_input.py index 3c0ac2f4e5..415d826a1b 100644 --- a/core/agents/human_input.py +++ b/core/agents/human_input.py @@ -1,5 +1,6 @@ from core.agents.base import BaseAgent from core.agents.response import AgentResponse, ResponseType +from core.config.actions import HUMAN_INTERVENTION_QUESTION class HumanInput(BaseAgent): @@ -16,7 +17,7 @@ class HumanInput(BaseAgent): description = step["human_intervention_description"] await self.ask_question( - f"I need human intervention: {description}", + f"{HUMAN_INTERVENTION_QUESTION} {description}", buttons={"continue": "Continue"}, default="continue", buttons_only=True, diff --git a/core/agents/mixins.py b/core/agents/mixins.py index 578275901e..8a782b493e 100644 --- a/core/agents/mixins.py +++ b/core/agents/mixins.py @@ -1,13 +1,14 @@ import asyncio import json -from difflib import unified_diff from typing import List, Optional from pydantic import BaseModel, Field from core.agents.convo import AgentConvo from core.agents.response import AgentResponse +from core.cli.helpers import get_line_changes from core.config import GET_RELEVANT_FILES_AGENT_NAME, TASK_BREAKDOWN_AGENT_NAME, TROUBLESHOOTER_BUG_REPORT +from core.config.actions import MIX_BREAKDOWN_CHAT_PROMPT from core.config.constants import CONVO_ITERATIONS_LIMIT from core.llm.parser import JSONParser from core.log import get_logger @@ -56,7 +57,7 @@ class ChatWithBreakdownMixin: ) chat = await self.ask_question( - "Are you happy with the breakdown? Now is a good time to ask questions or suggest changes.", + MIX_BREAKDOWN_CHAT_PROMPT, buttons={"yes": "Yes, looks good!"}, default="yes", verbose=False, @@ -191,18 +192,4 @@ class FileDiffMixin: :return: a tuple (added_lines, deleted_lines) """ - from_lines = old_content.splitlines(keepends=True) - to_lines = new_content.splitlines(keepends=True) - - diff_gen = unified_diff(from_lines, to_lines) - - added_lines = 0 - deleted_lines = 0 - - for line in diff_gen: - if line.startswith("+") and not line.startswith("+++"): # Exclude the file headers - added_lines += 1 - elif line.startswith("-") and not line.startswith("---"): # Exclude the file headers - deleted_lines += 1 - - return added_lines, deleted_lines + return get_line_changes(old_content, new_content) diff --git a/core/agents/spec_writer.py b/core/agents/spec_writer.py index 2a6d312b8a..8d2403fa4a 100644 --- a/core/agents/spec_writer.py +++ b/core/agents/spec_writer.py @@ -2,6 +2,7 @@ from core.agents.base import BaseAgent from core.agents.convo import AgentConvo from core.agents.response import AgentResponse, ResponseType from core.config import SPEC_WRITER_AGENT_NAME +from core.config.actions import SPEC_CHANGE_FEATURE_STEP_NAME, SPEC_CHANGE_STEP_NAME, SPEC_CREATE_STEP_NAME from core.db.models import Complexity from core.db.models.project_state import IterationStatus from core.llm.parser import StringParser @@ -14,10 +15,6 @@ ANALYZE_THRESHOLD = 1500 INITIAL_PROJECT_HOWTO_URL = ( "https://github.com/Pythagora-io/gpt-pilot/wiki/How-to-write-a-good-initial-project-description" ) -SPEC_CREATE_STEP_NAME = "Create specification" -SPEC_CHANGE_STEP_NAME = "Change specification" -SPEC_CHANGE_FEATURE_STEP_NAME = "Change specification due to new feature" - log = get_logger(__name__) diff --git a/core/agents/task_completer.py b/core/agents/task_completer.py index 0b13386dcd..88c01955a6 100644 --- a/core/agents/task_completer.py +++ b/core/agents/task_completer.py @@ -1,13 +1,12 @@ from core.agents.base import BaseAgent from core.agents.git import GitMixin from core.agents.response import AgentResponse +from core.config.actions import TC_TASK_DONE from core.log import get_logger from core.telemetry import telemetry log = get_logger(__name__) -TC_TASK_DONE = "Task #{} complete" - class TaskCompleter(BaseAgent, GitMixin): agent_type = "pythagora" diff --git a/core/agents/tech_lead.py b/core/agents/tech_lead.py index b935166f03..956430cbe8 100644 --- a/core/agents/tech_lead.py +++ b/core/agents/tech_lead.py @@ -9,6 +9,13 @@ from core.agents.convo import AgentConvo from core.agents.mixins import RelevantFilesMixin from core.agents.response import AgentResponse from core.config import TECH_LEAD_EPIC_BREAKDOWN, TECH_LEAD_PLANNING +from core.config.actions import ( + TL_CREATE_INITIAL_EPIC, + TL_CREATE_PLAN, + TL_EDIT_DEV_PLAN, + TL_INITIAL_PROJECT_NAME, + TL_START_FEATURE, +) from core.db.models import Complexity from core.db.models.project_state import TaskStatus from core.llm.parser import JSONParser @@ -46,11 +53,6 @@ class EpicPlan(BaseModel): plan: list[Task] = Field(description="List of tasks that need to be done to implement the entire epic.") -TL_CREATE_INITIAL_EPIC = "Create initial project epic" -TL_CREATE_PLAN = "Create a development plan for epic: {}" -TL_START_FEATURE = "Start of feature #{}" - - class TechLead(RelevantFilesMixin, BaseAgent): agent_type = "tech-lead" display_name = "Tech Lead" @@ -94,7 +96,7 @@ class TechLead(RelevantFilesMixin, BaseAgent): self.next_state.epics = self.current_state.epics + [ { "id": uuid4().hex, - "name": "Initial Project", + "name": TL_INITIAL_PROJECT_NAME, "source": "app", "description": self.current_state.specification.description, "test_instructions": None, @@ -301,7 +303,7 @@ class TechLead(RelevantFilesMixin, BaseAgent): await self.ui.send_project_stage({"stage": ProjectStage.OPEN_PLAN}) response = await self.ask_question( - "Open and edit your development plan in the Progress tab", + TL_EDIT_DEV_PLAN, buttons={"done_editing": "I'm done editing, the plan looks good"}, default="done_editing", buttons_only=True, diff --git a/core/agents/tech_writer.py b/core/agents/tech_writer.py index 0dd97365ae..f951ccb513 100644 --- a/core/agents/tech_writer.py +++ b/core/agents/tech_writer.py @@ -1,14 +1,13 @@ from core.agents.base import BaseAgent from core.agents.convo import AgentConvo from core.agents.response import AgentResponse +from core.config.actions import TW_WRITE from core.db.models.project_state import TaskStatus from core.log import get_logger from core.ui.base import success_source log = get_logger(__name__) -TW_WRITE = "Write documentation" - class TechnicalWriter(BaseAgent): agent_type = "tech-writer" diff --git a/core/agents/troubleshooter.py b/core/agents/troubleshooter.py index c9dd51bd17..48ca70ac0d 100644 --- a/core/agents/troubleshooter.py +++ b/core/agents/troubleshooter.py @@ -9,6 +9,7 @@ from core.agents.convo import AgentConvo from core.agents.mixins import ChatWithBreakdownMixin, IterationPromptMixin, RelevantFilesMixin, TestSteps from core.agents.response import AgentResponse from core.config import TROUBLESHOOTER_GET_RUN_COMMAND +from core.config.actions import TS_ALT_SOLUTION, TS_APP_WORKING, TS_DESCRIBE_ISSUE, TS_TASK_REVIEWED from core.db.models.file import File from core.db.models.project_state import IterationStatus, TaskStatus from core.llm.parser import JSONParser, OptionalCodeBlockParser @@ -31,10 +32,6 @@ class RouteFilePaths(BaseModel): files: list[str] = Field(description="List of paths for files that contain routes") -TS_TASK_REVIEWED = "Task #{} reviewed" -TS_ALT_SOLUTION = "Alternative solution (attempt #{})" - - class Troubleshooter(ChatWithBreakdownMixin, IterationPromptMixin, RelevantFilesMixin, BaseAgent): agent_type = "troubleshooter" display_name = "Troubleshooter" @@ -265,7 +262,7 @@ class Troubleshooter(ChatWithBreakdownMixin, IterationPromptMixin, RelevantFiles while True: await self.ui.send_project_stage({"stage": ProjectStage.GET_USER_FEEDBACK}) - test_message = "Please check if the app is working" + test_message = TS_APP_WORKING if user_instructions: hint = " Here is a description of what should be working:\n\n" + user_instructions @@ -307,7 +304,7 @@ class Troubleshooter(ChatWithBreakdownMixin, IterationPromptMixin, RelevantFiles elif user_response.button == "bug": await self.ui.send_project_stage({"stage": ProjectStage.DESCRIBE_ISSUE}) user_description = await self.ask_question( - "Please describe the issue you found (one at a time) and share any relevant server logs", + TS_DESCRIBE_ISSUE, extra_info="collect_logs", buttons={"back": "Back"}, ) diff --git a/core/agents/wizard.py b/core/agents/wizard.py index 08321c08ab..3ab5fb83dc 100644 --- a/core/agents/wizard.py +++ b/core/agents/wizard.py @@ -9,9 +9,9 @@ import httpx import yaml from core.agents.base import BaseAgent -from core.agents.frontend import FE_INIT from core.agents.response import AgentResponse from core.config import SWAGGER_EMBEDDINGS_API +from core.config.actions import FE_INIT from core.log import get_logger from core.telemetry import telemetry from core.templates.registry import PROJECT_TEMPLATES diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 6574f56ce8..84117ee7b6 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -3,22 +3,48 @@ import os import os.path import sys from argparse import ArgumentParser, ArgumentTypeError, Namespace +from difflib import unified_diff from typing import Optional from urllib.parse import urlparse from uuid import UUID from core.config import Config, LLMProvider, LocalIPCConfig, ProviderConfig, UIAdapter, get_config, loader +from core.config.actions import ( + BH_ADDITIONAL_FEEDBACK, + BH_HUMAN_TEST_AGAIN, + BH_IS_BUG_FIXED, + BH_START_BUG_HUNT, + BH_START_USER_TEST, + BH_STARTING_PAIR_PROGRAMMING, + BH_WAIT_BUG_REP_INSTRUCTIONS, + CM_UPDATE_FILES, + DEV_EXECUTE_TASK, + DEV_TASK_BREAKDOWN, + DEV_TASK_START, + DEV_TROUBLESHOOT, + FE_CHANGE_REQ, + FE_DONE_WITH_UI, + HUMAN_INTERVENTION_QUESTION, + MIX_BREAKDOWN_CHAT_PROMPT, + RUN_COMMAND, + TC_TASK_DONE, + TL_EDIT_DEV_PLAN, + TS_APP_WORKING, + TS_DESCRIBE_ISSUE, +) from core.config.env_importer import import_from_dotenv from core.config.version import get_version from core.db.session import SessionManager from core.db.setup import run_migrations -from core.log import setup +from core.log import get_logger, setup from core.state.state_manager import StateManager -from core.ui.base import UIBase +from core.ui.base import AgentSource, UIBase, UISource from core.ui.console import PlainConsoleUI from core.ui.ipc_client import IPCClientUI from core.ui.virtual import VirtualUI +log = get_logger(__name__) + def parse_llm_endpoint(value: str) -> Optional[tuple[LLMProvider, str]]: """ @@ -47,6 +73,35 @@ def parse_llm_endpoint(value: str) -> Optional[tuple[LLMProvider, str]]: return provider, url.geturl() +def get_line_changes(old_content: str, new_content: str) -> tuple[int, int]: + """ + Get the number of added and deleted lines between two files. + + This uses Python difflib to produce a unified diff, then counts + the number of added and deleted lines. + + :param old_content: old file content + :param new_content: new file content + :return: a tuple (added_lines, deleted_lines) + """ + + from_lines = old_content.splitlines(keepends=True) + to_lines = new_content.splitlines(keepends=True) + + diff_gen = unified_diff(from_lines, to_lines) + + added_lines = 0 + deleted_lines = 0 + + for line in diff_gen: + if line.startswith("+") and not line.startswith("+++"): # Exclude the file headers + added_lines += 1 + elif line.startswith("-") and not line.startswith("---"): # Exclude the file headers + deleted_lines += 1 + + return added_lines, deleted_lines + + def parse_llm_key(value: str) -> Optional[tuple[LLMProvider, str]]: """ Parse --llm-key command-line option. @@ -196,45 +251,346 @@ async def list_projects_json(db: SessionManager): """ sm = StateManager(db) projects = await sm.list_projects() - - data = [] - for project in projects: - last_updated = None - p = { - "name": project.name, - "project_type": project.project_type, - "id": project.id.hex, - "branches": [], - } - for branch in project.branches: - b = { - "name": branch.name, - "id": branch.id.hex, - "steps": [], + projects_list = [] + for row in projects: + project_id, project_name, created_at, folder_name = row + projects_list.append( + { + "id": project_id.hex, + "name": project_name, + "folder_name": folder_name, + "updated_at": created_at.isoformat(), } - for state in branch.states: - if not last_updated or state.created_at > last_updated: - last_updated = state.created_at - s = { - "name": state.action or f"Step #{state.step_index}", - "step": state.step_index, - } - b["steps"].append(s) - if b["steps"]: - b["steps"][-1]["name"] = "Latest step" - p["branches"].append(b) - p["updated_at"] = last_updated.isoformat() if last_updated else None - data.append(p) - - print(json.dumps(data, indent=2)) - - -async def list_projects(db: SessionManager): + ) + + print(json.dumps(projects_list, indent=2, default=str)) + + +def find_first_todo_task(tasks): + """ + Find the first task with status 'todo' from a list of tasks. + + :param tasks: List of task objects + :return: First task with status 'todo', or None if not found + """ + if not tasks: + return None + + for task in tasks: + if task.get("status") == "todo": + return task + + return None + + +def trim_logs(logs: str) -> str: """ - List all projects in the database. + Trim logs by removing everything after specific marker phrases. + + This function cuts off the string at the first occurrence of + "Here are the backend logs" or "Here are the frontend logs". + + :param logs: Log text to trim + :return: Trimmed log text with the marker phrase removed + """ + if not logs: + return "" + + # Define marker phrases + markers = ["Here are the backend logs", "Here are the frontend logs"] + + # Find the first occurrence of any marker + index = float("inf") + for marker in markers: + pos = logs.find(marker) + if pos != -1 and pos < index: + index = pos + + # If a marker was found, trim the string + if index != float("inf"): + return logs[:index] + + return logs + + +def get_source_for_history(msg_type: Optional[str] = "", question: Optional[str] = ""): + if question in [TL_EDIT_DEV_PLAN]: + return AgentSource("Tech Lead", "tech-lead") + + if question in [FE_CHANGE_REQ, FE_DONE_WITH_UI]: + return AgentSource("Frontend", "frontend") + + elif question in [ + TS_DESCRIBE_ISSUE, + BH_HUMAN_TEST_AGAIN, + BH_IS_BUG_FIXED, + TS_APP_WORKING, + BH_ADDITIONAL_FEEDBACK, + ] or msg_type in ["instructions", "bh_breakdown"]: + return AgentSource("Bug Hunter", "bug-hunter") + + elif msg_type in ["bug_reproduction_instructions", "bug_description"]: + return AgentSource("Troubleshooter", "troubleshooter") + + elif HUMAN_INTERVENTION_QUESTION in question: + return AgentSource("Human Input", "human-input") + + elif RUN_COMMAND in question: + return AgentSource("Executor", "executor") + + elif msg_type in ["task_description", "task_breakdown"]: + return AgentSource("Developer", "developer") + + else: + return UISource("Pythagora", "pythagora") + + +async def print_convo( + ui: UIBase, + convo: list, +): + for msg in convo: + if "bh_breakdown" in msg: + await ui.send_message( + msg["bh_breakdown"], + source=get_source_for_history(msg_type="bh_breakdown"), + project_state_id=msg["id"], + ) + + if "task_description" in msg: + await ui.send_message( + msg["task_description"], + source=get_source_for_history(msg_type="task_description"), + project_state_id=msg["id"], + ) + + if "task_breakdown" in msg: + await ui.send_message( + msg["task_breakdown"], + source=get_source_for_history(msg_type="task_breakdown"), + project_state_id=msg["id"], + ) + + if "test_instructions" in msg: + await ui.send_test_instructions( + msg["test_instructions"], + project_state_id=msg["id"], + ) + + if "bh_testing_instructions" in msg: + await ui.send_test_instructions( + msg["bh_testing_instructions"], + project_state_id=msg["id"], + ) + + if "files" in msg: + for f in msg["files"]: + await ui.send_file_status(f["path"], "done") + await ui.generate_diff( + file_path=f["path"], + file_old=f.get("old_content", ""), + file_new=f.get("new_content", ""), + n_new_lines=f["diff"][0], + n_del_lines=f["diff"][1], + ) + + if "user_inputs" in msg and msg["user_inputs"]: + for input_item in msg["user_inputs"]: + if "question" in input_item: + await ui.send_message( + input_item["question"], + source=get_source_for_history(question=input_item["question"]), + project_state_id=msg["id"], + ) + + if "answer" in input_item: + if input_item["question"] != TL_EDIT_DEV_PLAN: + await ui.send_user_input_history(input_item["answer"], project_state_id=msg["id"]) + + +async def load_convo( + sm: StateManager, + project_id: Optional[UUID] = None, + branch_id: Optional[UUID] = None, +) -> list: + """ + Loads the conversation from an existing project. + returns: list of dictionaries with the conversation history + """ + convo = [] + + if branch_id is None and project_id is not None: + branches = await sm.get_branches_for_project_id(project_id) + if not branches: + return convo + branch_id = branches[0].id + + project_states = await sm.get_project_states(project_id, branch_id) + + task_counter = 1 + + for i, state in enumerate(project_states): + prev_state = project_states[i - 1] if i > 0 else None + + convo_el = {} + convo_el["id"] = str(state.id) + user_inputs = await sm.find_user_input(state, branch_id) + + todo_task = find_first_todo_task(state.tasks) + if todo_task: + task_counter = state.tasks.index(todo_task) + 1 + + if user_inputs: + convo_el["user_inputs"] = [] + for ui in user_inputs: + if ui.question: + if ui.question == MIX_BREAKDOWN_CHAT_PROMPT: + if len(state.iterations) > 0: + # as it's not available in the current state, take the next state's description - that is the bug description! + next_state = project_states[i + 1] if i + 1 < len(project_states) else None + if next_state is not None and next_state.iterations is not None: + si = next_state.iterations[-1] + if si is not None: + if si.get("description", None) is not None: + convo_el["bh_breakdown"] = si["description"] + else: + # if there are no iterations, it means developer made task breakdown, take the next state's first task with status = todo + next_state = project_states[i + 1] if i + 1 < len(project_states) else None + if next_state is not None: + task = find_first_todo_task(next_state.tasks) + if task.get("test_instructions", None) is not None: + convo_el["test_instructions"] = task["test_instructions"] + if task.get("instructions", None) is not None: + convo_el["task_breakdown"] = task["instructions"] + # skip parsing that questions and its answers due to the fact that we do not keep states inside breakdown convo + break + + if ui.question == BH_HUMAN_TEST_AGAIN: + if len(state.iterations) > 0: + si = state.iterations[-1] + if si is not None: + if si.get("bug_reproduction_description", None) is not None: + convo_el["bh_testing_instructions"] = si["bug_reproduction_description"] + + if ui.question == TS_APP_WORKING: + task = find_first_todo_task(state.tasks) + if task: + if task.get("test_instructions", None) is not None: + convo_el["test_instructions"] = task["test_instructions"] + + if ui.question == DEV_EXECUTE_TASK: + task = find_first_todo_task(state.tasks) + if task: + if task.get("description", None) is not None: + convo_el["task_description"] = f"Task #{task_counter} - " + task["description"] + + answer = trim_logs(ui.answer_text) if ui.answer_text is not None else ui.answer_button + if answer == "bug": + answer = "There is an issue" + elif answer == "change": + answer = "I want to make a change" + convo_el["user_inputs"].append({"question": ui.question, "answer": answer}) + + if state.action is not None: + if state.action == DEV_TROUBLESHOOT.format(task_counter): + if state.iterations is not None and len(state.iterations) > 0: + si = state.iterations[-1] + if si is not None: + if si.get("user_feedback", None) is not None: + convo_el["user_feedback"] = si["user_feedback"] + if si.get("description", None) is not None: + convo_el["description"] = si["description"] + + elif state.action == DEV_TASK_BREAKDOWN.format(task_counter): + task = state.tasks[task_counter - 1] + if task.get("description", None) is not None: + convo_el["task_description"] = f"Task #{task_counter} - " + task["description"] + + if task.get("instructions", None) is not None: + convo_el["task_breakdown"] = task["instructions"] + + elif state.action == TC_TASK_DONE.format(task_counter): + if state.tasks: + next_task = find_first_todo_task(state.tasks) + if next_task is not None and next_task.get("description", None) is not None: + convo_el["task_description"] = f"Task #{task_counter} - " + next_task["description"] + + elif state.action == DEV_TASK_START: + task = state.tasks[task_counter - 1] + if task.get("instructions", None) is not None: + convo_el["task_breakdown"] = task["instructions"] + + elif state.action == CM_UPDATE_FILES: + files = [] + for steps in state.steps: + file = {} + if "save_file" in steps and "path" in steps["save_file"]: + path = steps["save_file"]["path"] + file["path"] = path + + current_file = await sm.get_file_for_project(state.id, path) + prev_file = await sm.get_file_for_project(prev_state.id, path) if prev_state else None + + old_content = prev_file.content.content if prev_file and prev_file.content else "" + new_content = current_file.content.content if current_file and current_file.content else "" + + file["diff"] = get_line_changes( + old_content=old_content, + new_content=new_content, + ) + file["old_content"] = old_content + file["new_content"] = new_content + + if file["diff"] != (0, 0): + files.append(file) + + convo_el["files"] = files + + if state.iterations is not None and len(state.iterations) > 0: + si = state.iterations[-1] + + if state.action == BH_START_BUG_HUNT.format(task_counter): + if si.get("user_feedback", None) is not None: + convo_el["user_feedback"] = si["user_feedback"] + + if si.get("description", None) is not None: + convo_el["description"] = si["description"] + + elif state.action == BH_WAIT_BUG_REP_INSTRUCTIONS.format(task_counter): + for si in state.iterations: + if si.get("bug_reproduction_description", None) is not None: + convo_el["bug_reproduction_description"] = si["bug_reproduction_description"] + + elif state.action == BH_START_USER_TEST.format(task_counter): + if si.get("bug_hunting_cycles", None) is not None: + cycle = si["bug_hunting_cycles"][-1] + if cycle is not None: + if "user_feedback" in cycle and cycle["user_feedback"] is not None: + convo_el["user_feedback"] = cycle["user_feedback"] + if ( + "human_readable_instructions" in cycle + and cycle["human_readable_instructions"] is not None + ): + convo_el["human_readable_instructions"] = cycle["human_readable_instructions"] + + elif state.action == BH_STARTING_PAIR_PROGRAMMING.format(task_counter): + if "user_feedback" in si and si["user_feedback"] is not None: + convo_el["user_feedback"] = si["user_feedback"] + if "initial_explanation" in si and si["initial_explanation"] is not None: + convo_el["initial_explanation"] = si["initial_explanation"] + + convo_el["action"] = state.action + convo.append(convo_el) + + return convo + + +async def list_projects_branches_states(db: SessionManager): + """ + List all projects in the database, including their branches and project states """ sm = StateManager(db) - projects = await sm.list_projects() + projects = await sm.list_projects_with_branches_states() print(f"Available projects ({len(projects)}):") for project in projects: @@ -330,4 +686,11 @@ def init() -> tuple[UIBase, SessionManager, Namespace]: return (ui, db, args) -__all__ = ["parse_arguments", "load_config", "list_projects_json", "list_projects", "load_project", "init"] +__all__ = [ + "parse_arguments", + "load_config", + "list_projects_json", + "list_projects_branches_states", + "load_project", + "init", +] diff --git a/core/cli/main.py b/core/cli/main.py index 0b7ff94ca2..48f1acba31 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -14,7 +14,16 @@ except ImportError: SENTRY_AVAILABLE = False from core.agents.orchestrator import Orchestrator -from core.cli.helpers import delete_project, init, list_projects, list_projects_json, load_project, show_config +from core.cli.helpers import ( + delete_project, + init, + list_projects_branches_states, + list_projects_json, + load_convo, + load_project, + print_convo, + show_config, +) from core.db.session import SessionManager from core.db.v0importer import LegacyDatabaseImporter from core.llm.anthropic_client import CustomAssertionError @@ -22,11 +31,16 @@ from core.llm.base import APIError from core.log import get_logger from core.state.state_manager import StateManager from core.telemetry import telemetry -from core.ui.base import ProjectStage, UIBase, UIClosedError, UserInput, pythagora_source +from core.ui.base import ( + ProjectStage, + UIBase, + UIClosedError, + UserInput, + pythagora_source, +) log = get_logger(__name__) - telemetry_sent = False @@ -200,6 +214,10 @@ async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): success = await load_project(sm, args.project, args.branch, args.step) if not success: return False + + convo = await load_convo(sm, args.project, args.branch) + await print_convo(ui, convo) + else: success = await start_new_project(sm, ui) if not success: @@ -224,7 +242,7 @@ async def async_main( global telemetry_sent if args.list: - await list_projects(db) + await list_projects_branches_states(db) return True elif args.list_json: await list_projects_json(db) diff --git a/core/config/__init__.py b/core/config/__init__.py index 71657ca34a..0d93d7c494 100644 --- a/core/config/__init__.py +++ b/core/config/__init__.py @@ -5,6 +5,8 @@ from typing import Any, Literal, Optional, Union from pydantic import BaseModel, ConfigDict, Field, field_validator from typing_extensions import Annotated +from core.config.constants import LOGS_LINE_LIMIT + ROOT_DIR = abspath(join(dirname(__file__), "..", "..")) DEFAULT_IGNORE_PATHS = [ ".git", @@ -219,6 +221,10 @@ class LogConfig(_StrictModel): "pythagora.log", description="Output file for logs (if not specified, logs are printed to stderr)", ) + max_lines: int = Field( + LOGS_LINE_LIMIT, + description="Maximum number of lines to keep in the log file", + ) class DBConfig(_StrictModel): @@ -348,7 +354,7 @@ class Config(_StrictModel): ), FRONTEND_AGENT_NAME: AgentLLMConfig( provider=LLMProvider.OPENAI, - model="claude-3-5-sonnet-20241022", + model="claude-3-7-sonnet-20250219", temperature=0.0, ), GET_RELEVANT_FILES_AGENT_NAME: AgentLLMConfig( diff --git a/core/config/actions.py b/core/config/actions.py new file mode 100644 index 0000000000..7f11372cfb --- /dev/null +++ b/core/config/actions.py @@ -0,0 +1,57 @@ +BH_START_BUG_HUNT = "Start bug hunt for task #{}" +BH_WAIT_BUG_REP_INSTRUCTIONS = "Awaiting bug reproduction instructions for task #{}" +BH_START_USER_TEST = "Start user testing for task #{}" +BH_STARTING_PAIR_PROGRAMMING = "Start pair programming for task #{}" + +CM_UPDATE_FILES = "Updating files" + + +DEV_WAIT_TEST = "Awaiting user test" +DEV_TASK_START = "Task #{} start" +DEV_TASK_BREAKDOWN = "Task #{} breakdown" +DEV_TROUBLESHOOT = "Troubleshooting #{}" +DEV_TASK_REVIEW_FEEDBACK = "Task review feedback" + +TC_TASK_DONE = "Task #{} complete" + + +FE_INIT = "Frontend init" +FE_START = "Frontend start" +FE_CONTINUE = "Frontend continue" +FE_ITERATION = "Frontend iteration" +FE_ITERATION_DONE = "Frontend iteration done" + +TL_CREATE_INITIAL_EPIC = "Create initial project epic" +TL_CREATE_PLAN = "Create a development plan for epic: {}" +TL_START_FEATURE = "Start of feature #{}" +TL_INITIAL_PROJECT_NAME = "Initial Project" + +TW_WRITE = "Write documentation" + +EX_SKIP_COMMAND = 'Skip "{}"' +EX_RUN_COMMAND = 'Run "{}"' + +SPEC_CREATE_STEP_NAME = "Create specification" +SPEC_CHANGE_STEP_NAME = "Change specification" +SPEC_CHANGE_FEATURE_STEP_NAME = "Change specification due to new feature" + +TS_TASK_REVIEWED = "Task #{} reviewed" +TS_ALT_SOLUTION = "Alternative solution (attempt #{})" +TS_APP_WORKING = "Please check if the app is working" + +PS_EPIC_COMPLETE = "Epic {} completed" + +# other constants +TL_EDIT_DEV_PLAN = "Open and edit your development plan in the Progress tab" +MIX_BREAKDOWN_CHAT_PROMPT = "Are you happy with the breakdown? Now is a good time to ask questions or suggest changes." +FE_CHANGE_REQ = ( + "Do you want to change anything or report a bug? Keep in mind that currently ONLY frontend is implemented." +) +FE_DONE_WITH_UI = "Are you sure you're done building the UI and want to start building the backend functionality now?" +TS_DESCRIBE_ISSUE = "Please describe the issue you found (one at a time) and share any relevant server logs" +BH_HUMAN_TEST_AGAIN = "Please test the app again." +BH_IS_BUG_FIXED = "Is the bug you reported fixed now?" +BH_ADDITIONAL_FEEDBACK = "Please add any additional feedback that could help Pythagora solve this bug" +HUMAN_INTERVENTION_QUESTION = "I need human intervention:" +RUN_COMMAND = "Can I run command:" +DEV_EXECUTE_TASK = "Do you want to execute the above task?" diff --git a/core/config/constants.py b/core/config/constants.py index 38880b120b..aaeeb0399e 100644 --- a/core/config/constants.py +++ b/core/config/constants.py @@ -1 +1,2 @@ CONVO_ITERATIONS_LIMIT = 8 +LOGS_LINE_LIMIT = 20000 diff --git a/core/db/models/project.py b/core/db/models/project.py index f33283490a..1a057a291b 100644 --- a/core/db/models/project.py +++ b/core/db/models/project.py @@ -4,12 +4,12 @@ from typing import TYPE_CHECKING, Optional, Union from unicodedata import normalize from uuid import UUID, uuid4 -from sqlalchemy import and_, delete, inspect, select +from sqlalchemy import Row, and_, delete, inspect, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Mapped, mapped_column, relationship, selectinload from sqlalchemy.sql import func -from core.db.models import Base +from core.db.models import Base, File if TYPE_CHECKING: from core.db.models import Branch @@ -68,7 +68,28 @@ class Project(Base): return result.scalar_one_or_none() @staticmethod - async def get_all_projects(session: "AsyncSession") -> list["Project"]: + async def get_file_for_project(session: AsyncSession, project_state_id: UUID, path: str) -> Optional["File"]: + file_result = await session.execute( + select(File).where(File.project_state_id == project_state_id, File.path == path) + ) + return file_result.scalar_one_or_none() + + @staticmethod + async def get_branches_for_project_id(session: AsyncSession, project_id: UUID) -> list["Branch"]: + from core.db.models import Branch + + branch_result = await session.execute(select(Branch).where(Branch.project_id == project_id)) + return branch_result.scalars().all() + + @staticmethod + async def get_all_projects(session: "AsyncSession") -> list[Row]: + query = select(Project.id, Project.name, Project.created_at, Project.folder_name).order_by(Project.name) + + result = await session.execute(query) + return result.fetchall() + + @staticmethod + async def get_all_projects_with_branches_states(session: "AsyncSession") -> list["Project"]: """ Get all projects. diff --git a/core/db/models/project_state.py b/core/db/models/project_state.py index 409a2149e6..1190d0a404 100644 --- a/core/db/models/project_state.py +++ b/core/db/models/project_state.py @@ -3,12 +3,13 @@ from datetime import datetime from typing import TYPE_CHECKING, Optional, Union from uuid import UUID, uuid4 -from sqlalchemy import ForeignKey, UniqueConstraint, delete, inspect +from sqlalchemy import ForeignKey, UniqueConstraint, delete, inspect, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.sql import func +from core.config.actions import PS_EPIC_COMPLETE from core.db.models import Base, FileContent from core.log import get_logger @@ -46,10 +47,6 @@ class IterationStatus: DONE = "done" -PS_EPIC_COMPLETE = "Epic {} completed" -PS_TASK_COMPLETE = "Task {} completed" - - class ProjectState(Base): __tablename__ = "project_states" __table_args__ = ( @@ -216,8 +213,41 @@ class ProjectState(Base): branch=branch, specification=Specification(), step_index=1, + action="Initial project state", ) + @staticmethod + async def get_project_states( + session: "AsyncSession", + project_id: Optional[UUID] = None, + branch_id: Optional[UUID] = None, + ) -> list["ProjectState"]: + from core.db.models import Branch, ProjectState + + branch = None + limit = 100 + + if branch_id: + branch = await session.execute(select(Branch).where(Branch.id == branch_id)) + branch = branch.scalar_one_or_none() + elif project_id: + branch = await session.execute(select(Branch).where(Branch.project_id == project_id)) + branch = branch.scalar_one_or_none() + + if branch: + query = ( + select(ProjectState) + .where(ProjectState.branch_id == branch.id) + .order_by(ProjectState.step_index.desc()) # Get the latest 100 states + .limit(limit) + ) + + project_states_result = await session.execute(query) + project_states = project_states_result.scalars().all() + return sorted(project_states, key=lambda x: x.step_index) + + return [] + async def create_next_state(self) -> "ProjectState": """ Create the next project state for the branch. @@ -436,18 +466,43 @@ class ProjectState(Base): async def delete_after(self): """ - Delete all states in the branch after this one. + Delete all states in the branch after this one, along with related data. + + This includes: + - ProjectState records after this one + - Related UserInput records (including those for the current state) + - Related File records + - Orphaned FileContent records """ + from core.db.models import FileContent, UserInput session: AsyncSession = inspect(self).async_session log.debug(f"Deleting all project states in branch {self.branch_id} after {self.id}") - await session.execute( - delete(ProjectState).where( + + # Get all project states to be deleted + states_to_delete = await session.execute( + select(ProjectState).where( ProjectState.branch_id == self.branch_id, ProjectState.step_index > self.step_index, ) ) + states_to_delete = states_to_delete.scalars().all() + state_ids = [state.id for state in states_to_delete] + + # Delete user inputs for the current state + await session.execute(delete(UserInput).where(UserInput.project_state_id == self.id)) + + if state_ids: + # Delete related user inputs for states to be deleted + await session.execute(delete(UserInput).where(UserInput.project_state_id.in_(state_ids))) + + # Delete project states + await session.execute(delete(ProjectState).where(ProjectState.id.in_(state_ids))) + + # Clean up orphaned file content and user inputs + await FileContent.delete_orphans(session) + await UserInput.delete_orphans(session) def get_last_iteration_steps(self) -> list: """ diff --git a/core/db/models/user_input.py b/core/db/models/user_input.py index c068143db4..d68012b1c6 100644 --- a/core/db/models/user_input.py +++ b/core/db/models/user_input.py @@ -2,7 +2,8 @@ from datetime import datetime from typing import TYPE_CHECKING, Optional from uuid import UUID -from sqlalchemy import ForeignKey, inspect +from sqlalchemy import ForeignKey, and_, delete, inspect, select +from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.sql import func @@ -57,3 +58,24 @@ class UserInput(Base): ) session.add(obj) return obj + + @staticmethod + async def find_user_inputs(session: AsyncSession, project_state, branch_id) -> Optional[list["UserInput"]]: + from core.db.models import UserInput + + user_input = await session.execute( + select(UserInput).where( + and_(UserInput.branch_id == branch_id, UserInput.project_state_id == project_state.id) + ) + ) + user_input = user_input.scalars().all() + return user_input if len(user_input) > 0 else [] + + @classmethod + async def delete_orphans(cls, session: AsyncSession): + """ + Delete UserInput objects that have no associated ProjectState. + + :param session: The database session. + """ + await session.execute(delete(UserInput).where(UserInput.project_state_id.is_(None))) diff --git a/core/log/__init__.py b/core/log/__init__.py index 67eb2ffcca..4737cdf363 100644 --- a/core/log/__init__.py +++ b/core/log/__init__.py @@ -1,6 +1,65 @@ +import os +from collections import deque from logging import FileHandler, Formatter, Logger, StreamHandler, getLogger from core.config import LogConfig +from core.config.constants import LOGS_LINE_LIMIT + + +class LineCountLimitedFileHandler(FileHandler): + """ + A file handler that limits the number of lines in the log file. + It keeps a fixed number of the most recent log lines. + """ + + def __init__(self, filename, max_lines=LOGS_LINE_LIMIT, mode="a", encoding=None, delay=False): + """ + Initialize the handler with the file and max lines. + + :param filename: Log file path + :param max_lines: Maximum number of lines to keep in the file + :param mode: File open mode + :param encoding: File encoding + :param delay: Delay file opening until first emit + """ + super().__init__(filename, mode, encoding, delay) + self.max_lines = max_lines + self.line_buffer = deque(maxlen=max_lines) + self._load_existing_lines() + + def _load_existing_lines(self): + """Load existing lines from the file into the buffer if the file exists.""" + if os.path.exists(self.baseFilename): + try: + with open(self.baseFilename, "r", encoding=self.encoding) as f: + for line in f: + if len(self.line_buffer) < self.max_lines: + self.line_buffer.append(line) + else: + self.line_buffer.popleft() + self.line_buffer.append(line) + except Exception: + # If there's an error reading the file, we'll just start with an empty buffer + self.line_buffer.clear() + + def emit(self, record): + """ + Emit a record and maintain the line count limit. + + :param record: Log record to emit + """ + try: + msg = self.format(record) + line = msg + self.terminator + self.line_buffer.append(line) + + # Rewrite the entire file with the current buffer + with open(self.baseFilename, "w", encoding=self.encoding) as f: + f.writelines(self.line_buffer) + + self.flush() + except Exception: + self.handleError(record) def setup(config: LogConfig, force: bool = False): @@ -27,7 +86,9 @@ def setup(config: LogConfig, force: bool = False): formatter = Formatter(config.format) if config.output: - handler = FileHandler(config.output, encoding="utf-8") + # Use our custom handler that limits line count + max_lines = getattr(config, "max_lines", LOGS_LINE_LIMIT) + handler = LineCountLimitedFileHandler(config.output, max_lines=max_lines, encoding="utf-8") else: handler = StreamHandler() diff --git a/core/prompts/frontend/build_frontend.prompt b/core/prompts/frontend/build_frontend.prompt index 95b4193d93..8e4085f3ee 100644 --- a/core/prompts/frontend/build_frontend.prompt +++ b/core/prompts/frontend/build_frontend.prompt @@ -19,6 +19,7 @@ Use material design and nice icons for the design to be appealing and modern. Us 3. Heroicons: For a set of sleek, customizable icons that integrate well with modern designs. 4. React Hook Form: For efficient form handling with minimal re-rendering, ensuring a smooth user experience in form-heavy applications. 5. Use Tailwind built-in animations to enhance the visual appeal of the app + 6. Make the app look colorful and modern but also have the colors be subtle. Choose a flat color palette and make sure that the text is readable and follow design best practices to make the text readable. Also, Implement these design features onto the page - gradient background, frosted glass effects, rounded corner, buttons need to be in the brand colors, and interactive feedback on hover and focus. diff --git a/core/prompts/tech-writer/create_readme.prompt b/core/prompts/tech-writer/create_readme.prompt index 4d9e8d956a..48550906fc 100644 --- a/core/prompts/tech-writer/create_readme.prompt +++ b/core/prompts/tech-writer/create_readme.prompt @@ -2,11 +2,13 @@ You are working on a project called "{{ state.branch.project.name }}" and you ne {% include "partials/project_details.prompt" %} {% include "partials/features_list.prompt" %} -{% include "partials/files_list.prompt" %} -DO NOT specify commands to create any folders or files, they will be created automatically - just specify the relative path to file that needs to be written. +~~FILE_DESCRIPTIONS_IN_THE_CODEBASE~~ +{% include "partials/files_descriptions.prompt" %} +~~END_OF_FILE_DESCRIPTIONS_IN_THE_CODEBASE~~ -{% include "partials/relative_paths.prompt" %} +Project dependencies are installed using "npm install" in the root folder. That will install all the necessary dependencies in the root, client and server folders. +Project is started using "npm run start" in the root folder. That will start both the frontend and backend. Now, based on the project details provided, think step by step and create README.md file for this project. The file should have the following format: diff --git a/core/state/state_manager.py b/core/state/state_manager.py index d853b13ee9..2da6db50be 100644 --- a/core/state/state_manager.py +++ b/core/state/state_manager.py @@ -6,7 +6,7 @@ from contextlib import asynccontextmanager from typing import TYPE_CHECKING, Optional from uuid import UUID, uuid4 -from sqlalchemy import inspect, select +from sqlalchemy import Row, inspect, select from tenacity import retry, stop_after_attempt, wait_fixed from core.config import FileSystemType, get_config @@ -69,11 +69,9 @@ class StateManager: finally: self.blockDb = False # Unset the block - async def list_projects(self) -> list[Project]: + async def list_projects(self) -> list[Row]: """ - List projects with branches - - :return: List of projects with all their branches. + :return: List of projects """ async with self.session_manager as session: return await Project.get_all_projects(session) @@ -83,6 +81,25 @@ class StateManager: raise ValueError("No database session open.") return await File.get_referencing_files(self.current_session, project_state, file_content) + async def list_projects_with_branches_states(self) -> list[Project]: + """ + :return: List of projects with branches and states (old) - for debugging + """ + async with self.session_manager as session: + return await Project.get_all_projects_with_branches_states(session) + + async def get_project_states(self, project_id: Optional[UUID], branch_id: Optional[UUID]) -> list[ProjectState]: + return await ProjectState.get_project_states(self.current_session, project_id, branch_id) + + async def get_branches_for_project_id(self, project_id: UUID) -> list[Branch]: + return await Project.get_branches_for_project_id(self.current_session, project_id) + + async def find_user_input(self, project_state, branch_id) -> Optional[list["UserInput"]]: + return await UserInput.find_user_inputs(self.current_session, project_state, branch_id) + + async def get_file_for_project(self, state_id: UUID, path: str): + return await Project.get_file_for_project(self.current_session, state_id, path) + async def create_project( self, name: str, project_type: Optional[str] = "node", folder_name: Optional[str] = None ) -> Project: diff --git a/core/ui/base.py b/core/ui/base.py index 8c033b3de3..b7db27abb8 100644 --- a/core/ui/base.py +++ b/core/ui/base.py @@ -122,6 +122,21 @@ class UIBase: """ raise NotImplementedError() + async def send_user_input_history( + self, + message: str, + source: Optional[UISource] = None, + project_state_id: Optional[str] = None, + ): + """ + Send a user input history (what the user said to Pythagora) message to the UI. + + :param message: Message content. + :param source: Source of the message (if any). + :param project_state_id: Current project state id. + """ + raise NotImplementedError() + async def send_message( self, message: str, diff --git a/core/ui/console.py b/core/ui/console.py index fb989f496c..5ed952ab5b 100644 --- a/core/ui/console.py +++ b/core/ui/console.py @@ -29,6 +29,17 @@ class PlainConsoleUI(UIBase): else: print(chunk, end="", flush=True) + async def send_user_input_history( + self, + message: str, + source: Optional[UISource] = None, + project_state_id: Optional[str] = None, + ): + if source: + print(f"[{source}] {message}") + else: + print(message) + async def send_message( self, message: str, diff --git a/core/ui/ipc_client.py b/core/ui/ipc_client.py index d7f39a10ab..71ba3542a1 100644 --- a/core/ui/ipc_client.py +++ b/core/ui/ipc_client.py @@ -56,6 +56,7 @@ class MessageType(str, Enum): KNOWLEDGE_BASE_UPDATE = "updatedKnowledgeBase" STOP_APP = "stopApp" TOKEN_EXPIRED = "tokenExpired" + USER_INPUT_HISTORY = "userInputHistory" class Message(BaseModel): @@ -210,6 +211,19 @@ class IPCClientUI(UIBase): project_state_id=project_state_id, ) + async def send_user_input_history( + self, + message: str, + source: Optional[UISource] = None, + project_state_id: Optional[str] = None, + ): + await self._send( + MessageType.USER_INPUT_HISTORY, + content=message, + category=source.type_name if source else None, + project_state_id=project_state_id, + ) + async def send_message( self, message: str, @@ -497,7 +511,12 @@ class IPCClientUI(UIBase): content=stats, ) - async def send_test_instructions(self, test_instructions: str, project_state_id: Optional[str] = None): + async def send_test_instructions( + self, + test_instructions: str, + project_state_id: Optional[str] = None, + source: Optional[UISource] = None, + ): try: log.debug("Sending test instructions") parsed_instructions = json.loads(test_instructions) @@ -511,6 +530,7 @@ class IPCClientUI(UIBase): "test_instructions": parsed_instructions, }, project_state_id=project_state_id, + category=source.type_name if source else None, ) async def knowledge_base_update(self, knowledge_base: dict): diff --git a/core/ui/virtual.py b/core/ui/virtual.py index 0faba17dee..7568fbe54f 100644 --- a/core/ui/virtual.py +++ b/core/ui/virtual.py @@ -30,6 +30,14 @@ class VirtualUI(UIBase): else: print(chunk, end="", flush=True) + async def send_user_input_history( + self, + message: str, + source: Optional[UISource] = None, + project_state_id: Optional[str] = None, + ): + raise NotImplementedError() + async def send_message( self, message: str, diff --git a/pyproject.toml b/pyproject.toml index afa8fc0b3a..34dc857cc3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pythagora-core" -version = "1.3.6" +version = "1.3.8" description = "Build complete apps using AI agents" authors = ["Leon Ostrez "] license = "FSL-1.1-MIT" diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 41af267915..a46c47865e 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -7,7 +7,7 @@ import pytest from core.cli.helpers import ( init, - list_projects, + list_projects_branches_states, list_projects_json, load_config, load_project, @@ -167,49 +167,29 @@ def test_show_default_config(capsys): async def test_list_projects_json(mock_StateManager, capsys): sm = mock_StateManager.return_value - branch = MagicMock( - id=MagicMock(hex="1234"), - states=[ - MagicMock(step_index=1, action="foo", created_at=datetime(2021, 1, 1)), - MagicMock(step_index=2, action=None, created_at=datetime(2021, 1, 2)), - MagicMock(step_index=3, action="baz", created_at=datetime(2021, 1, 3)), - ], - ) - branch.name = "branch1" + project_id1 = MagicMock(hex="abcd") + project_id2 = MagicMock(hex="efgh") - project = MagicMock( - id=MagicMock(hex="abcd"), - project_type=MagicMock(hex="abcd"), - branches=[branch], - ) - project.name = "project1" - project.project_type = "node" - sm.list_projects = AsyncMock(return_value=[project]) + created_at1 = datetime(2021, 1, 1) + created_at2 = datetime(2021, 1, 2) + + projects_data = [ + (project_id1, "project1", created_at1, "folder1"), + (project_id2, "project2", created_at2, "folder2"), + ] + + sm.list_projects = AsyncMock(return_value=projects_data) await list_projects_json(None) mock_StateManager.assert_called_once_with(None) sm.list_projects.assert_awaited_once_with() - data = json.loads(capsys.readouterr().out) + captured = capsys.readouterr().out + data = json.loads(captured) assert data == [ - { - "name": "project1", - "id": "abcd", - "project_type": "node", - "updated_at": "2021-01-03T00:00:00", - "branches": [ - { - "name": "branch1", - "id": "1234", - "steps": [ - {"step": 1, "name": "foo"}, - {"step": 2, "name": "Step #2"}, - {"step": 3, "name": "Latest step"}, - ], - }, - ], - }, + {"id": "abcd", "name": "project1", "folder_name": "folder1", "updated_at": "2021-01-01T00:00:00"}, + {"id": "efgh", "name": "project2", "folder_name": "folder2", "updated_at": "2021-01-02T00:00:00"}, ] @@ -233,11 +213,12 @@ async def test_list_projects(mock_StateManager, capsys): branches=[branch], ) project.name = "project1" - sm.list_projects = AsyncMock(return_value=[project]) - await list_projects(None) + + sm.list_projects_with_branches_states = AsyncMock(return_value=[project]) + await list_projects_branches_states(None) mock_StateManager.assert_called_once_with(None) - sm.list_projects.assert_awaited_once_with() + sm.list_projects_with_branches_states.assert_awaited_once_with() data = capsys.readouterr().out diff --git a/tests/db/test_project.py b/tests/db/test_project.py index 39b713cb50..e21314a2f4 100644 --- a/tests/db/test_project.py +++ b/tests/db/test_project.py @@ -1,8 +1,13 @@ +import json +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock, patch from uuid import uuid4 import pytest +from core.cli.helpers import list_projects_json from core.db.models import Branch, Project +from core.state.state_manager import StateManager from .factories import create_project_state @@ -72,16 +77,58 @@ async def test_get_branch_no_session(): @pytest.mark.asyncio -async def test_get_all_projects(testdb): - state1 = create_project_state() - state2 = create_project_state() +async def test_get_all_projects(testdb, capsys): + state1 = create_project_state(project_name="Test Project 1") + state2 = create_project_state(project_name="Test Project 2") + testdb.add(state1) testdb.add(state2) - - projects = await Project.get_all_projects(testdb) - assert len(projects) == 2 - assert state1.branch.project in projects - assert state2.branch.project in projects + await testdb.commit() # Ensure changes are committed + + # Set folder names for the test + folder_name1 = "folder1" + folder_name2 = "folder2" + + sm = StateManager(testdb) + sm.list_projects = AsyncMock( + return_value=[ + ( + MagicMock(hex=state1.branch.project.id.hex), + state1.branch.project.name, + datetime(2021, 1, 1), + folder_name1, + ), + ( + MagicMock(hex=state2.branch.project.id.hex), + state2.branch.project.name, + datetime(2021, 1, 2), + folder_name2, + ), + ] + ) + + with patch("core.cli.helpers.StateManager", return_value=sm): + await list_projects_json(testdb) + + captured = capsys.readouterr() + data = json.loads(captured.out) + + expected_output = [ + { + "id": state1.branch.project.id.hex, + "name": "Test Project 1", + "folder_name": folder_name1, + "updated_at": "2021-01-01T00:00:00", + }, + { + "id": state2.branch.project.id.hex, + "name": "Test Project 2", + "folder_name": folder_name2, + "updated_at": "2021-01-02T00:00:00", + }, + ] + + assert data == expected_output @pytest.mark.asyncio diff --git a/tests/state/test_state_manager.py b/tests/state/test_state_manager.py index 7ecfe781c7..49472abe90 100644 --- a/tests/state/test_state_manager.py +++ b/tests/state/test_state_manager.py @@ -31,7 +31,8 @@ async def test_create_project(mock_get_config, testmanager): assert sm.current_state == initial_state projects = await sm.list_projects() - assert projects == [project] + assert projects[0][0] == project.id + assert projects[0][1] == project.name @pytest.mark.asyncio @@ -54,7 +55,8 @@ async def test_delete_project(mock_get_config, testmanager): project = await sm.create_project("test") projects = await sm.list_projects() - assert projects == [project] + assert projects[0][0] == project.id + assert projects[0][1] == project.name await sm.delete_project(project.id) projects = await sm.list_projects()