From b852ff9742800b04fee953908f01365a8b1f53d8 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Fri, 21 Mar 2025 14:15:51 +0100 Subject: [PATCH 01/43] Fix command working dir --- core/agents/frontend.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/agents/frontend.py b/core/agents/frontend.py index e4b0d90cff..1d04d8d989 100644 --- a/core/agents/frontend.py +++ b/core/agents/frontend.py @@ -291,6 +291,11 @@ class Frontend(FileDiffMixin, GitMixin, BaseAgent): # Add "cd client" prefix if not already present if not command.startswith("cd "): command = f"cd client && {command}" + # if command is cd client && some_command client/ -> won't work, we need to remove client/ after && + prefix, cmd_part = command.split("&&", 1) + cmd_part = cmd_part.strip().replace("client/", "") + command = f"{prefix} && {cmd_part}" + await self.send_message(f"Running command: `{command}`...") await self.process_manager.run_command(command) else: From 02a8ae21017fc939c3cd95e087387fd820031d63 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Fri, 21 Mar 2025 15:56:01 +0100 Subject: [PATCH 02/43] Remove code reviewer for now --- core/agents/code_monkey.py | 38 +++----------------------------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/core/agents/code_monkey.py b/core/agents/code_monkey.py index bb7c2771c9..9341d49bb7 100644 --- a/core/agents/code_monkey.py +++ b/core/agents/code_monkey.py @@ -2,7 +2,7 @@ import asyncio import re from difflib import unified_diff from enum import Enum -from typing import Optional, Union +from typing import Optional from pydantic import BaseModel, Field @@ -69,12 +69,7 @@ class CodeMonkey(FileDiffMixin, BaseAgent): return await self.describe_files() else: data = await self.implement_changes() - code_review_done = False - while not code_review_done: - review_response = await self.run_code_review(data) - if isinstance(review_response, AgentResponse): - return review_response - data = await self.implement_changes(review_response) + return await self.accept_changes(data["path"], data["old_content"], data["new_content"]) async def implement_changes(self, data: Optional[dict] = None) -> dict: file_name = self.step["save_file"]["path"] @@ -191,33 +186,6 @@ class CodeMonkey(FileDiffMixin, BaseAgent): # CODE REVIEW # ------------------------------ - async def run_code_review(self, data: Optional[dict]) -> Union[AgentResponse, dict]: - await self.ui.send_file_status(data["path"], "reviewing", source=self.ui_source) - if ( - data is not None - and not data["old_content"] - or data["new_content"] == data["old_content"] - or data["attempt"] >= MAX_CODING_ATTEMPTS - ): - # we always auto-accept new files and unchanged files, or if we've tried too many times - return await self.accept_changes(data["path"], data["old_content"], data["new_content"]) - - approved_content, feedback = await self.review_change( - data["path"], - data["instructions"], - data["old_content"], - data["new_content"], - ) - if feedback: - return { - "new_content": data["new_content"], - "approved_content": approved_content, - "feedback": feedback, - "attempt": data["attempt"], - } - else: - return await self.accept_changes(data["path"], data["old_content"], approved_content) - async def accept_changes(self, file_path: str, old_content: str, new_content: str) -> AgentResponse: await self.ui.send_file_status(file_path, "done", source=self.ui_source) @@ -367,7 +335,7 @@ class CodeMonkey(FileDiffMixin, BaseAgent): """ Get the diff between two files. - This uses Python difflib to produce an unified diff, then splits + This uses Python difflib to produce a unified diff, then splits it into hunks that will be separately reviewed by the reviewer. :param file_name: name of the file being modified From 8a96502642ccf988f8ee96c702b58030ab3caa34 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Fri, 21 Mar 2025 16:45:47 +0100 Subject: [PATCH 03/43] Solidify logic --- core/agents/frontend.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/agents/frontend.py b/core/agents/frontend.py index 1d04d8d989..a81b43c6ce 100644 --- a/core/agents/frontend.py +++ b/core/agents/frontend.py @@ -291,10 +291,11 @@ class Frontend(FileDiffMixin, GitMixin, BaseAgent): # Add "cd client" prefix if not already present if not command.startswith("cd "): command = f"cd client && {command}" - # if command is cd client && some_command client/ -> won't work, we need to remove client/ after && - prefix, cmd_part = command.split("&&", 1) - cmd_part = cmd_part.strip().replace("client/", "") - command = f"{prefix} && {cmd_part}" + + # if command is cd client && some_command client/ -> won't work, we need to remove client/ after && + prefix, cmd_part = command.split("&&", 1) + cmd_part = cmd_part.strip().replace("client/", "") + command = f"{prefix} && {cmd_part}" await self.send_message(f"Running command: `{command}`...") await self.process_manager.run_command(command) From f0ad9e3602d2c1e322e92a55f1fcb677068e8cee Mon Sep 17 00:00:00 2001 From: Artur Nesterenko Date: Mon, 24 Mar 2025 16:30:48 +0100 Subject: [PATCH 04/43] hotfix to stream message important --- core/agents/bug_hunter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/agents/bug_hunter.py b/core/agents/bug_hunter.py index 837ef7bc0a..c9976c76bd 100644 --- a/core/agents/bug_hunter.py +++ b/core/agents/bug_hunter.py @@ -78,6 +78,7 @@ class BugHunter(ChatWithBreakdownMixin, BaseAgent): async def get_bug_reproduction_instructions(self): await self.send_message("Finding a way to reproduce the bug ...") + await self.ui.start_important_stream() llm = self.get_llm() convo = ( AgentConvo(self) From c399210f13c4e3f1fff7e1d676e263ba3e42d12c Mon Sep 17 00:00:00 2001 From: mijauexe Date: Tue, 25 Mar 2025 09:07:42 +0100 Subject: [PATCH 05/43] Optimize project load --- core/cli/helpers.py | 55 +++++++++++------------ core/db/models/project.py | 48 ++++++++------------- core/db/models/project_state.py | 1 + core/state/state_manager.py | 6 +-- tests/cli/test_cli.py | 35 ++++++++------- tests/db/test_project.py | 72 +++++++++++++++++++++++++++---- tests/state/test_state_manager.py | 6 ++- 7 files changed, 135 insertions(+), 88 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 953e7efe79..8ef9b887cf 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -3,6 +3,7 @@ import os import os.path import sys from argparse import ArgumentParser, ArgumentTypeError, Namespace +from collections import defaultdict from typing import Optional from urllib.parse import urlparse from uuid import UUID @@ -196,36 +197,32 @@ async def list_projects_json(db: SessionManager): """ sm = StateManager(db) projects = await sm.list_projects() + projects_dict = defaultdict(lambda: {"branches": [], "updated_at": None}) + for row in projects: + project_id, project_name, branch_id, branch_name, state_id, step_index, action, created_at = row - data = [] - for project in projects: - last_updated = None - p = { - "name": project.name, - "id": project.id.hex, - "branches": [], - } - for branch in project.branches: - b = { - "name": branch.name, - "id": branch.id.hex, - "steps": [], - } - for state in branch.states: - if not last_updated or state.created_at > last_updated: - last_updated = state.created_at - s = { - "name": state.action or f"Step #{state.step_index}", - "step": state.step_index, - } - b["steps"].append(s) - if b["steps"]: - b["steps"][-1]["name"] = "Latest step" - p["branches"].append(b) - p["updated_at"] = last_updated.isoformat() if last_updated else None - data.append(p) - - print(json.dumps(data, indent=2)) + project = projects_dict[project_id] + project["id"] = project_id.hex + project["name"] = project_name + + branch = next((b for b in project["branches"] if b["id"] == branch_id.hex), None) + if not branch: + branch = {"id": branch_id.hex, "name": branch_name, "steps": []} + project["branches"].append(branch) + + if created_at and (project["updated_at"] is None or created_at > project["updated_at"]): + project["updated_at"] = created_at + + branch["steps"].append({"name": action, "step": step_index}) + + for project in projects_dict.values(): + for branch in project["branches"]: + if branch["steps"]: + branch["steps"][-1]["name"] = "Latest step" + + project["updated_at"] = project["updated_at"].isoformat() if project["updated_at"] else None + + print(json.dumps(list(projects_dict.values()), indent=2, default=str)) async def list_projects(db: SessionManager): diff --git a/core/db/models/project.py b/core/db/models/project.py index 4f1a716aa1..33a6d20f9f 100644 --- a/core/db/models/project.py +++ b/core/db/models/project.py @@ -4,9 +4,9 @@ from typing import TYPE_CHECKING, Optional, Union from unicodedata import normalize from uuid import UUID, uuid4 -from sqlalchemy import and_, delete, inspect, select +from sqlalchemy import Row, delete, inspect, select from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import Mapped, mapped_column, relationship, selectinload +from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.sql import func from core.db.models import Base @@ -67,40 +67,28 @@ class Project(Base): return result.scalar_one_or_none() @staticmethod - async def get_all_projects(session: "AsyncSession") -> list["Project"]: - """ - Get all projects. - - This assumes the projects have at least one branch and one state. - - :param session: The SQLAlchemy session. - :return: List of Project objects. - """ + async def get_all_projects(session: "AsyncSession") -> list[Row]: from core.db.models import Branch, ProjectState - latest_state_query = ( - select(ProjectState.branch_id, func.max(ProjectState.step_index).label("max_index")) - .group_by(ProjectState.branch_id) - .subquery() - ) - query = ( - select(Project, Branch, ProjectState) - .join(Branch, Project.branches) - .join(ProjectState, Branch.states) - .join( - latest_state_query, - and_( - ProjectState.branch_id == latest_state_query.columns.branch_id, - ProjectState.step_index == latest_state_query.columns.max_index, - ), + select( + Project.id, + Project.name, + Branch.id, + Branch.name, + ProjectState.id, + ProjectState.step_index, + ProjectState.action, + ProjectState.created_at, ) - .options(selectinload(Project.branches), selectinload(Branch.states)) - .order_by(Project.name, Branch.name) + .join(Branch, Project.branches) + .join(ProjectState, Branch.id == ProjectState.branch_id) + .where(ProjectState.action.isnot(None)) + .order_by(Project.id, Branch.id, ProjectState.step_index.asc()) ) - results = await session.execute(query) - return results.scalars().all() + result = await session.execute(query) + return result.fetchall() @staticmethod def get_folder_from_project_name(name: str): diff --git a/core/db/models/project_state.py b/core/db/models/project_state.py index c65bdf6d38..2bfd9fd031 100644 --- a/core/db/models/project_state.py +++ b/core/db/models/project_state.py @@ -216,6 +216,7 @@ class ProjectState(Base): branch=branch, specification=Specification(), step_index=1, + action="Initial project state", ) async def create_next_state(self) -> "ProjectState": diff --git a/core/state/state_manager.py b/core/state/state_manager.py index 8921078db8..5de4de9552 100644 --- a/core/state/state_manager.py +++ b/core/state/state_manager.py @@ -6,7 +6,7 @@ from contextlib import asynccontextmanager from typing import TYPE_CHECKING, Optional from uuid import UUID, uuid4 -from sqlalchemy import inspect, select +from sqlalchemy import Row, inspect, select from tenacity import retry, stop_after_attempt, wait_fixed from core.config import FileSystemType, get_config @@ -69,11 +69,11 @@ class StateManager: finally: self.blockDb = False # Unset the block - async def list_projects(self) -> list[Project]: + async def list_projects(self) -> list[Row]: """ List projects with branches - :return: List of projects with all their branches. + :return: List of projects with all their branches and project states """ async with self.session_manager as session: return await Project.get_all_projects(session) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index a9dd5f3d90..f19309675b 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -167,22 +167,25 @@ def test_show_default_config(capsys): async def test_list_projects_json(mock_StateManager, capsys): sm = mock_StateManager.return_value - branch = MagicMock( - id=MagicMock(hex="1234"), - states=[ - MagicMock(step_index=1, action="foo", created_at=datetime(2021, 1, 1)), - MagicMock(step_index=2, action=None, created_at=datetime(2021, 1, 2)), - MagicMock(step_index=3, action="baz", created_at=datetime(2021, 1, 3)), - ], - ) - branch.name = "branch1" + # Mock the return value to match the expected format + # Format: project_id, project_name, branch_id, branch_name, state_id, step_index, action, created_at + project_id = MagicMock(hex="abcd") + branch_id = MagicMock(hex="1234") + state_id1 = MagicMock(hex="state1") + state_id2 = MagicMock(hex="state2") + state_id3 = MagicMock(hex="state3") + + created_at1 = datetime(2021, 1, 1) + created_at2 = datetime(2021, 1, 2) + created_at3 = datetime(2021, 1, 3) + + projects_data = [ + (project_id, "project1", branch_id, "branch1", state_id1, 1, "foo", created_at1), + (project_id, "project1", branch_id, "branch1", state_id2, 2, None, created_at2), + (project_id, "project1", branch_id, "branch1", state_id3, 3, "baz", created_at3), + ] - project = MagicMock( - id=MagicMock(hex="abcd"), - branches=[branch], - ) - project.name = "project1" - sm.list_projects = AsyncMock(return_value=[project]) + sm.list_projects = AsyncMock(return_value=projects_data) await list_projects_json(None) mock_StateManager.assert_called_once_with(None) @@ -201,7 +204,7 @@ async def test_list_projects_json(mock_StateManager, capsys): "id": "1234", "steps": [ {"step": 1, "name": "foo"}, - {"step": 2, "name": "Step #2"}, + {"step": 2, "name": None}, {"step": 3, "name": "Latest step"}, ], }, diff --git a/tests/db/test_project.py b/tests/db/test_project.py index 2b3c54019f..7424e0bfd3 100644 --- a/tests/db/test_project.py +++ b/tests/db/test_project.py @@ -1,8 +1,13 @@ +import json +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock, patch from uuid import uuid4 import pytest +from core.cli.helpers import list_projects_json from core.db.models import Branch, Project +from core.state.state_manager import StateManager from .factories import create_project_state @@ -72,16 +77,67 @@ async def test_get_branch_no_session(): @pytest.mark.asyncio -async def test_get_all_projects(testdb): - state1 = create_project_state() - state2 = create_project_state() +async def test_get_all_projects(testdb, capsys): + state1 = create_project_state(project_name="Test Project 1") + state2 = create_project_state(project_name="Test Project 2") + testdb.add(state1) testdb.add(state2) - - projects = await Project.get_all_projects(testdb) - assert len(projects) == 2 - assert state1.branch.project in projects - assert state2.branch.project in projects + await testdb.commit() # Ensure changes are committed + + # Simulate the behavior of StateManager.list_projects + sm = StateManager(testdb) + sm.list_projects = AsyncMock( + return_value=[ + ( + MagicMock(hex=state1.branch.project.id.hex), # project_id + state1.branch.project.name, # project_name + MagicMock(hex=state1.branch.id.hex), # branch_id + state1.branch.name, # branch_name + MagicMock(hex=state1.id.hex), # state_id + 1, # step_index + "foo", # action + datetime(2021, 1, 1), # created_at + ), + ( + MagicMock(hex=state2.branch.project.id.hex), + state2.branch.project.name, + MagicMock(hex=state2.branch.id.hex), + state2.branch.name, + MagicMock(hex=state2.id.hex), + 1, + "bar", + datetime(2021, 1, 2), + ), + ] + ) + + with patch("core.cli.helpers.StateManager", return_value=sm): + await list_projects_json(testdb) + + captured = capsys.readouterr() + data = json.loads(captured.out) + + expected_output = [ + { + "id": state1.branch.project.id.hex, + "name": "Test Project 1", + "updated_at": "2021-01-01T00:00:00", + "branches": [ + {"id": state1.branch.id.hex, "name": state1.branch.name, "steps": [{"name": "Latest step", "step": 1}]} + ], + }, + { + "id": state2.branch.project.id.hex, + "name": "Test Project 2", + "updated_at": "2021-01-02T00:00:00", + "branches": [ + {"id": state2.branch.id.hex, "name": state2.branch.name, "steps": [{"name": "Latest step", "step": 1}]} + ], + }, + ] + + assert data == expected_output @pytest.mark.asyncio diff --git a/tests/state/test_state_manager.py b/tests/state/test_state_manager.py index 7ecfe781c7..49472abe90 100644 --- a/tests/state/test_state_manager.py +++ b/tests/state/test_state_manager.py @@ -31,7 +31,8 @@ async def test_create_project(mock_get_config, testmanager): assert sm.current_state == initial_state projects = await sm.list_projects() - assert projects == [project] + assert projects[0][0] == project.id + assert projects[0][1] == project.name @pytest.mark.asyncio @@ -54,7 +55,8 @@ async def test_delete_project(mock_get_config, testmanager): project = await sm.create_project("test") projects = await sm.list_projects() - assert projects == [project] + assert projects[0][0] == project.id + assert projects[0][1] == project.name await sm.delete_project(project.id) projects = await sm.list_projects() From c964373e5bcded4512c63637cf29b292c47bf25d Mon Sep 17 00:00:00 2001 From: LeonOstrez Date: Tue, 25 Mar 2025 10:52:21 +0100 Subject: [PATCH 06/43] fix creation of readme.md --- core/prompts/tech-writer/create_readme.prompt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/prompts/tech-writer/create_readme.prompt b/core/prompts/tech-writer/create_readme.prompt index 4d9e8d956a..48550906fc 100644 --- a/core/prompts/tech-writer/create_readme.prompt +++ b/core/prompts/tech-writer/create_readme.prompt @@ -2,11 +2,13 @@ You are working on a project called "{{ state.branch.project.name }}" and you ne {% include "partials/project_details.prompt" %} {% include "partials/features_list.prompt" %} -{% include "partials/files_list.prompt" %} -DO NOT specify commands to create any folders or files, they will be created automatically - just specify the relative path to file that needs to be written. +~~FILE_DESCRIPTIONS_IN_THE_CODEBASE~~ +{% include "partials/files_descriptions.prompt" %} +~~END_OF_FILE_DESCRIPTIONS_IN_THE_CODEBASE~~ -{% include "partials/relative_paths.prompt" %} +Project dependencies are installed using "npm install" in the root folder. That will install all the necessary dependencies in the root, client and server folders. +Project is started using "npm run start" in the root folder. That will start both the frontend and backend. Now, based on the project details provided, think step by step and create README.md file for this project. The file should have the following format: From a514b30f59da700fc1874f649953748cd33bec54 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Thu, 27 Feb 2025 16:17:51 +0100 Subject: [PATCH 07/43] Add more actions to agents --- core/agents/developer.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/core/agents/developer.py b/core/agents/developer.py index 7878d9635f..3e1ff8613c 100644 --- a/core/agents/developer.py +++ b/core/agents/developer.py @@ -72,13 +72,6 @@ class TaskSteps(BaseModel): steps: list[Step] -DEV_WAIT_TEST = "Awaiting user test" -DEV_TASK_STARTING = "Starting task #{}" -DEV_TASK_BREAKDOWN = "Task #{} breakdown" -DEV_TROUBLESHOOT = "Troubleshooting #{}" -DEV_TASK_REVIEW_FEEDBACK = "Task review feedback" - - class Developer(ChatWithBreakdownMixin, RelevantFilesMixin, BaseAgent): agent_type = "developer" display_name = "Developer" @@ -214,6 +207,8 @@ class Developer(ChatWithBreakdownMixin, RelevantFilesMixin, BaseAgent): # Check which files are relevant to the current task await self.get_relevant_files_parallel() + current_task_index = self.current_state.tasks.index(current_task) + await self.send_message("Thinking about how to implement this task ...") await self.ui.start_breakdown_stream() @@ -255,6 +250,7 @@ class Developer(ChatWithBreakdownMixin, RelevantFilesMixin, BaseAgent): # There might be state leftovers from previous tasks that we need to clean here self.next_state.modified_files = {} self.set_next_steps(response, source) + self.next_state.action = f"Task #{current_task_index + 1} start" await telemetry.trace_code_event( "task-start", { From 609a96c3fff64394866fc6d517ee70cb2432e9d2 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Tue, 4 Mar 2025 09:39:50 +0100 Subject: [PATCH 08/43] Add a function for restoring convo --- core/agents/bug_hunter.py | 12 +-- core/agents/code_monkey.py | 4 +- core/agents/developer.py | 7 ++ core/agents/executor.py | 5 +- core/agents/frontend.py | 7 +- core/agents/mixins.py | 18 +--- core/agents/spec_writer.py | 5 +- core/agents/task_completer.py | 3 +- core/agents/tech_lead.py | 8 +- core/agents/tech_writer.py | 3 +- core/agents/troubleshooter.py | 5 +- core/cli/helpers.py | 164 +++++++++++++++++++++++++++++++- core/cli/main.py | 14 ++- core/config/actions.py | 41 ++++++++ core/db/models/project.py | 16 +++- core/db/models/project_state.py | 20 +++- core/db/models/user_input.py | 24 ++++- core/state/state_manager.py | 20 ++++ 18 files changed, 314 insertions(+), 62 deletions(-) create mode 100644 core/config/actions.py diff --git a/core/agents/bug_hunter.py b/core/agents/bug_hunter.py index c9976c76bd..55b35c6441 100644 --- a/core/agents/bug_hunter.py +++ b/core/agents/bug_hunter.py @@ -9,6 +9,12 @@ from core.agents.convo import AgentConvo from core.agents.mixins import ChatWithBreakdownMixin, TestSteps from core.agents.response import AgentResponse from core.config import CHECK_LOGS_AGENT_NAME, magic_words +from core.config.actions import ( + BH_START_BUG_HUNT, + BH_START_USER_TEST, + BH_STARTING_PAIR_PROGRAMMING, + BH_WAIT_BUG_REP_INSTRUCTIONS, +) from core.config.constants import CONVO_ITERATIONS_LIMIT from core.db.models.project_state import IterationStatus from core.llm.parser import JSONParser @@ -45,12 +51,6 @@ class ImportantLogsForDebugging(BaseModel): logs: list[ImportantLog] = Field(description="Important logs that will help the human debug the current bug.") -BH_STARTING_PAIR_PROGRAMMING = "Start pair programming for task #{}" -BH_START_USER_TEST = "Start user testing for task #{}" -BH_WAIT_BUG_REP_INSTRUCTIONS = "Awaiting bug reproduction instructions for task #{}" -BH_START_BUG_HUNT = "Start bug hunt for task #{}" - - class BugHunter(ChatWithBreakdownMixin, BaseAgent): agent_type = "bug-hunter" display_name = "Bug Hunter" diff --git a/core/agents/code_monkey.py b/core/agents/code_monkey.py index bb7c2771c9..7353dd2055 100644 --- a/core/agents/code_monkey.py +++ b/core/agents/code_monkey.py @@ -11,6 +11,7 @@ from core.agents.convo import AgentConvo from core.agents.mixins import FileDiffMixin from core.agents.response import AgentResponse, ResponseType from core.config import CODE_MONKEY_AGENT_NAME, CODE_REVIEW_AGENT_NAME, DESCRIBE_FILES_AGENT_NAME +from core.config.actions import CM_UPDATE_FILES from core.db.models import File from core.llm.parser import JSONParser, OptionalCodeBlockParser from core.log import get_logger @@ -57,9 +58,6 @@ class FileDescription(BaseModel): ) -CM_UPDATE_FILES = "Updating files" - - class CodeMonkey(FileDiffMixin, BaseAgent): agent_type = "code-monkey" display_name = "Code Monkey" diff --git a/core/agents/developer.py b/core/agents/developer.py index 3e1ff8613c..979593d915 100644 --- a/core/agents/developer.py +++ b/core/agents/developer.py @@ -10,6 +10,13 @@ from core.agents.convo import AgentConvo from core.agents.mixins import ChatWithBreakdownMixin, RelevantFilesMixin from core.agents.response import AgentResponse from core.config import PARSE_TASK_AGENT_NAME, TASK_BREAKDOWN_AGENT_NAME +from core.config.actions import ( + DEV_TASK_BREAKDOWN, + DEV_TASK_REVIEW_FEEDBACK, + DEV_TASK_STARTING, + DEV_TROUBLESHOOT, + DEV_WAIT_TEST, +) from core.db.models.project_state import IterationStatus, TaskStatus from core.db.models.specification import Complexity from core.llm.parser import JSONParser diff --git a/core/agents/executor.py b/core/agents/executor.py index 24bdaa25fc..5068e5ca29 100644 --- a/core/agents/executor.py +++ b/core/agents/executor.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, Field from core.agents.base import BaseAgent from core.agents.convo import AgentConvo from core.agents.response import AgentResponse +from core.config.actions import EX_RUN_COMMAND, EX_SKIP_COMMAND from core.llm.parser import JSONParser from core.log import get_logger from core.proc.exec_log import ExecLog @@ -32,10 +33,6 @@ class CommandResult(BaseModel): ) -EX_SKIP_COMMAND = 'Skip "{}"' -EX_RUN_COMMAND = 'Run "{}"' - - class Executor(BaseAgent): agent_type = "executor" display_name = "Executor" diff --git a/core/agents/frontend.py b/core/agents/frontend.py index e4b0d90cff..cb8f84bf73 100644 --- a/core/agents/frontend.py +++ b/core/agents/frontend.py @@ -8,6 +8,7 @@ from core.agents.git import GitMixin from core.agents.mixins import FileDiffMixin from core.agents.response import AgentResponse from core.config import FRONTEND_AGENT_NAME +from core.config.actions import FE_CONTINUE, FE_INIT, FE_ITERATION, FE_ITERATION_DONE, FE_START from core.llm.parser import DescriptiveCodeBlockParser from core.log import get_logger from core.telemetry import telemetry @@ -16,12 +17,6 @@ from core.ui.base import ProjectStage log = get_logger(__name__) -FE_INIT = "Frontend init" -FE_START = "Frontend start" -FE_CONTINUE = "Frontend continue" -FE_ITERATION = "Frontend iteration" -FE_ITERATION_DONE = "Frontend iteration done" - class Frontend(FileDiffMixin, GitMixin, BaseAgent): agent_type = "frontend" diff --git a/core/agents/mixins.py b/core/agents/mixins.py index 578275901e..48ec9448ed 100644 --- a/core/agents/mixins.py +++ b/core/agents/mixins.py @@ -1,12 +1,12 @@ import asyncio import json -from difflib import unified_diff from typing import List, Optional from pydantic import BaseModel, Field from core.agents.convo import AgentConvo from core.agents.response import AgentResponse +from core.cli.helpers import get_line_changes from core.config import GET_RELEVANT_FILES_AGENT_NAME, TASK_BREAKDOWN_AGENT_NAME, TROUBLESHOOTER_BUG_REPORT from core.config.constants import CONVO_ITERATIONS_LIMIT from core.llm.parser import JSONParser @@ -191,18 +191,4 @@ class FileDiffMixin: :return: a tuple (added_lines, deleted_lines) """ - from_lines = old_content.splitlines(keepends=True) - to_lines = new_content.splitlines(keepends=True) - - diff_gen = unified_diff(from_lines, to_lines) - - added_lines = 0 - deleted_lines = 0 - - for line in diff_gen: - if line.startswith("+") and not line.startswith("+++"): # Exclude the file headers - added_lines += 1 - elif line.startswith("-") and not line.startswith("---"): # Exclude the file headers - deleted_lines += 1 - - return added_lines, deleted_lines + return get_line_changes(old_content, new_content) diff --git a/core/agents/spec_writer.py b/core/agents/spec_writer.py index 2a6d312b8a..8d2403fa4a 100644 --- a/core/agents/spec_writer.py +++ b/core/agents/spec_writer.py @@ -2,6 +2,7 @@ from core.agents.base import BaseAgent from core.agents.convo import AgentConvo from core.agents.response import AgentResponse, ResponseType from core.config import SPEC_WRITER_AGENT_NAME +from core.config.actions import SPEC_CHANGE_FEATURE_STEP_NAME, SPEC_CHANGE_STEP_NAME, SPEC_CREATE_STEP_NAME from core.db.models import Complexity from core.db.models.project_state import IterationStatus from core.llm.parser import StringParser @@ -14,10 +15,6 @@ ANALYZE_THRESHOLD = 1500 INITIAL_PROJECT_HOWTO_URL = ( "https://github.com/Pythagora-io/gpt-pilot/wiki/How-to-write-a-good-initial-project-description" ) -SPEC_CREATE_STEP_NAME = "Create specification" -SPEC_CHANGE_STEP_NAME = "Change specification" -SPEC_CHANGE_FEATURE_STEP_NAME = "Change specification due to new feature" - log = get_logger(__name__) diff --git a/core/agents/task_completer.py b/core/agents/task_completer.py index 0b13386dcd..88c01955a6 100644 --- a/core/agents/task_completer.py +++ b/core/agents/task_completer.py @@ -1,13 +1,12 @@ from core.agents.base import BaseAgent from core.agents.git import GitMixin from core.agents.response import AgentResponse +from core.config.actions import TC_TASK_DONE from core.log import get_logger from core.telemetry import telemetry log = get_logger(__name__) -TC_TASK_DONE = "Task #{} complete" - class TaskCompleter(BaseAgent, GitMixin): agent_type = "pythagora" diff --git a/core/agents/tech_lead.py b/core/agents/tech_lead.py index b935166f03..24cc46061d 100644 --- a/core/agents/tech_lead.py +++ b/core/agents/tech_lead.py @@ -9,6 +9,7 @@ from core.agents.convo import AgentConvo from core.agents.mixins import RelevantFilesMixin from core.agents.response import AgentResponse from core.config import TECH_LEAD_EPIC_BREAKDOWN, TECH_LEAD_PLANNING +from core.config.actions import TL_CREATE_INITIAL_EPIC, TL_CREATE_PLAN, TL_INITIAL_PROJECT_NAME, TL_START_FEATURE from core.db.models import Complexity from core.db.models.project_state import TaskStatus from core.llm.parser import JSONParser @@ -46,11 +47,6 @@ class EpicPlan(BaseModel): plan: list[Task] = Field(description="List of tasks that need to be done to implement the entire epic.") -TL_CREATE_INITIAL_EPIC = "Create initial project epic" -TL_CREATE_PLAN = "Create a development plan for epic: {}" -TL_START_FEATURE = "Start of feature #{}" - - class TechLead(RelevantFilesMixin, BaseAgent): agent_type = "tech-lead" display_name = "Tech Lead" @@ -94,7 +90,7 @@ class TechLead(RelevantFilesMixin, BaseAgent): self.next_state.epics = self.current_state.epics + [ { "id": uuid4().hex, - "name": "Initial Project", + "name": TL_INITIAL_PROJECT_NAME, "source": "app", "description": self.current_state.specification.description, "test_instructions": None, diff --git a/core/agents/tech_writer.py b/core/agents/tech_writer.py index 0dd97365ae..f951ccb513 100644 --- a/core/agents/tech_writer.py +++ b/core/agents/tech_writer.py @@ -1,14 +1,13 @@ from core.agents.base import BaseAgent from core.agents.convo import AgentConvo from core.agents.response import AgentResponse +from core.config.actions import TW_WRITE from core.db.models.project_state import TaskStatus from core.log import get_logger from core.ui.base import success_source log = get_logger(__name__) -TW_WRITE = "Write documentation" - class TechnicalWriter(BaseAgent): agent_type = "tech-writer" diff --git a/core/agents/troubleshooter.py b/core/agents/troubleshooter.py index c9dd51bd17..de78863c60 100644 --- a/core/agents/troubleshooter.py +++ b/core/agents/troubleshooter.py @@ -9,6 +9,7 @@ from core.agents.convo import AgentConvo from core.agents.mixins import ChatWithBreakdownMixin, IterationPromptMixin, RelevantFilesMixin, TestSteps from core.agents.response import AgentResponse from core.config import TROUBLESHOOTER_GET_RUN_COMMAND +from core.config.actions import TS_ALT_SOLUTION, TS_TASK_REVIEWED from core.db.models.file import File from core.db.models.project_state import IterationStatus, TaskStatus from core.llm.parser import JSONParser, OptionalCodeBlockParser @@ -31,10 +32,6 @@ class RouteFilePaths(BaseModel): files: list[str] = Field(description="List of paths for files that contain routes") -TS_TASK_REVIEWED = "Task #{} reviewed" -TS_ALT_SOLUTION = "Alternative solution (attempt #{})" - - class Troubleshooter(ChatWithBreakdownMixin, IterationPromptMixin, RelevantFilesMixin, BaseAgent): agent_type = "troubleshooter" display_name = "Troubleshooter" diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 953e7efe79..ccf95e14f5 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -3,22 +3,36 @@ import os import os.path import sys from argparse import ArgumentParser, ArgumentTypeError, Namespace +from difflib import unified_diff from typing import Optional from urllib.parse import urlparse from uuid import UUID from core.config import Config, LLMProvider, LocalIPCConfig, ProviderConfig, UIAdapter, get_config, loader +from core.config.actions import ( + BH_START_BUG_HUNT, + BH_START_USER_TEST, + BH_STARTING_PAIR_PROGRAMMING, + BH_WAIT_BUG_REP_INSTRUCTIONS, + CM_UPDATE_FILES, + DEV_TASK_BREAKDOWN, + DEV_TASK_STARTING, + DEV_TROUBLESHOOT, + TC_TASK_DONE, +) from core.config.env_importer import import_from_dotenv from core.config.version import get_version from core.db.session import SessionManager from core.db.setup import run_migrations -from core.log import setup +from core.log import get_logger, setup from core.state.state_manager import StateManager from core.ui.base import UIBase from core.ui.console import PlainConsoleUI from core.ui.ipc_client import IPCClientUI from core.ui.virtual import VirtualUI +log = get_logger(__name__) + def parse_llm_endpoint(value: str) -> Optional[tuple[LLMProvider, str]]: """ @@ -47,6 +61,35 @@ def parse_llm_endpoint(value: str) -> Optional[tuple[LLMProvider, str]]: return provider, url.geturl() +def get_line_changes(old_content: str, new_content: str) -> tuple[int, int]: + """ + Get the number of added and deleted lines between two files. + + This uses Python difflib to produce a unified diff, then counts + the number of added and deleted lines. + + :param old_content: old file content + :param new_content: new file content + :return: a tuple (added_lines, deleted_lines) + """ + + from_lines = old_content.splitlines(keepends=True) + to_lines = new_content.splitlines(keepends=True) + + diff_gen = unified_diff(from_lines, to_lines) + + added_lines = 0 + deleted_lines = 0 + + for line in diff_gen: + if line.startswith("+") and not line.startswith("+++"): # Exclude the file headers + added_lines += 1 + elif line.startswith("-") and not line.startswith("---"): # Exclude the file headers + deleted_lines += 1 + + return added_lines, deleted_lines + + def parse_llm_key(value: str) -> Optional[tuple[LLMProvider, str]]: """ Parse --llm-key command-line option. @@ -228,6 +271,125 @@ async def list_projects_json(db: SessionManager): print(json.dumps(data, indent=2)) +async def load_convo( + sm: StateManager, + project_id: Optional[UUID] = None, + step_index: Optional[int] = None, +) -> list: + """ + List all projects in the database. + """ + convo = [] + + project_states = await sm.get_project_states(project_id) + project_states = [state for state in project_states if 0 <= state.step_index <= step_index] + + branches = await sm.get_branches_for_project_id(project_id) + branch_id = branches[0].id + + task_counter = 1 + + for i, state in enumerate(project_states): + prev_state = project_states[i - 1] if i > 0 else None + + convo_el = {} + ui = await sm.find_user_input(state, branch_id) + + if ui is not None and ui.question is not None: + convo_el["question"] = ui.question + if ui.answer_text is not None: + convo_el["answer"] = str(ui.answer_text) + else: + convo_el["answer"] = str(ui.answer_button) + + if state.action is not None: + if DEV_TASK_STARTING[:-2] in state.action: + task_counter = int(state.action.split("#")[-1]) + + elif state.action == DEV_TROUBLESHOOT.format(task_counter): + if state.iterations is not None: + si = state.iterations[-1] + if si is not None: + if si["user_feedback"] is not None: + convo_el["user_feedback"] = si["user_feedback"] + if si["description"] is not None: + convo_el["description"] = si["description"] + + elif state.action == DEV_TASK_BREAKDOWN.format(task_counter): + task = state.tasks[task_counter - 1] + if "description" in task and task["description"] is not None: + convo_el["description"] = task["description"] + + if "instructions" in task and task["instructions"] is not None: + convo_el["instructions"] = task["instructions"] + + elif state.action == TC_TASK_DONE.format(task_counter): + if len(state.tasks) > 0: + task = state.tasks[task_counter - 1] + if "test_instructions" in task and task["test_instructions"] is not None: + convo_el["test_instructions"] = task["test_instructions"] + + elif state.action == CM_UPDATE_FILES: + files = {} + for steps in state.steps: + if "save_file" in steps and "path" in steps["save_file"]: + path = steps["save_file"]["path"] + files["path"] = path + + current_file = await sm.get_file_for_project(state.id, path) + prev_file = await sm.get_file_for_project(prev_state.id, path) + + if current_file and prev_file: + files["diff"] = get_line_changes( + old_content=prev_file.content.content, new_content=current_file.content.content + ) + + convo_el["files"] = files + + elif state.action == BH_START_BUG_HUNT.format(task_counter): + si = state.iterations[-1] + if si is not None: + if "user_feedback" in si and si["user_feedback"] is not None: + convo_el["user_feedback"] = si["user_feedback"] + + if "description" in si and si["description"] is not None: + convo_el["description"] = si["description"] + + elif state.action == BH_WAIT_BUG_REP_INSTRUCTIONS.format(task_counter): + if state.iterations is not None: + for si in state.iterations: + if "bug_reproduction_description" in si and si["bug_reproduction_description"] is not None: + convo_el["bug_reproduction_description"] = si["bug_reproduction_description"] + + elif state.action == BH_START_USER_TEST.format(task_counter): + si = state.iterations[-1] + if si is not None: + if "bug_hunting_cycles" in si and si["bug_hunting_cycles"] is not None: + cycle = si["bug_hunting_cycles"][-1] + if cycle is not None: + if "user_feedback" in cycle and cycle["user_feedback"] is not None: + convo_el["user_feedback"] = cycle["user_feedback"] + if ( + "human_readable_instructions" in cycle + and cycle["human_readable_instructions"] is not None + ): + convo_el["human_readable_instructions"] = cycle["human_readable_instructions"] + + elif state.action == BH_STARTING_PAIR_PROGRAMMING.format(task_counter): + si = state.iterations[-1] + if si is not None: + if "user_feedback" in si and si["user_feedback"] is not None: + convo_el["user_feedback"] = si["user_feedback"] + if "initial_explanation" in si and si["initial_explanation"] is not None: + convo_el["initial_explanation"] = si["initial_explanation"] + + if len(convo_el.keys()) > 0: + convo_el["action"] = state.action + convo.append(convo_el) + + return convo + + async def list_projects(db: SessionManager): """ List all projects in the database. diff --git a/core/cli/main.py b/core/cli/main.py index af7a841b4c..a3296a7811 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -14,7 +14,15 @@ except ImportError: SENTRY_AVAILABLE = False from core.agents.orchestrator import Orchestrator -from core.cli.helpers import delete_project, init, list_projects, list_projects_json, load_project, show_config +from core.cli.helpers import ( + delete_project, + init, + list_projects, + list_projects_json, + load_convo, + load_project, + show_config, +) from core.db.session import SessionManager from core.db.v0importer import LegacyDatabaseImporter from core.llm.anthropic_client import CustomAssertionError @@ -196,6 +204,10 @@ async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): :return: True if the application ran successfully, False otherwise. """ + if args.project and args.step: + convo = await load_convo(sm, args.project, args.step) + log.debug(f"Convo exists: {len(convo) > 0}") + if args.project or args.branch or args.step: telemetry.set("is_continuation", True) success = await load_project(sm, args.project, args.branch, args.step) diff --git a/core/config/actions.py b/core/config/actions.py new file mode 100644 index 0000000000..103b2e13e7 --- /dev/null +++ b/core/config/actions.py @@ -0,0 +1,41 @@ +BH_START_BUG_HUNT = "Start bug hunt for task #{}" +BH_WAIT_BUG_REP_INSTRUCTIONS = "Awaiting bug reproduction instructions for task #{}" +BH_START_USER_TEST = "Start user testing for task #{}" +BH_STARTING_PAIR_PROGRAMMING = "Start pair programming for task #{}" + +CM_UPDATE_FILES = "Updating files" + + +DEV_WAIT_TEST = "Awaiting user test" +DEV_TASK_STARTING = "Starting task #{}" +DEV_TASK_BREAKDOWN = "Task #{} breakdown" +DEV_TROUBLESHOOT = "Troubleshooting #{}" +DEV_TASK_REVIEW_FEEDBACK = "Task review feedback" + +TC_TASK_DONE = "Task #{} complete" + + +FE_INIT = "Frontend init" +FE_START = "Frontend start" +FE_CONTINUE = "Frontend continue" +FE_ITERATION = "Frontend iteration" +FE_ITERATION_DONE = "Frontend iteration done" + +TL_CREATE_INITIAL_EPIC = "Create initial project epic" +TL_CREATE_PLAN = "Create a development plan for epic: {}" +TL_START_FEATURE = "Start of feature #{}" +TL_INITIAL_PROJECT_NAME = "Initial Project" + +TW_WRITE = "Write documentation" + +EX_SKIP_COMMAND = 'Skip "{}"' +EX_RUN_COMMAND = 'Run "{}"' + +SPEC_CREATE_STEP_NAME = "Create specification" +SPEC_CHANGE_STEP_NAME = "Change specification" +SPEC_CHANGE_FEATURE_STEP_NAME = "Change specification due to new feature" + +TS_TASK_REVIEWED = "Task #{} reviewed" +TS_ALT_SOLUTION = "Alternative solution (attempt #{})" + +PS_EPIC_COMPLETE = "Epic {} completed" diff --git a/core/db/models/project.py b/core/db/models/project.py index 4f1a716aa1..547a6b34f2 100644 --- a/core/db/models/project.py +++ b/core/db/models/project.py @@ -9,7 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Mapped, mapped_column, relationship, selectinload from sqlalchemy.sql import func -from core.db.models import Base +from core.db.models import Base, File if TYPE_CHECKING: from core.db.models import Branch @@ -66,6 +66,20 @@ class Project(Base): result = await session.execute(select(Branch).where(Branch.project_id == self.id, Branch.name == name)) return result.scalar_one_or_none() + @staticmethod + async def get_file_for_project(session: AsyncSession, project_state_id: UUID, path: str) -> Optional["File"]: + file_result = await session.execute( + select(File).where(File.project_state_id == project_state_id, File.path == path) + ) + return file_result.scalar_one_or_none() + + @staticmethod + async def get_branches_for_project_id(session: AsyncSession, project_id: UUID) -> list["Branch"]: + from core.db.models import Branch + + branch_result = await session.execute(select(Branch).where(Branch.project_id == project_id)) + return branch_result.scalars().all() + @staticmethod async def get_all_projects(session: "AsyncSession") -> list["Project"]: """ diff --git a/core/db/models/project_state.py b/core/db/models/project_state.py index c65bdf6d38..49ce930ea2 100644 --- a/core/db/models/project_state.py +++ b/core/db/models/project_state.py @@ -3,12 +3,13 @@ from datetime import datetime from typing import TYPE_CHECKING, Optional, Union from uuid import UUID, uuid4 -from sqlalchemy import ForeignKey, UniqueConstraint, delete, inspect +from sqlalchemy import ForeignKey, UniqueConstraint, delete, inspect, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.sql import func +from core.config.actions import PS_EPIC_COMPLETE from core.db.models import Base, FileContent from core.log import get_logger @@ -46,10 +47,6 @@ class IterationStatus: DONE = "done" -PS_EPIC_COMPLETE = "Epic {} completed" -PS_TASK_COMPLETE = "Task {} completed" - - class ProjectState(Base): __tablename__ = "project_states" __table_args__ = ( @@ -218,6 +215,19 @@ class ProjectState(Base): step_index=1, ) + @staticmethod + async def get_project_states( + session: "AsyncSession", + project_id: UUID, + ) -> list["ProjectState"]: + from core.db.models import Branch, ProjectState + + branch = await session.execute(select(Branch).where(Branch.project_id == project_id)) + branch = branch.scalar_one_or_none() + + project_states_result = await session.execute(select(ProjectState).where(ProjectState.branch_id == branch.id)) + return project_states_result.scalars().all() + async def create_next_state(self) -> "ProjectState": """ Create the next project state for the branch. diff --git a/core/db/models/user_input.py b/core/db/models/user_input.py index c068143db4..a5493c61e9 100644 --- a/core/db/models/user_input.py +++ b/core/db/models/user_input.py @@ -2,7 +2,8 @@ from datetime import datetime from typing import TYPE_CHECKING, Optional from uuid import UUID -from sqlalchemy import ForeignKey, inspect +from sqlalchemy import ForeignKey, and_, inspect, select +from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.sql import func @@ -57,3 +58,24 @@ class UserInput(Base): ) session.add(obj) return obj + + @staticmethod + async def find_user_input(session: AsyncSession, project_state, branch_id) -> Optional["UserInput"]: + from core.db.models import UserInput + + user_input = await session.execute( + select(UserInput).where( + and_(UserInput.branch_id == branch_id, UserInput.project_state_id == project_state.id) + ) + ) + user_input = user_input.scalars().all() + + if user_input is None: + user_input = await session.execute( + select(UserInput).where( + and_(UserInput.branch_id == branch_id, UserInput.project_state_id == project_state.prev_state_id) + ) + ) + user_input = user_input.scalars().all() + + return user_input[0] if len(user_input) > 0 else None diff --git a/core/state/state_manager.py b/core/state/state_manager.py index 8921078db8..3259c5f9e0 100644 --- a/core/state/state_manager.py +++ b/core/state/state_manager.py @@ -78,6 +78,26 @@ class StateManager: async with self.session_manager as session: return await Project.get_all_projects(session) + async def get_convo(self): + async with self.session_manager as session: + return await Project.get_convo(session) + + async def get_project_states(self, project_id: UUID) -> list[ProjectState]: + async with self.session_manager as session: + return await ProjectState.get_project_states(session, project_id) + + async def get_branches_for_project_id(self, project_id: UUID) -> list[Branch]: + async with self.session_manager as session: + return await Project.get_branches_for_project_id(session, project_id) + + async def find_user_input(self, project_state, branch_id) -> Optional["UserInput"]: + async with self.session_manager as session: + return await UserInput.find_user_input(session, project_state, branch_id) + + async def get_file_for_project(self, state_id: UUID, path: str): + async with self.session_manager as session: + return await Project.get_file_for_project(session, state_id, path) + async def create_project(self, name: str, folder_name: Optional[str] = None) -> Project: """ Create a new project and set it as the current one. From d412e43696f6cc01fdc8a161fef9d164a862fa0b Mon Sep 17 00:00:00 2001 From: mijauexe Date: Wed, 26 Mar 2025 16:31:01 +0100 Subject: [PATCH 09/43] rendering messages WIP --- core/agents/developer.py | 5 +- core/agents/mixins.py | 3 +- core/agents/tech_lead.py | 10 ++- core/cli/helpers.py | 117 ++++++++++++++++++++++++++++---- core/cli/main.py | 89 +++++++++++++++++++++++- core/config/actions.py | 6 +- core/db/models/project_state.py | 1 + core/db/models/user_input.py | 4 +- core/state/state_manager.py | 4 +- core/ui/base.py | 10 ++- core/ui/console.py | 4 +- core/ui/ipc_client.py | 8 ++- core/ui/virtual.py | 4 +- 13 files changed, 234 insertions(+), 31 deletions(-) diff --git a/core/agents/developer.py b/core/agents/developer.py index 979593d915..ccce91d3d6 100644 --- a/core/agents/developer.py +++ b/core/agents/developer.py @@ -13,7 +13,7 @@ from core.config import PARSE_TASK_AGENT_NAME, TASK_BREAKDOWN_AGENT_NAME from core.config.actions import ( DEV_TASK_BREAKDOWN, DEV_TASK_REVIEW_FEEDBACK, - DEV_TASK_STARTING, + DEV_TASK_START, DEV_TROUBLESHOOT, DEV_WAIT_TEST, ) @@ -257,7 +257,7 @@ class Developer(ChatWithBreakdownMixin, RelevantFilesMixin, BaseAgent): # There might be state leftovers from previous tasks that we need to clean here self.next_state.modified_files = {} self.set_next_steps(response, source) - self.next_state.action = f"Task #{current_task_index + 1} start" + self.next_state.action = DEV_TASK_START.format({current_task_index + 1}) await telemetry.trace_code_event( "task-start", { @@ -325,7 +325,6 @@ class Developer(ChatWithBreakdownMixin, RelevantFilesMixin, BaseAgent): "task_index": task_index, } ) - self.next_state.action = DEV_TASK_STARTING.format(task_index) await self.send_message(f"Starting task #{task_index} with the description:\n\n" + description) if self.current_state.run_command: await self.ui.send_run_command(self.current_state.run_command) diff --git a/core/agents/mixins.py b/core/agents/mixins.py index 48ec9448ed..8a782b493e 100644 --- a/core/agents/mixins.py +++ b/core/agents/mixins.py @@ -8,6 +8,7 @@ from core.agents.convo import AgentConvo from core.agents.response import AgentResponse from core.cli.helpers import get_line_changes from core.config import GET_RELEVANT_FILES_AGENT_NAME, TASK_BREAKDOWN_AGENT_NAME, TROUBLESHOOTER_BUG_REPORT +from core.config.actions import MIX_BREAKDOWN_CHAT_PROMPT from core.config.constants import CONVO_ITERATIONS_LIMIT from core.llm.parser import JSONParser from core.log import get_logger @@ -56,7 +57,7 @@ class ChatWithBreakdownMixin: ) chat = await self.ask_question( - "Are you happy with the breakdown? Now is a good time to ask questions or suggest changes.", + MIX_BREAKDOWN_CHAT_PROMPT, buttons={"yes": "Yes, looks good!"}, default="yes", verbose=False, diff --git a/core/agents/tech_lead.py b/core/agents/tech_lead.py index 24cc46061d..956430cbe8 100644 --- a/core/agents/tech_lead.py +++ b/core/agents/tech_lead.py @@ -9,7 +9,13 @@ from core.agents.convo import AgentConvo from core.agents.mixins import RelevantFilesMixin from core.agents.response import AgentResponse from core.config import TECH_LEAD_EPIC_BREAKDOWN, TECH_LEAD_PLANNING -from core.config.actions import TL_CREATE_INITIAL_EPIC, TL_CREATE_PLAN, TL_INITIAL_PROJECT_NAME, TL_START_FEATURE +from core.config.actions import ( + TL_CREATE_INITIAL_EPIC, + TL_CREATE_PLAN, + TL_EDIT_DEV_PLAN, + TL_INITIAL_PROJECT_NAME, + TL_START_FEATURE, +) from core.db.models import Complexity from core.db.models.project_state import TaskStatus from core.llm.parser import JSONParser @@ -297,7 +303,7 @@ class TechLead(RelevantFilesMixin, BaseAgent): await self.ui.send_project_stage({"stage": ProjectStage.OPEN_PLAN}) response = await self.ask_question( - "Open and edit your development plan in the Progress tab", + TL_EDIT_DEV_PLAN, buttons={"done_editing": "I'm done editing, the plan looks good"}, default="done_editing", buttons_only=True, diff --git a/core/cli/helpers.py b/core/cli/helpers.py index ccf95e14f5..15c9776936 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -16,8 +16,8 @@ from core.config.actions import ( BH_WAIT_BUG_REP_INSTRUCTIONS, CM_UPDATE_FILES, DEV_TASK_BREAKDOWN, - DEV_TASK_STARTING, DEV_TROUBLESHOOT, + MIX_BREAKDOWN_CHAT_PROMPT, TC_TASK_DONE, ) from core.config.env_importer import import_from_dotenv @@ -271,6 +271,49 @@ async def list_projects_json(db: SessionManager): print(json.dumps(data, indent=2)) +def find_first_todo_task(tasks): + """ + Find the first task with status 'todo' from a list of tasks. + + :param tasks: List of task objects + :return: First task with status 'todo', or None if not found + """ + for task in tasks: + if task.get("status") == "todo": + return task + return None + + +def trim_logs(logs: str) -> str: + """ + Trim logs by removing everything after specific marker phrases. + + This function cuts off the string at the first occurrence of + "Here are the backend logs" or "Here are the frontend logs". + + :param logs: Log text to trim + :return: Trimmed log text with the marker phrase removed + """ + if not logs: + return "" + + # Define marker phrases + markers = ["Here are the backend logs", "Here are the frontend logs"] + + # Find the first occurrence of any marker + index = float("inf") + for marker in markers: + pos = logs.find(marker) + if pos != -1 and pos < index: + index = pos + + # If a marker was found, trim the string + if index != float("inf"): + return logs[:index] + + return logs + + async def load_convo( sm: StateManager, project_id: Optional[UUID] = None, @@ -293,18 +336,19 @@ async def load_convo( prev_state = project_states[i - 1] if i > 0 else None convo_el = {} - ui = await sm.find_user_input(state, branch_id) + convo_el["id"] = str(state.id) + user_inputs = await sm.find_user_input(state, branch_id) - if ui is not None and ui.question is not None: - convo_el["question"] = ui.question - if ui.answer_text is not None: - convo_el["answer"] = str(ui.answer_text) - else: - convo_el["answer"] = str(ui.answer_button) + if user_inputs: + convo_el["user_inputs"] = [] + for ui in user_inputs: + if ui.question: + answer = trim_logs(ui.answer_text) if ui.answer_text is not None else ui.answer_button + convo_el["user_inputs"].append({"question": ui.question, "answer": answer}) if state.action is not None: - if DEV_TASK_STARTING[:-2] in state.action: - task_counter = int(state.action.split("#")[-1]) + if "Task" in state.action and "start" in state.action: + task_counter = int(state.action.split("#")[1].split()[0]) elif state.action == DEV_TROUBLESHOOT.format(task_counter): if state.iterations is not None: @@ -324,25 +368,32 @@ async def load_convo( convo_el["instructions"] = task["instructions"] elif state.action == TC_TASK_DONE.format(task_counter): - if len(state.tasks) > 0: + if state.tasks: + # find the next task description and print it in question + next_task = find_first_todo_task(state.tasks) + # next_task_alt = find_first_todo_task(prev_state.tasks) + if next_task: + convo_el["task_description"] = next_task["description"] task = state.tasks[task_counter - 1] if "test_instructions" in task and task["test_instructions"] is not None: convo_el["test_instructions"] = task["test_instructions"] elif state.action == CM_UPDATE_FILES: - files = {} + files = [] for steps in state.steps: + file = {} if "save_file" in steps and "path" in steps["save_file"]: path = steps["save_file"]["path"] - files["path"] = path + file["path"] = path current_file = await sm.get_file_for_project(state.id, path) prev_file = await sm.get_file_for_project(prev_state.id, path) if current_file and prev_file: - files["diff"] = get_line_changes( + file["diff"] = get_line_changes( old_content=prev_file.content.content, new_content=current_file.content.content ) + files.append(file) convo_el["files"] = files @@ -383,7 +434,43 @@ async def load_convo( if "initial_explanation" in si and si["initial_explanation"] is not None: convo_el["initial_explanation"] = si["initial_explanation"] - if len(convo_el.keys()) > 0: + if convo_el.get("user_inputs", None) is not None or convo_el.get("files", None) is not None: + for j, ui in enumerate(user_inputs): + if ui.question and ui.question == MIX_BREAKDOWN_CHAT_PROMPT: + if len(state.tasks) == 1: + if state.tasks[0]["instructions"] is not None: + # convo_el["user_inputs"].insert(j, {'question': state.tasks[task_counter - 1]["instructions"]}) + convo_el["breakdown"] = state.tasks[task_counter - 1]["instructions"] + break + elif ( + state.tasks[task_counter - 1] is not None + and state.tasks[task_counter - 1]["instructions"] is not None + ): + # convo_el["user_inputs"].insert(j, {'question': state.tasks[task_counter - 1]["instructions"]}) + convo_el["breakdown"] = state.tasks[task_counter - 1]["instructions"] + break + elif state.iterations: + if ( + state.iterations[-1].get("bug_hunting_cycles", None) is not None + and state.iterations[-1]["bug_hunting_cycles"] + and state.iterations[-1]["bug_hunting_cycles"][-1].get("human_readable_instructions", None) + is not None + ): + # convo_el["user_inputs"].insert(j, {'question': state.iterations[-1]["bug_hunting_cycles"][-1]["human_readable_instructions"]}) + convo_el["breakdown"] = state.iterations[-1]["bug_hunting_cycles"][-1][ + "human_readable_instructions" + ] + break + else: + # try to get the next state's instructions from the task + next_state = project_states[i + 1] + if next_state.tasks is not None: + next_task = find_first_todo_task(next_state.tasks) + if next_task: + # convo_el["user_inputs"].insert(j, {'question': next_task["description"]}) + convo_el["task_description"] = next_task["description"] + break + convo_el["action"] = state.action convo.append(convo_el) diff --git a/core/cli/main.py b/core/cli/main.py index a3296a7811..cf1c26fa26 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -5,6 +5,8 @@ import sys from argparse import Namespace from asyncio import run +from core.config.actions import TL_EDIT_DEV_PLAN + try: import sentry_sdk from sentry_sdk.integrations.asyncio import AsyncioIntegration @@ -30,12 +32,23 @@ from core.llm.base import APIError from core.log import get_logger from core.state.state_manager import StateManager from core.telemetry import telemetry -from core.ui.base import ProjectStage, UIBase, UIClosedError, UserInput, pythagora_source +from core.ui.base import ( + ProjectStage, + UIBase, + UIClosedError, + UserInput, + bug_hunter_source, + developer_source, + history_source1, + history_source2, + pythagora_source, +) log = get_logger(__name__) telemetry_sent = False +source_alt = True def init_sentry(): @@ -194,6 +207,14 @@ async def start_new_project(sm: StateManager, ui: UIBase) -> bool: return project_state is not None +def alternate_source(): + """Toggle between pythagora_source and history_source1.""" + global source_alt + source = history_source1 if source_alt else history_source2 + source_alt = not source_alt + return source + + async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): """ Run a Pythagora session. @@ -206,6 +227,72 @@ async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): if args.project and args.step: convo = await load_convo(sm, args.project, args.step) + # jsonnnnn = json.dumps(convo, indent=4) + + for msg in convo: + if "breakdown" in msg: + await ui.send_message(msg["breakdown"], source=developer_source, project_state_id=msg["id"]) + + if "instructions" in msg: + await ui.send_message(msg["instructions"], source=bug_hunter_source, project_state_id=msg["id"]) + + if "task_description" in msg: + await ui.send_message(msg["task_description"], source=developer_source, project_state_id=msg["id"]) + + if "user_inputs" in msg and msg["user_inputs"]: + for input_item in msg["user_inputs"]: + # Process question if present + if "question" in input_item: + await ui.send_message( + input_item["question"], source=alternate_source(), project_state_id=msg["id"] + ) + + # Process answer if present + if "answer" in input_item: + if input_item["question"] == TL_EDIT_DEV_PLAN: + await ui.send_message( + "Starting task with the description:\n\n" + input_item["answer"], + project_state_id=msg["id"], + ) + else: + await ui.send_message( + input_item["answer"], source=alternate_source(), project_state_id=msg["id"] + ) + + if "test_instructions" in msg: + await ui.send_test_instructions( + msg["test_instructions"], project_state_id=msg["id"], source=alternate_source() + ) + + if "files" in msg: + source = alternate_source() + for f in msg["files"]: + await ui.send_file_status(f["path"], "done", source=source) + + # Process any other fields in the message + for key, value in msg.items(): + if ( + key + not in [ + "user_inputs", + "action", + "id", + "test_instructions", + "files", + "task_description", + "breakdown", + ] + and value + ): + if isinstance(value, str): + await ui.send_message(f"{key}: {value}", source=alternate_source(), project_state_id=msg["id"]) + elif isinstance(value, dict): + for k, v in value.items(): + if v and isinstance(v, str): + await ui.send_message( + f"{k}: {v}", source=alternate_source(), project_state_id=msg["id"] + ) + log.debug(f"Convo exists: {len(convo) > 0}") if args.project or args.branch or args.step: diff --git a/core/config/actions.py b/core/config/actions.py index 103b2e13e7..e01687a2e3 100644 --- a/core/config/actions.py +++ b/core/config/actions.py @@ -7,7 +7,7 @@ CM_UPDATE_FILES = "Updating files" DEV_WAIT_TEST = "Awaiting user test" -DEV_TASK_STARTING = "Starting task #{}" +DEV_TASK_START = "Task #{} start" DEV_TASK_BREAKDOWN = "Task #{} breakdown" DEV_TROUBLESHOOT = "Troubleshooting #{}" DEV_TASK_REVIEW_FEEDBACK = "Task review feedback" @@ -39,3 +39,7 @@ TS_TASK_REVIEWED = "Task #{} reviewed" TS_ALT_SOLUTION = "Alternative solution (attempt #{})" PS_EPIC_COMPLETE = "Epic {} completed" + +# other constants +TL_EDIT_DEV_PLAN = "Open and edit your development plan in the Progress tab" +MIX_BREAKDOWN_CHAT_PROMPT = "Are you happy with the breakdown? Now is a good time to ask questions or suggest changes." diff --git a/core/db/models/project_state.py b/core/db/models/project_state.py index 49ce930ea2..1eef30e3da 100644 --- a/core/db/models/project_state.py +++ b/core/db/models/project_state.py @@ -226,6 +226,7 @@ class ProjectState(Base): branch = branch.scalar_one_or_none() project_states_result = await session.execute(select(ProjectState).where(ProjectState.branch_id == branch.id)) + # project_states_result = await session.execute(select(ProjectState).where(and_(ProjectState.branch_id == branch.id), ProjectState.action.isnot(None))) return project_states_result.scalars().all() async def create_next_state(self) -> "ProjectState": diff --git a/core/db/models/user_input.py b/core/db/models/user_input.py index a5493c61e9..c262836bb8 100644 --- a/core/db/models/user_input.py +++ b/core/db/models/user_input.py @@ -60,7 +60,7 @@ class UserInput(Base): return obj @staticmethod - async def find_user_input(session: AsyncSession, project_state, branch_id) -> Optional["UserInput"]: + async def find_user_inputs(session: AsyncSession, project_state, branch_id) -> Optional[list["UserInput"]]: from core.db.models import UserInput user_input = await session.execute( @@ -78,4 +78,4 @@ class UserInput(Base): ) user_input = user_input.scalars().all() - return user_input[0] if len(user_input) > 0 else None + return user_input if len(user_input) > 0 else [] diff --git a/core/state/state_manager.py b/core/state/state_manager.py index 3259c5f9e0..0570dafefe 100644 --- a/core/state/state_manager.py +++ b/core/state/state_manager.py @@ -90,9 +90,9 @@ class StateManager: async with self.session_manager as session: return await Project.get_branches_for_project_id(session, project_id) - async def find_user_input(self, project_state, branch_id) -> Optional["UserInput"]: + async def find_user_input(self, project_state, branch_id) -> Optional[list["UserInput"]]: async with self.session_manager as session: - return await UserInput.find_user_input(session, project_state, branch_id) + return await UserInput.find_user_inputs(session, project_state, branch_id) async def get_file_for_project(self, state_id: UUID, path: str): async with self.session_manager as session: diff --git a/core/ui/base.py b/core/ui/base.py index de78fcf584..983925a1d9 100644 --- a/core/ui/base.py +++ b/core/ui/base.py @@ -368,7 +368,9 @@ class UIBase: """ raise NotImplementedError() - async def send_test_instructions(self, test_instructions: str, project_state_id: Optional[str] = None): + async def send_test_instructions( + self, test_instructions: str, project_state_id: Optional[str] = None, source: Optional[UISource] = None + ): """ Send test instructions. @@ -474,6 +476,12 @@ class UIBase: pythagora_source = UISource("Pythagora", "pythagora") + +developer_source = AgentSource("Developer", "developer") +bug_hunter_source = AgentSource("Bug Hunter", "bug-hunter") + +history_source1 = UISource("Developer", "developer") +history_source2 = UISource("Bug Hunter", "bug-hunter") success_source = UISource("Congratulations", "success") diff --git a/core/ui/console.py b/core/ui/console.py index 28fe5248a8..3e369a6281 100644 --- a/core/ui/console.py +++ b/core/ui/console.py @@ -169,7 +169,9 @@ class PlainConsoleUI(UIBase): async def send_project_stats(self, stats: dict): pass - async def send_test_instructions(self, test_instructions: str, project_state_id: Optional[str] = None): + async def send_test_instructions( + self, test_instructions: str, project_state_id: Optional[str] = None, source: Optional[UISource] = None + ): pass async def knowledge_base_update(self, knowledge_base: dict): diff --git a/core/ui/ipc_client.py b/core/ui/ipc_client.py index d839e279b9..8cc6445fc8 100644 --- a/core/ui/ipc_client.py +++ b/core/ui/ipc_client.py @@ -497,7 +497,12 @@ class IPCClientUI(UIBase): content=stats, ) - async def send_test_instructions(self, test_instructions: str, project_state_id: Optional[str] = None): + async def send_test_instructions( + self, + test_instructions: str, + project_state_id: Optional[str] = None, + source: Optional[UISource] = None, + ): try: log.debug("Sending test instructions") parsed_instructions = json.loads(test_instructions) @@ -511,6 +516,7 @@ class IPCClientUI(UIBase): "test_instructions": parsed_instructions, }, project_state_id=project_state_id, + category=source.type_name if source else None, ) async def knowledge_base_update(self, knowledge_base: dict): diff --git a/core/ui/virtual.py b/core/ui/virtual.py index 58467b34a9..503a5aa5d6 100644 --- a/core/ui/virtual.py +++ b/core/ui/virtual.py @@ -168,7 +168,9 @@ class VirtualUI(UIBase): async def send_project_stats(self, stats: dict): pass - async def send_test_instructions(self, test_instructions: str, project_state_id: Optional[str] = None): + async def send_test_instructions( + self, test_instructions: str, project_state_id: Optional[str] = None, source: Optional[UISource] = None + ): pass async def knowledge_base_update(self, knowledge_base: dict): From c91353083cfd0e538dd673b8f5db9960c9ce1a31 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Wed, 26 Mar 2025 17:00:14 +0100 Subject: [PATCH 10/43] Add new message type --- core/cli/main.py | 9 ++------- core/ui/base.py | 15 +++++++++++++++ core/ui/console.py | 11 +++++++++++ core/ui/ipc_client.py | 14 ++++++++++++++ core/ui/virtual.py | 8 ++++++++ 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/core/cli/main.py b/core/cli/main.py index cf1c26fa26..f06c27276e 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -249,13 +249,8 @@ async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): # Process answer if present if "answer" in input_item: - if input_item["question"] == TL_EDIT_DEV_PLAN: - await ui.send_message( - "Starting task with the description:\n\n" + input_item["answer"], - project_state_id=msg["id"], - ) - else: - await ui.send_message( + if input_item["question"] != TL_EDIT_DEV_PLAN: + await ui.send_user_input_history( input_item["answer"], source=alternate_source(), project_state_id=msg["id"] ) diff --git a/core/ui/base.py b/core/ui/base.py index 983925a1d9..5116b6fb47 100644 --- a/core/ui/base.py +++ b/core/ui/base.py @@ -122,6 +122,21 @@ class UIBase: """ raise NotImplementedError() + async def send_user_input_history( + self, + message: str, + source: Optional[UISource] = None, + project_state_id: Optional[str] = None, + ): + """ + Send a user input history (what the user said to Pythagora) message to the UI. + + :param message: Message content. + :param source: Source of the message (if any). + :param project_state_id: Current project state id. + """ + raise NotImplementedError() + async def send_message( self, message: str, diff --git a/core/ui/console.py b/core/ui/console.py index 3e369a6281..3c0c63e8a3 100644 --- a/core/ui/console.py +++ b/core/ui/console.py @@ -29,6 +29,17 @@ class PlainConsoleUI(UIBase): else: print(chunk, end="", flush=True) + async def send_user_input_history( + self, + message: str, + source: Optional[UISource] = None, + project_state_id: Optional[str] = None, + ): + if source: + print(f"[{source}] {message}") + else: + print(message) + async def send_message( self, message: str, diff --git a/core/ui/ipc_client.py b/core/ui/ipc_client.py index 8cc6445fc8..5aeac44724 100644 --- a/core/ui/ipc_client.py +++ b/core/ui/ipc_client.py @@ -56,6 +56,7 @@ class MessageType(str, Enum): KNOWLEDGE_BASE_UPDATE = "updatedKnowledgeBase" STOP_APP = "stopApp" TOKEN_EXPIRED = "tokenExpired" + USER_INPUT_HISTORY = "userInputHistory" class Message(BaseModel): @@ -210,6 +211,19 @@ class IPCClientUI(UIBase): project_state_id=project_state_id, ) + async def send_user_input_history( + self, + message: str, + source: Optional[UISource] = None, + project_state_id: Optional[str] = None, + ): + await self._send( + MessageType.USER_INPUT_HISTORY, + content=message, + category=source.type_name if source else None, + project_state_id=project_state_id, + ) + async def send_message( self, message: str, diff --git a/core/ui/virtual.py b/core/ui/virtual.py index 503a5aa5d6..2165d92451 100644 --- a/core/ui/virtual.py +++ b/core/ui/virtual.py @@ -30,6 +30,14 @@ class VirtualUI(UIBase): else: print(chunk, end="", flush=True) + async def send_user_input_history( + self, + message: str, + source: Optional[UISource] = None, + project_state_id: Optional[str] = None, + ): + raise NotImplementedError() + async def send_message( self, message: str, From d2735751790bc443f79c04f67ae2f3037dc00ff0 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Thu, 27 Mar 2025 09:31:17 +0100 Subject: [PATCH 11/43] Add file changes --- core/cli/helpers.py | 15 ++++++--------- core/cli/main.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 15c9776936..3e767795ae 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -371,7 +371,6 @@ async def load_convo( if state.tasks: # find the next task description and print it in question next_task = find_first_todo_task(state.tasks) - # next_task_alt = find_first_todo_task(prev_state.tasks) if next_task: convo_el["task_description"] = next_task["description"] task = state.tasks[task_counter - 1] @@ -389,10 +388,12 @@ async def load_convo( current_file = await sm.get_file_for_project(state.id, path) prev_file = await sm.get_file_for_project(prev_state.id, path) - if current_file and prev_file: - file["diff"] = get_line_changes( - old_content=prev_file.content.content, new_content=current_file.content.content - ) + file["diff"] = get_line_changes( + old_content=prev_file.content.content if prev_file else "", + new_content=current_file.content.content, + ) + file["old_content"] = prev_file.content.content if prev_file else "" + file["new_content"] = current_file.content.content files.append(file) convo_el["files"] = files @@ -439,14 +440,12 @@ async def load_convo( if ui.question and ui.question == MIX_BREAKDOWN_CHAT_PROMPT: if len(state.tasks) == 1: if state.tasks[0]["instructions"] is not None: - # convo_el["user_inputs"].insert(j, {'question': state.tasks[task_counter - 1]["instructions"]}) convo_el["breakdown"] = state.tasks[task_counter - 1]["instructions"] break elif ( state.tasks[task_counter - 1] is not None and state.tasks[task_counter - 1]["instructions"] is not None ): - # convo_el["user_inputs"].insert(j, {'question': state.tasks[task_counter - 1]["instructions"]}) convo_el["breakdown"] = state.tasks[task_counter - 1]["instructions"] break elif state.iterations: @@ -456,7 +455,6 @@ async def load_convo( and state.iterations[-1]["bug_hunting_cycles"][-1].get("human_readable_instructions", None) is not None ): - # convo_el["user_inputs"].insert(j, {'question': state.iterations[-1]["bug_hunting_cycles"][-1]["human_readable_instructions"]}) convo_el["breakdown"] = state.iterations[-1]["bug_hunting_cycles"][-1][ "human_readable_instructions" ] @@ -467,7 +465,6 @@ async def load_convo( if next_state.tasks is not None: next_task = find_first_todo_task(next_state.tasks) if next_task: - # convo_el["user_inputs"].insert(j, {'question': next_task["description"]}) convo_el["task_description"] = next_task["description"] break diff --git a/core/cli/main.py b/core/cli/main.py index f06c27276e..04745514cd 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -253,6 +253,10 @@ async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): await ui.send_user_input_history( input_item["answer"], source=alternate_source(), project_state_id=msg["id"] ) + else: + await ui.send_message( + input_item["question"], source=alternate_source(), project_state_id=msg["id"] + ) if "test_instructions" in msg: await ui.send_test_instructions( @@ -263,6 +267,14 @@ async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): source = alternate_source() for f in msg["files"]: await ui.send_file_status(f["path"], "done", source=source) + await ui.generate_diff( + file_path=f["path"], + file_old=f.get("old_content", ""), + file_new=f.get("new_content", ""), + n_new_lines=f["diff"][0], + n_del_lines=f["diff"][1], + source=source, + ) # Process any other fields in the message for key, value in msg.items(): From dfb55f14cab4326cce76d70109cc5f351e74f940 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Thu, 27 Mar 2025 10:38:39 +0100 Subject: [PATCH 12/43] Append only changed files --- core/cli/helpers.py | 6 +++++- core/cli/main.py | 2 -- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 3e767795ae..bb157f227f 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -394,7 +394,11 @@ async def load_convo( ) file["old_content"] = prev_file.content.content if prev_file else "" file["new_content"] = current_file.content.content - files.append(file) + + # hack that works fine because state.steps.completed is false until file is updated, but + # if we don't do this, we would have to compare to previous state which is complicated + if file["diff"] != (0, 0): + files.append(file) convo_el["files"] = files diff --git a/core/cli/main.py b/core/cli/main.py index 04745514cd..e6069d1291 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -241,13 +241,11 @@ async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): if "user_inputs" in msg and msg["user_inputs"]: for input_item in msg["user_inputs"]: - # Process question if present if "question" in input_item: await ui.send_message( input_item["question"], source=alternate_source(), project_state_id=msg["id"] ) - # Process answer if present if "answer" in input_item: if input_item["question"] != TL_EDIT_DEV_PLAN: await ui.send_user_input_history( From 2f4e2ed6a59716f6a6c15971080f62ac8198795c Mon Sep 17 00:00:00 2001 From: mijauexe Date: Thu, 27 Mar 2025 10:45:12 +0100 Subject: [PATCH 13/43] Load only last 50 states --- core/cli/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/cli/main.py b/core/cli/main.py index e6069d1291..d94a17ec6b 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -226,8 +226,7 @@ async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): """ if args.project and args.step: - convo = await load_convo(sm, args.project, args.step) - # jsonnnnn = json.dumps(convo, indent=4) + convo = (await load_convo(sm, args.project, args.step))[-50:] for msg in convo: if "breakdown" in msg: From 36202cb86a67e4882e6b102af2b7d628156103a9 Mon Sep 17 00:00:00 2001 From: LeonOstrez Date: Thu, 27 Mar 2025 11:37:45 +0100 Subject: [PATCH 14/43] version 1.3.7 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index afa8fc0b3a..9f6c93e55a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pythagora-core" -version = "1.3.6" +version = "1.3.7" description = "Build complete apps using AI agents" authors = ["Leon Ostrez "] license = "FSL-1.1-MIT" From 98dc02f690de0cb0669fea9757f25f209f49341d Mon Sep 17 00:00:00 2001 From: mijauexe Date: Thu, 27 Mar 2025 18:08:38 +0100 Subject: [PATCH 15/43] Simplify project load --- core/cli/helpers.py | 38 ++++++++++++-------------------------- core/db/models/project.py | 19 +------------------ tests/cli/test_cli.py | 36 ++++++++---------------------------- tests/db/test_project.py | 31 +++++++++++-------------------- 4 files changed, 32 insertions(+), 92 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 8ef9b887cf..ddcc51f980 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -3,7 +3,6 @@ import os import os.path import sys from argparse import ArgumentParser, ArgumentTypeError, Namespace -from collections import defaultdict from typing import Optional from urllib.parse import urlparse from uuid import UUID @@ -197,32 +196,19 @@ async def list_projects_json(db: SessionManager): """ sm = StateManager(db) projects = await sm.list_projects() - projects_dict = defaultdict(lambda: {"branches": [], "updated_at": None}) + projects_list = [] for row in projects: - project_id, project_name, branch_id, branch_name, state_id, step_index, action, created_at = row - - project = projects_dict[project_id] - project["id"] = project_id.hex - project["name"] = project_name - - branch = next((b for b in project["branches"] if b["id"] == branch_id.hex), None) - if not branch: - branch = {"id": branch_id.hex, "name": branch_name, "steps": []} - project["branches"].append(branch) - - if created_at and (project["updated_at"] is None or created_at > project["updated_at"]): - project["updated_at"] = created_at - - branch["steps"].append({"name": action, "step": step_index}) - - for project in projects_dict.values(): - for branch in project["branches"]: - if branch["steps"]: - branch["steps"][-1]["name"] = "Latest step" - - project["updated_at"] = project["updated_at"].isoformat() if project["updated_at"] else None - - print(json.dumps(list(projects_dict.values()), indent=2, default=str)) + project_id, project_name, created_at, folder_name = row + projects_list.append( + { + "id": project_id.hex, + "name": project_name, + "folder_name": folder_name, + "updated_at": created_at.isoformat(), + } + ) + + print(json.dumps(projects_list, indent=2, default=str)) async def list_projects(db: SessionManager): diff --git a/core/db/models/project.py b/core/db/models/project.py index 33a6d20f9f..c0bb519528 100644 --- a/core/db/models/project.py +++ b/core/db/models/project.py @@ -68,24 +68,7 @@ class Project(Base): @staticmethod async def get_all_projects(session: "AsyncSession") -> list[Row]: - from core.db.models import Branch, ProjectState - - query = ( - select( - Project.id, - Project.name, - Branch.id, - Branch.name, - ProjectState.id, - ProjectState.step_index, - ProjectState.action, - ProjectState.created_at, - ) - .join(Branch, Project.branches) - .join(ProjectState, Branch.id == ProjectState.branch_id) - .where(ProjectState.action.isnot(None)) - .order_by(Project.id, Branch.id, ProjectState.step_index.asc()) - ) + query = select(Project.id, Project.name, Project.created_at, Project.folder_name).order_by(Project.name) result = await session.execute(query) return result.fetchall() diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index f19309675b..284fbf4f79 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -167,22 +167,15 @@ def test_show_default_config(capsys): async def test_list_projects_json(mock_StateManager, capsys): sm = mock_StateManager.return_value - # Mock the return value to match the expected format - # Format: project_id, project_name, branch_id, branch_name, state_id, step_index, action, created_at - project_id = MagicMock(hex="abcd") - branch_id = MagicMock(hex="1234") - state_id1 = MagicMock(hex="state1") - state_id2 = MagicMock(hex="state2") - state_id3 = MagicMock(hex="state3") + project_id1 = MagicMock(hex="abcd") + project_id2 = MagicMock(hex="efgh") created_at1 = datetime(2021, 1, 1) created_at2 = datetime(2021, 1, 2) - created_at3 = datetime(2021, 1, 3) projects_data = [ - (project_id, "project1", branch_id, "branch1", state_id1, 1, "foo", created_at1), - (project_id, "project1", branch_id, "branch1", state_id2, 2, None, created_at2), - (project_id, "project1", branch_id, "branch1", state_id3, 3, "baz", created_at3), + (project_id1, "project1", created_at1, "folder1"), + (project_id2, "project2", created_at2, "folder2"), ] sm.list_projects = AsyncMock(return_value=projects_data) @@ -191,25 +184,12 @@ async def test_list_projects_json(mock_StateManager, capsys): mock_StateManager.assert_called_once_with(None) sm.list_projects.assert_awaited_once_with() - data = json.loads(capsys.readouterr().out) + captured = capsys.readouterr().out + data = json.loads(captured) assert data == [ - { - "name": "project1", - "id": "abcd", - "updated_at": "2021-01-03T00:00:00", - "branches": [ - { - "name": "branch1", - "id": "1234", - "steps": [ - {"step": 1, "name": "foo"}, - {"step": 2, "name": None}, - {"step": 3, "name": "Latest step"}, - ], - }, - ], - }, + {"id": "abcd", "name": "project1", "folder_name": "folder1", "updated_at": "2021-01-01T00:00:00"}, + {"id": "efgh", "name": "project2", "folder_name": "folder2", "updated_at": "2021-01-02T00:00:00"}, ] diff --git a/tests/db/test_project.py b/tests/db/test_project.py index 7424e0bfd3..1ef16242dc 100644 --- a/tests/db/test_project.py +++ b/tests/db/test_project.py @@ -85,29 +85,24 @@ async def test_get_all_projects(testdb, capsys): testdb.add(state2) await testdb.commit() # Ensure changes are committed - # Simulate the behavior of StateManager.list_projects + # Set folder names for the test + folder_name1 = "folder1" + folder_name2 = "folder2" + sm = StateManager(testdb) sm.list_projects = AsyncMock( return_value=[ ( - MagicMock(hex=state1.branch.project.id.hex), # project_id - state1.branch.project.name, # project_name - MagicMock(hex=state1.branch.id.hex), # branch_id - state1.branch.name, # branch_name - MagicMock(hex=state1.id.hex), # state_id - 1, # step_index - "foo", # action - datetime(2021, 1, 1), # created_at + MagicMock(hex=state1.branch.project.id.hex), + state1.branch.project.name, + datetime(2021, 1, 1), + folder_name1, ), ( MagicMock(hex=state2.branch.project.id.hex), state2.branch.project.name, - MagicMock(hex=state2.branch.id.hex), - state2.branch.name, - MagicMock(hex=state2.id.hex), - 1, - "bar", datetime(2021, 1, 2), + folder_name2, ), ] ) @@ -122,18 +117,14 @@ async def test_get_all_projects(testdb, capsys): { "id": state1.branch.project.id.hex, "name": "Test Project 1", + "folder_name": folder_name1, "updated_at": "2021-01-01T00:00:00", - "branches": [ - {"id": state1.branch.id.hex, "name": state1.branch.name, "steps": [{"name": "Latest step", "step": 1}]} - ], }, { "id": state2.branch.project.id.hex, "name": "Test Project 2", + "folder_name": folder_name2, "updated_at": "2021-01-02T00:00:00", - "branches": [ - {"id": state2.branch.id.hex, "name": state2.branch.name, "steps": [{"name": "Latest step", "step": 1}]} - ], }, ] From 7777a53b0cdbf9ec07747ff9ba0eeb57b3afabed Mon Sep 17 00:00:00 2001 From: mijauexe Date: Thu, 27 Mar 2025 18:47:48 +0100 Subject: [PATCH 16/43] Remove wrong doc --- core/state/state_manager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/state/state_manager.py b/core/state/state_manager.py index 5de4de9552..f0d2f5454b 100644 --- a/core/state/state_manager.py +++ b/core/state/state_manager.py @@ -71,9 +71,7 @@ class StateManager: async def list_projects(self) -> list[Row]: """ - List projects with branches - - :return: List of projects with all their branches and project states + :return: List of projects """ async with self.session_manager as session: return await Project.get_all_projects(session) From cf0409d28a22d26c3f20106ee2040b4d0af4e790 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Mon, 31 Mar 2025 14:25:41 +0200 Subject: [PATCH 17/43] Fix most of the history print errors --- core/agents/bug_hunter.py | 9 +- core/agents/developer.py | 3 +- core/agents/executor.py | 6 +- core/agents/frontend.py | 14 +- core/agents/human_input.py | 3 +- core/agents/troubleshooter.py | 6 +- core/cli/helpers.py | 227 ++++++++++++++++++++++---------- core/cli/main.py | 94 +------------ core/config/actions.py | 12 ++ core/db/models/project_state.py | 23 +++- core/db/models/user_input.py | 9 -- core/state/state_manager.py | 8 +- core/ui/base.py | 6 - 13 files changed, 223 insertions(+), 197 deletions(-) diff --git a/core/agents/bug_hunter.py b/core/agents/bug_hunter.py index 55b35c6441..e6bc0554f5 100644 --- a/core/agents/bug_hunter.py +++ b/core/agents/bug_hunter.py @@ -10,6 +10,9 @@ from core.agents.mixins import ChatWithBreakdownMixin, TestSteps from core.agents.response import AgentResponse from core.config import CHECK_LOGS_AGENT_NAME, magic_words from core.config.actions import ( + BH_ADDITIONAL_FEEDBACK, + BH_HUMAN_TEST_AGAIN, + BH_IS_BUG_FIXED, BH_START_BUG_HUNT, BH_START_USER_TEST, BH_STARTING_PAIR_PROGRAMMING, @@ -162,7 +165,7 @@ class BugHunter(ChatWithBreakdownMixin, BaseAgent): await self.ui.send_run_command(self.current_state.run_command) await self.ask_question( - "Please test the app again.", + BH_HUMAN_TEST_AGAIN, buttons={"done": "I am done testing"}, buttons_only=True, default="continue", @@ -173,7 +176,7 @@ class BugHunter(ChatWithBreakdownMixin, BaseAgent): if awaiting_user_test: buttons = {"yes": "Yes, the issue is fixed", "no": "No", "start_pair_programming": "Start Pair Programming"} user_feedback = await self.ask_question( - "Is the bug you reported fixed now?", + BH_IS_BUG_FIXED, buttons=buttons, default="yes", buttons_only=True, @@ -201,7 +204,7 @@ class BugHunter(ChatWithBreakdownMixin, BaseAgent): } ) user_feedback = await self.ask_question( - "Please add any additional feedback that could help Pythagora solve this bug", + BH_ADDITIONAL_FEEDBACK, buttons=buttons, default="continue", extra_info="collect_logs", diff --git a/core/agents/developer.py b/core/agents/developer.py index ccce91d3d6..50d0752db0 100644 --- a/core/agents/developer.py +++ b/core/agents/developer.py @@ -11,6 +11,7 @@ from core.agents.mixins import ChatWithBreakdownMixin, RelevantFilesMixin from core.agents.response import AgentResponse from core.config import PARSE_TASK_AGENT_NAME, TASK_BREAKDOWN_AGENT_NAME from core.config.actions import ( + DEV_EXECUTE_TASK, DEV_TASK_BREAKDOWN, DEV_TASK_REVIEW_FEEDBACK, DEV_TASK_START, @@ -329,7 +330,7 @@ class Developer(ChatWithBreakdownMixin, RelevantFilesMixin, BaseAgent): if self.current_state.run_command: await self.ui.send_run_command(self.current_state.run_command) user_response = await self.ask_question( - "Do you want to execute the above task?", + DEV_EXECUTE_TASK, buttons=buttons, default="yes", buttons_only=True, diff --git a/core/agents/executor.py b/core/agents/executor.py index 5068e5ca29..5af478a1fb 100644 --- a/core/agents/executor.py +++ b/core/agents/executor.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, Field from core.agents.base import BaseAgent from core.agents.convo import AgentConvo from core.agents.response import AgentResponse -from core.config.actions import EX_RUN_COMMAND, EX_SKIP_COMMAND +from core.config.actions import EX_RUN_COMMAND, EX_SKIP_COMMAND, RUN_COMMAND from core.llm.parser import JSONParser from core.log import get_logger from core.proc.exec_log import ExecLog @@ -79,9 +79,9 @@ class Executor(BaseAgent): timeout = options.get("timeout") if timeout: - q = f"Can I run command: {cmd} with {timeout}s timeout?" + q = f"{RUN_COMMAND} {cmd} with {timeout}s timeout?" else: - q = f"Can I run command: {cmd}?" + q = f"{RUN_COMMAND} {cmd}?" confirm = await self.ask_question( q, diff --git a/core/agents/frontend.py b/core/agents/frontend.py index cb8f84bf73..d773d7bc46 100644 --- a/core/agents/frontend.py +++ b/core/agents/frontend.py @@ -8,7 +8,15 @@ from core.agents.git import GitMixin from core.agents.mixins import FileDiffMixin from core.agents.response import AgentResponse from core.config import FRONTEND_AGENT_NAME -from core.config.actions import FE_CONTINUE, FE_INIT, FE_ITERATION, FE_ITERATION_DONE, FE_START +from core.config.actions import ( + FE_CHANGE_REQ, + FE_CONTINUE, + FE_DONE_WITH_UI, + FE_INIT, + FE_ITERATION, + FE_ITERATION_DONE, + FE_START, +) from core.llm.parser import DescriptiveCodeBlockParser from core.log import get_logger from core.telemetry import telemetry @@ -171,7 +179,7 @@ class Frontend(FileDiffMixin, GitMixin, BaseAgent): await self.ui.send_project_stage({"stage": ProjectStage.ITERATE_FRONTEND}) answer = await self.ask_question( - "Do you want to change anything or report a bug? Keep in mind that currently ONLY frontend is implemented.", + FE_CHANGE_REQ, buttons={ "yes": "I'm done building the UI", }, @@ -182,7 +190,7 @@ class Frontend(FileDiffMixin, GitMixin, BaseAgent): if answer.button == "yes": answer = await self.ask_question( - "Are you sure you're done building the UI and want to start building the backend functionality now?", + FE_DONE_WITH_UI, buttons={ "yes": "Yes, let's build the backend", "no": "No, continue working on the UI", diff --git a/core/agents/human_input.py b/core/agents/human_input.py index 3c0ac2f4e5..415d826a1b 100644 --- a/core/agents/human_input.py +++ b/core/agents/human_input.py @@ -1,5 +1,6 @@ from core.agents.base import BaseAgent from core.agents.response import AgentResponse, ResponseType +from core.config.actions import HUMAN_INTERVENTION_QUESTION class HumanInput(BaseAgent): @@ -16,7 +17,7 @@ class HumanInput(BaseAgent): description = step["human_intervention_description"] await self.ask_question( - f"I need human intervention: {description}", + f"{HUMAN_INTERVENTION_QUESTION} {description}", buttons={"continue": "Continue"}, default="continue", buttons_only=True, diff --git a/core/agents/troubleshooter.py b/core/agents/troubleshooter.py index de78863c60..48ca70ac0d 100644 --- a/core/agents/troubleshooter.py +++ b/core/agents/troubleshooter.py @@ -9,7 +9,7 @@ from core.agents.convo import AgentConvo from core.agents.mixins import ChatWithBreakdownMixin, IterationPromptMixin, RelevantFilesMixin, TestSteps from core.agents.response import AgentResponse from core.config import TROUBLESHOOTER_GET_RUN_COMMAND -from core.config.actions import TS_ALT_SOLUTION, TS_TASK_REVIEWED +from core.config.actions import TS_ALT_SOLUTION, TS_APP_WORKING, TS_DESCRIBE_ISSUE, TS_TASK_REVIEWED from core.db.models.file import File from core.db.models.project_state import IterationStatus, TaskStatus from core.llm.parser import JSONParser, OptionalCodeBlockParser @@ -262,7 +262,7 @@ class Troubleshooter(ChatWithBreakdownMixin, IterationPromptMixin, RelevantFiles while True: await self.ui.send_project_stage({"stage": ProjectStage.GET_USER_FEEDBACK}) - test_message = "Please check if the app is working" + test_message = TS_APP_WORKING if user_instructions: hint = " Here is a description of what should be working:\n\n" + user_instructions @@ -304,7 +304,7 @@ class Troubleshooter(ChatWithBreakdownMixin, IterationPromptMixin, RelevantFiles elif user_response.button == "bug": await self.ui.send_project_stage({"stage": ProjectStage.DESCRIBE_ISSUE}) user_description = await self.ask_question( - "Please describe the issue you found (one at a time) and share any relevant server logs", + TS_DESCRIBE_ISSUE, extra_info="collect_logs", buttons={"back": "Back"}, ) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index bb157f227f..6752d13575 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -10,15 +10,26 @@ from uuid import UUID from core.config import Config, LLMProvider, LocalIPCConfig, ProviderConfig, UIAdapter, get_config, loader from core.config.actions import ( + BH_ADDITIONAL_FEEDBACK, + BH_HUMAN_TEST_AGAIN, + BH_IS_BUG_FIXED, BH_START_BUG_HUNT, BH_START_USER_TEST, BH_STARTING_PAIR_PROGRAMMING, BH_WAIT_BUG_REP_INSTRUCTIONS, CM_UPDATE_FILES, DEV_TASK_BREAKDOWN, + DEV_TASK_START, DEV_TROUBLESHOOT, + FE_CHANGE_REQ, + FE_DONE_WITH_UI, + HUMAN_INTERVENTION_QUESTION, MIX_BREAKDOWN_CHAT_PROMPT, + RUN_COMMAND, TC_TASK_DONE, + TL_EDIT_DEV_PLAN, + TS_APP_WORKING, + TS_DESCRIBE_ISSUE, ) from core.config.env_importer import import_from_dotenv from core.config.version import get_version @@ -26,7 +37,7 @@ from core.db.session import SessionManager from core.db.setup import run_migrations from core.log import get_logger, setup from core.state.state_manager import StateManager -from core.ui.base import UIBase +from core.ui.base import AgentSource, UIBase, UISource from core.ui.console import PlainConsoleUI from core.ui.ipc_client import IPCClientUI from core.ui.virtual import VirtualUI @@ -314,21 +325,107 @@ def trim_logs(logs: str) -> str: return logs +def get_source_for_history(msg_type: Optional[str] = "", question: Optional[str] = ""): + if question in [TL_EDIT_DEV_PLAN]: + return AgentSource("Tech Lead", "tech-lead") + + if question in [FE_CHANGE_REQ, FE_DONE_WITH_UI]: + return AgentSource("Frontend", "frontend") + + elif question in [ + TS_DESCRIBE_ISSUE, + BH_HUMAN_TEST_AGAIN, + BH_IS_BUG_FIXED, + TS_APP_WORKING, + BH_ADDITIONAL_FEEDBACK, + ] or msg_type in ["instructions", "bh_breakdown"]: + return AgentSource("Bug Hunter", "bug-hunter") + + elif msg_type in ["bug_reproduction_instructions", "bug_description"]: + return AgentSource("Troubleshooter", "troubleshooter") + + elif HUMAN_INTERVENTION_QUESTION in question: + return AgentSource("Human Input", "human-input") + + elif RUN_COMMAND in question: + return AgentSource("Executor", "executor") + + elif msg_type in ["task_description", "task_breakdown"]: + return AgentSource("Developer", "developer") + + else: + return UISource("Pythagora", "pythagora") + + +async def print_convo( + ui: UIBase, + convo: list, +): + for msg in convo: + if "bh_breakdown" in msg: + await ui.send_message( + msg["bh_breakdown"], + source=get_source_for_history(msg_type="bh_breakdown"), + project_state_id=msg["id"], + ) + + if "task_description" in msg: + await ui.send_message( + msg["task_description"], + source=get_source_for_history(msg_type="task_description"), + project_state_id=msg["id"], + ) + + if "task_breakdown" in msg: + await ui.send_message( + msg["task_breakdown"], + source=get_source_for_history(msg_type="task_breakdown"), + project_state_id=msg["id"], + ) + + if "test_instructions" in msg: + await ui.send_test_instructions( + msg["test_instructions"], + source=get_source_for_history("test_instructions"), + project_state_id=msg["id"], + ) + + if "files" in msg: + for f in msg["files"]: + await ui.send_file_status(f["path"], "done") + await ui.generate_diff( + file_path=f["path"], + file_old=f.get("old_content", ""), + file_new=f.get("new_content", ""), + n_new_lines=f["diff"][0], + n_del_lines=f["diff"][1], + ) + + if "user_inputs" in msg and msg["user_inputs"]: + for input_item in msg["user_inputs"]: + if "question" in input_item: + await ui.send_message( + input_item["question"], + source=get_source_for_history(question=input_item["question"]), + project_state_id=msg["id"], + ) + + if "answer" in input_item: + if input_item["question"] != TL_EDIT_DEV_PLAN: + await ui.send_user_input_history(input_item["answer"], project_state_id=msg["id"]) + + async def load_convo( sm: StateManager, project_id: Optional[UUID] = None, - step_index: Optional[int] = None, + branch_id: Optional[UUID] = None, ) -> list: """ List all projects in the database. """ convo = [] - project_states = await sm.get_project_states(project_id) - project_states = [state for state in project_states if 0 <= state.step_index <= step_index] - - branches = await sm.get_branches_for_project_id(project_id) - branch_id = branches[0].id + project_states = await sm.get_project_states(project_id, branch_id) task_counter = 1 @@ -343,7 +440,39 @@ async def load_convo( convo_el["user_inputs"] = [] for ui in user_inputs: if ui.question: + if ui.question == MIX_BREAKDOWN_CHAT_PROMPT: + if len(state.iterations) > 0: + # as it's not available in the current state, take the next state's description - that is the bug description! + next_state = project_states[i + 1] if i + 1 < len(project_states) else None + if next_state is not None and next_state.iterations is not None: + si = next_state.iterations[-1] + if si is not None: + if si.get("description", None) is not None: + convo_el["bh_breakdown"] = si["description"] + else: + # if there are no iterations, it means developer made task breakdown, take the next state's first task with status = todo + next_state = project_states[i + 1] if i + 1 < len(project_states) else None + if next_state is not None: + task = find_first_todo_task(next_state.tasks) + if task.get("test_instructions", None) is not None: + convo_el["test_instructions"] = task["test_instructions"] + if task.get("instructions", None) is not None: + convo_el["task_breakdown"] = task["instructions"] + + if ui.question == BH_HUMAN_TEST_AGAIN: + if state.iterations: + if state.iterations[0].get("bug_reproduction_description", None) is not None: + convo_el["bug_reproduction_description"] = state.iterations[0][ + "bug_reproduction_description" + ] + answer = trim_logs(ui.answer_text) if ui.answer_text is not None else ui.answer_button + if answer == "bug": + answer = "There is an issue" + elif answer == "continue": + answer = "Everything works" + elif answer == "change": + answer = "I want to make a change" convo_el["user_inputs"].append({"question": ui.question, "answer": answer}) if state.action is not None: @@ -351,31 +480,32 @@ async def load_convo( task_counter = int(state.action.split("#")[1].split()[0]) elif state.action == DEV_TROUBLESHOOT.format(task_counter): - if state.iterations is not None: + if state.iterations is not None and len(state.iterations) > 0: si = state.iterations[-1] if si is not None: - if si["user_feedback"] is not None: + if si.get("user_feedback", None) is not None: convo_el["user_feedback"] = si["user_feedback"] - if si["description"] is not None: + if si.get("description", None) is not None: convo_el["description"] = si["description"] elif state.action == DEV_TASK_BREAKDOWN.format(task_counter): task = state.tasks[task_counter - 1] - if "description" in task and task["description"] is not None: + if task.get("description", None) is not None: convo_el["description"] = task["description"] - if "instructions" in task and task["instructions"] is not None: + if task.get("instructions", None) is not None: convo_el["instructions"] = task["instructions"] elif state.action == TC_TASK_DONE.format(task_counter): if state.tasks: - # find the next task description and print it in question next_task = find_first_todo_task(state.tasks) - if next_task: - convo_el["task_description"] = next_task["description"] - task = state.tasks[task_counter - 1] - if "test_instructions" in task and task["test_instructions"] is not None: - convo_el["test_instructions"] = task["test_instructions"] + if next_task is not None and next_task.get("description", None) is not None: + convo_el["task_description"] = f"Task #{task_counter} - " + next_task["description"] + + elif state.action == DEV_TASK_START: + task = state.tasks[task_counter - 1] + if task.get("instructions", None) is not None: + convo_el["task_breakdown"] = task["instructions"] elif state.action == CM_UPDATE_FILES: files = [] @@ -402,25 +532,23 @@ async def load_convo( convo_el["files"] = files - elif state.action == BH_START_BUG_HUNT.format(task_counter): + if state.iterations is not None and len(state.iterations) > 0: si = state.iterations[-1] - if si is not None: - if "user_feedback" in si and si["user_feedback"] is not None: + + if state.action == BH_START_BUG_HUNT.format(task_counter): + if si.get("user_feedback", None) is not None: convo_el["user_feedback"] = si["user_feedback"] - if "description" in si and si["description"] is not None: + if si.get("description", None) is not None: convo_el["description"] = si["description"] - elif state.action == BH_WAIT_BUG_REP_INSTRUCTIONS.format(task_counter): - if state.iterations is not None: + elif state.action == BH_WAIT_BUG_REP_INSTRUCTIONS.format(task_counter): for si in state.iterations: - if "bug_reproduction_description" in si and si["bug_reproduction_description"] is not None: + if si.get("bug_reproduction_description", None) is not None: convo_el["bug_reproduction_description"] = si["bug_reproduction_description"] - elif state.action == BH_START_USER_TEST.format(task_counter): - si = state.iterations[-1] - if si is not None: - if "bug_hunting_cycles" in si and si["bug_hunting_cycles"] is not None: + elif state.action == BH_START_USER_TEST.format(task_counter): + if si.get("bug_hunting_cycles", None) is not None: cycle = si["bug_hunting_cycles"][-1] if cycle is not None: if "user_feedback" in cycle and cycle["user_feedback"] is not None: @@ -431,49 +559,14 @@ async def load_convo( ): convo_el["human_readable_instructions"] = cycle["human_readable_instructions"] - elif state.action == BH_STARTING_PAIR_PROGRAMMING.format(task_counter): - si = state.iterations[-1] - if si is not None: + elif state.action == BH_STARTING_PAIR_PROGRAMMING.format(task_counter): if "user_feedback" in si and si["user_feedback"] is not None: convo_el["user_feedback"] = si["user_feedback"] if "initial_explanation" in si and si["initial_explanation"] is not None: convo_el["initial_explanation"] = si["initial_explanation"] - if convo_el.get("user_inputs", None) is not None or convo_el.get("files", None) is not None: - for j, ui in enumerate(user_inputs): - if ui.question and ui.question == MIX_BREAKDOWN_CHAT_PROMPT: - if len(state.tasks) == 1: - if state.tasks[0]["instructions"] is not None: - convo_el["breakdown"] = state.tasks[task_counter - 1]["instructions"] - break - elif ( - state.tasks[task_counter - 1] is not None - and state.tasks[task_counter - 1]["instructions"] is not None - ): - convo_el["breakdown"] = state.tasks[task_counter - 1]["instructions"] - break - elif state.iterations: - if ( - state.iterations[-1].get("bug_hunting_cycles", None) is not None - and state.iterations[-1]["bug_hunting_cycles"] - and state.iterations[-1]["bug_hunting_cycles"][-1].get("human_readable_instructions", None) - is not None - ): - convo_el["breakdown"] = state.iterations[-1]["bug_hunting_cycles"][-1][ - "human_readable_instructions" - ] - break - else: - # try to get the next state's instructions from the task - next_state = project_states[i + 1] - if next_state.tasks is not None: - next_task = find_first_todo_task(next_state.tasks) - if next_task: - convo_el["task_description"] = next_task["description"] - break - - convo_el["action"] = state.action - convo.append(convo_el) + convo_el["action"] = state.action + convo.append(convo_el) return convo diff --git a/core/cli/main.py b/core/cli/main.py index d94a17ec6b..5fba40e17a 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -5,8 +5,6 @@ import sys from argparse import Namespace from asyncio import run -from core.config.actions import TL_EDIT_DEV_PLAN - try: import sentry_sdk from sentry_sdk.integrations.asyncio import AsyncioIntegration @@ -23,6 +21,7 @@ from core.cli.helpers import ( list_projects_json, load_convo, load_project, + print_convo, show_config, ) from core.db.session import SessionManager @@ -37,16 +36,11 @@ from core.ui.base import ( UIBase, UIClosedError, UserInput, - bug_hunter_source, - developer_source, - history_source1, - history_source2, pythagora_source, ) log = get_logger(__name__) - telemetry_sent = False source_alt = True @@ -207,14 +201,6 @@ async def start_new_project(sm: StateManager, ui: UIBase) -> bool: return project_state is not None -def alternate_source(): - """Toggle between pythagora_source and history_source1.""" - global source_alt - source = history_source1 if source_alt else history_source2 - source_alt = not source_alt - return source - - async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): """ Run a Pythagora session. @@ -225,85 +211,15 @@ async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): :return: True if the application ran successfully, False otherwise. """ - if args.project and args.step: - convo = (await load_convo(sm, args.project, args.step))[-50:] - - for msg in convo: - if "breakdown" in msg: - await ui.send_message(msg["breakdown"], source=developer_source, project_state_id=msg["id"]) - - if "instructions" in msg: - await ui.send_message(msg["instructions"], source=bug_hunter_source, project_state_id=msg["id"]) - - if "task_description" in msg: - await ui.send_message(msg["task_description"], source=developer_source, project_state_id=msg["id"]) - - if "user_inputs" in msg and msg["user_inputs"]: - for input_item in msg["user_inputs"]: - if "question" in input_item: - await ui.send_message( - input_item["question"], source=alternate_source(), project_state_id=msg["id"] - ) - - if "answer" in input_item: - if input_item["question"] != TL_EDIT_DEV_PLAN: - await ui.send_user_input_history( - input_item["answer"], source=alternate_source(), project_state_id=msg["id"] - ) - else: - await ui.send_message( - input_item["question"], source=alternate_source(), project_state_id=msg["id"] - ) - - if "test_instructions" in msg: - await ui.send_test_instructions( - msg["test_instructions"], project_state_id=msg["id"], source=alternate_source() - ) - - if "files" in msg: - source = alternate_source() - for f in msg["files"]: - await ui.send_file_status(f["path"], "done", source=source) - await ui.generate_diff( - file_path=f["path"], - file_old=f.get("old_content", ""), - file_new=f.get("new_content", ""), - n_new_lines=f["diff"][0], - n_del_lines=f["diff"][1], - source=source, - ) - - # Process any other fields in the message - for key, value in msg.items(): - if ( - key - not in [ - "user_inputs", - "action", - "id", - "test_instructions", - "files", - "task_description", - "breakdown", - ] - and value - ): - if isinstance(value, str): - await ui.send_message(f"{key}: {value}", source=alternate_source(), project_state_id=msg["id"]) - elif isinstance(value, dict): - for k, v in value.items(): - if v and isinstance(v, str): - await ui.send_message( - f"{k}: {v}", source=alternate_source(), project_state_id=msg["id"] - ) - - log.debug(f"Convo exists: {len(convo) > 0}") - if args.project or args.branch or args.step: + convo = (await load_convo(sm, args.project, args.branch))[-50:] + await print_convo(ui, convo) + telemetry.set("is_continuation", True) success = await load_project(sm, args.project, args.branch, args.step) if not success: return False + else: success = await start_new_project(sm, ui) if not success: diff --git a/core/config/actions.py b/core/config/actions.py index e01687a2e3..7f11372cfb 100644 --- a/core/config/actions.py +++ b/core/config/actions.py @@ -37,9 +37,21 @@ SPEC_CHANGE_FEATURE_STEP_NAME = "Change specification due to new feature" TS_TASK_REVIEWED = "Task #{} reviewed" TS_ALT_SOLUTION = "Alternative solution (attempt #{})" +TS_APP_WORKING = "Please check if the app is working" PS_EPIC_COMPLETE = "Epic {} completed" # other constants TL_EDIT_DEV_PLAN = "Open and edit your development plan in the Progress tab" MIX_BREAKDOWN_CHAT_PROMPT = "Are you happy with the breakdown? Now is a good time to ask questions or suggest changes." +FE_CHANGE_REQ = ( + "Do you want to change anything or report a bug? Keep in mind that currently ONLY frontend is implemented." +) +FE_DONE_WITH_UI = "Are you sure you're done building the UI and want to start building the backend functionality now?" +TS_DESCRIBE_ISSUE = "Please describe the issue you found (one at a time) and share any relevant server logs" +BH_HUMAN_TEST_AGAIN = "Please test the app again." +BH_IS_BUG_FIXED = "Is the bug you reported fixed now?" +BH_ADDITIONAL_FEEDBACK = "Please add any additional feedback that could help Pythagora solve this bug" +HUMAN_INTERVENTION_QUESTION = "I need human intervention:" +RUN_COMMAND = "Can I run command:" +DEV_EXECUTE_TASK = "Do you want to execute the above task?" diff --git a/core/db/models/project_state.py b/core/db/models/project_state.py index 1eef30e3da..2d5447fbeb 100644 --- a/core/db/models/project_state.py +++ b/core/db/models/project_state.py @@ -218,16 +218,27 @@ class ProjectState(Base): @staticmethod async def get_project_states( session: "AsyncSession", - project_id: UUID, + project_id: Optional[UUID] = None, + branch_id: Optional[UUID] = None, ) -> list["ProjectState"]: from core.db.models import Branch, ProjectState - branch = await session.execute(select(Branch).where(Branch.project_id == project_id)) - branch = branch.scalar_one_or_none() + branch = None - project_states_result = await session.execute(select(ProjectState).where(ProjectState.branch_id == branch.id)) - # project_states_result = await session.execute(select(ProjectState).where(and_(ProjectState.branch_id == branch.id), ProjectState.action.isnot(None))) - return project_states_result.scalars().all() + if branch_id: + branch = await session.execute(select(Branch).where(Branch.id == branch_id)) + branch = branch.scalar_one_or_none() + elif project_id: + branch = await session.execute(select(Branch).where(Branch.project_id == project_id)) + branch = branch.scalar_one_or_none() + + if branch: + project_states_result = await session.execute( + select(ProjectState).where(ProjectState.branch_id == branch.id) + ) + return project_states_result.scalars().all() + + return [] async def create_next_state(self) -> "ProjectState": """ diff --git a/core/db/models/user_input.py b/core/db/models/user_input.py index c262836bb8..d823712a76 100644 --- a/core/db/models/user_input.py +++ b/core/db/models/user_input.py @@ -69,13 +69,4 @@ class UserInput(Base): ) ) user_input = user_input.scalars().all() - - if user_input is None: - user_input = await session.execute( - select(UserInput).where( - and_(UserInput.branch_id == branch_id, UserInput.project_state_id == project_state.prev_state_id) - ) - ) - user_input = user_input.scalars().all() - return user_input if len(user_input) > 0 else [] diff --git a/core/state/state_manager.py b/core/state/state_manager.py index 0570dafefe..559bc19a85 100644 --- a/core/state/state_manager.py +++ b/core/state/state_manager.py @@ -78,13 +78,9 @@ class StateManager: async with self.session_manager as session: return await Project.get_all_projects(session) - async def get_convo(self): + async def get_project_states(self, project_id: Optional[UUID], branch_id: Optional[UUID]) -> list[ProjectState]: async with self.session_manager as session: - return await Project.get_convo(session) - - async def get_project_states(self, project_id: UUID) -> list[ProjectState]: - async with self.session_manager as session: - return await ProjectState.get_project_states(session, project_id) + return await ProjectState.get_project_states(session, project_id, branch_id) async def get_branches_for_project_id(self, project_id: UUID) -> list[Branch]: async with self.session_manager as session: diff --git a/core/ui/base.py b/core/ui/base.py index 5116b6fb47..360973f331 100644 --- a/core/ui/base.py +++ b/core/ui/base.py @@ -491,12 +491,6 @@ class UIBase: pythagora_source = UISource("Pythagora", "pythagora") - -developer_source = AgentSource("Developer", "developer") -bug_hunter_source = AgentSource("Bug Hunter", "bug-hunter") - -history_source1 = UISource("Developer", "developer") -history_source2 = UISource("Bug Hunter", "bug-hunter") success_source = UISource("Congratulations", "success") From ccb79f0c449dc71bad21a192b79a0c35fb26ff87 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Mon, 31 Mar 2025 16:18:37 +0200 Subject: [PATCH 18/43] Fix convo printing --- core/cli/helpers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 6752d13575..d6f7b6f1d9 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -425,6 +425,12 @@ async def load_convo( """ convo = [] + if branch_id is None and project_id is not None: + branches = await sm.get_branches_for_project_id(project_id) + if not branches: + return convo + branch_id = branches[0].id + project_states = await sm.get_project_states(project_id, branch_id) task_counter = 1 From c28a5b45229e64f8c0444b900ff2be27fc36bf16 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Tue, 1 Apr 2025 09:46:26 +0200 Subject: [PATCH 19/43] Add ability to list projects with branches the old way --- core/cli/helpers.py | 6 +++--- core/cli/main.py | 4 ++-- core/db/models/project.py | 40 +++++++++++++++++++++++++++++++++++-- core/state/state_manager.py | 7 +++++++ tests/cli/test_cli.py | 9 +++++---- 5 files changed, 55 insertions(+), 11 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index ab3978ac50..828a6be739 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -560,12 +560,12 @@ async def load_convo( return convo -async def list_projects(db: SessionManager): +async def list_projects_old(db: SessionManager): """ List all projects in the database. """ sm = StateManager(db) - projects = await sm.list_projects() + projects = await sm.list_projects_old() print(f"Available projects ({len(projects)}):") for project in projects: @@ -661,4 +661,4 @@ def init() -> tuple[UIBase, SessionManager, Namespace]: return (ui, db, args) -__all__ = ["parse_arguments", "load_config", "list_projects_json", "list_projects", "load_project", "init"] +__all__ = ["parse_arguments", "load_config", "list_projects_json", "list_projects_old", "load_project", "init"] diff --git a/core/cli/main.py b/core/cli/main.py index 5fba40e17a..fdbc856711 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -17,8 +17,8 @@ from core.agents.orchestrator import Orchestrator from core.cli.helpers import ( delete_project, init, - list_projects, list_projects_json, + list_projects_old, load_convo, load_project, print_convo, @@ -244,7 +244,7 @@ async def async_main( global telemetry_sent if args.list: - await list_projects(db) + await list_projects_old(db) return True elif args.list_json: await list_projects_json(db) diff --git a/core/db/models/project.py b/core/db/models/project.py index db15c80646..8974451d73 100644 --- a/core/db/models/project.py +++ b/core/db/models/project.py @@ -4,9 +4,9 @@ from typing import TYPE_CHECKING, Optional, Union from unicodedata import normalize from uuid import UUID, uuid4 -from sqlalchemy import Row, delete, inspect, select +from sqlalchemy import Row, and_, delete, inspect, select from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, mapped_column, relationship, selectinload from sqlalchemy.sql import func from core.db.models import Base, File @@ -87,6 +87,42 @@ class Project(Base): result = await session.execute(query) return result.fetchall() + @staticmethod + async def get_all_projects_old(session: "AsyncSession") -> list["Project"]: + """ + Get all projects. + + This assumes the projects have at least one branch and one state. + + :param session: The SQLAlchemy session. + :return: List of Project objects. + """ + from core.db.models import Branch, ProjectState + + latest_state_query = ( + select(ProjectState.branch_id, func.max(ProjectState.step_index).label("max_index")) + .group_by(ProjectState.branch_id) + .subquery() + ) + + query = ( + select(Project, Branch, ProjectState) + .join(Branch, Project.branches) + .join(ProjectState, Branch.states) + .join( + latest_state_query, + and_( + ProjectState.branch_id == latest_state_query.columns.branch_id, + ProjectState.step_index == latest_state_query.columns.max_index, + ), + ) + .options(selectinload(Project.branches), selectinload(Branch.states)) + .order_by(Project.name, Branch.name) + ) + + results = await session.execute(query) + return results.scalars().all() + @staticmethod def get_folder_from_project_name(name: str): """ diff --git a/core/state/state_manager.py b/core/state/state_manager.py index 6fbcce9dcf..437b7eb1ca 100644 --- a/core/state/state_manager.py +++ b/core/state/state_manager.py @@ -76,6 +76,13 @@ class StateManager: async with self.session_manager as session: return await Project.get_all_projects(session) + async def list_projects_old(self) -> list[Project]: + """ + :return: List of projects with branches and states (old) - for debugging + """ + async with self.session_manager as session: + return await Project.get_all_projects_old(session) + async def get_project_states(self, project_id: Optional[UUID], branch_id: Optional[UUID]) -> list[ProjectState]: async with self.session_manager as session: return await ProjectState.get_project_states(session, project_id, branch_id) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 284fbf4f79..3732be7627 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -7,8 +7,8 @@ import pytest from core.cli.helpers import ( init, - list_projects, list_projects_json, + list_projects_old, load_config, load_project, parse_arguments, @@ -212,11 +212,12 @@ async def test_list_projects(mock_StateManager, capsys): branches=[branch], ) project.name = "project1" - sm.list_projects = AsyncMock(return_value=[project]) - await list_projects(None) + + sm.list_projects_old = AsyncMock(return_value=[project]) + await list_projects_old(None) mock_StateManager.assert_called_once_with(None) - sm.list_projects.assert_awaited_once_with() + sm.list_projects_old.assert_awaited_once_with() data = capsys.readouterr().out From 2116a3e21c25d697dc085012fdf248a64a7a4ad6 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Tue, 1 Apr 2025 12:43:13 +0200 Subject: [PATCH 20/43] Add some missing messages --- core/cli/helpers.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 828a6be739..b7c30072f7 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -373,6 +373,13 @@ async def print_convo( project_state_id=msg["id"], ) + if "bh_testing_instructions" in msg: + await ui.send_test_instructions( + msg["bh_testing_instructions"], + source=get_source_for_history("test_instructions"), + project_state_id=msg["id"], + ) + if "files" in msg: for f in msg["files"]: await ui.send_file_status(f["path"], "done") @@ -449,11 +456,27 @@ async def load_convo( convo_el["task_breakdown"] = task["instructions"] if ui.question == BH_HUMAN_TEST_AGAIN: - if state.iterations: - if state.iterations[0].get("bug_reproduction_description", None) is not None: - convo_el["bug_reproduction_description"] = state.iterations[0][ - "bug_reproduction_description" - ] + if len(state.iterations) > 0: + si = state.iterations[-1] + if si is not None: + if si.get("bug_reproduction_description", None) is not None: + convo_el["bh_testing_instructions"] = si["bug_reproduction_description"] + if si.get("description", None) is not None: ###### + convo_el["bh_breakdown"] = si["description"] ###### + + if ui.question == TS_APP_WORKING: + if len(state.iterations) > 0: + si = state.iterations[-1] + if si is not None: + if si.get("bug_reproduction_description", None) is not None: + convo_el["bh_testing_instructions"] = si["bug_reproduction_description"] + else: + task = find_first_todo_task(state.tasks) + if task: + if task.get("test_instructions", None) is not None: + convo_el["test_instructions"] = task["test_instructions"] + if task.get("instructions", None) is not None: + convo_el["task_breakdown"] = task["instructions"] answer = trim_logs(ui.answer_text) if ui.answer_text is not None else ui.answer_button if answer == "bug": From e8b731169a54a17655014410875dc94bda375b04 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Tue, 1 Apr 2025 12:54:59 +0200 Subject: [PATCH 21/43] Check if file exists before parsing diff --- core/cli/helpers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index b7c30072f7..9ccc77ef45 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -530,6 +530,9 @@ async def load_convo( current_file = await sm.get_file_for_project(state.id, path) prev_file = await sm.get_file_for_project(prev_state.id, path) + if not current_file or not prev_file: + continue + file["diff"] = get_line_changes( old_content=prev_file.content.content if prev_file else "", new_content=current_file.content.content, From 244358f9a674968f357d2fe30524559fc0878f63 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Tue, 1 Apr 2025 13:36:01 +0200 Subject: [PATCH 22/43] Save task action and parse task properly --- core/agents/developer.py | 2 +- core/cli/helpers.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/agents/developer.py b/core/agents/developer.py index 50d0752db0..b2082e4246 100644 --- a/core/agents/developer.py +++ b/core/agents/developer.py @@ -258,7 +258,7 @@ class Developer(ChatWithBreakdownMixin, RelevantFilesMixin, BaseAgent): # There might be state leftovers from previous tasks that we need to clean here self.next_state.modified_files = {} self.set_next_steps(response, source) - self.next_state.action = DEV_TASK_START.format({current_task_index + 1}) + self.next_state.action = DEV_TASK_START.format(current_task_index + 1) await telemetry.trace_code_event( "task-start", { diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 9ccc77ef45..74797ce361 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -1,6 +1,7 @@ import json import os import os.path +import re import sys from argparse import ArgumentParser, ArgumentTypeError, Namespace from difflib import unified_diff @@ -489,7 +490,8 @@ async def load_convo( if state.action is not None: if "Task" in state.action and "start" in state.action: - task_counter = int(state.action.split("#")[1].split()[0]) + match = re.search(r"Task\s+#\s*(?:\{)?(\d+)(?:\})?", state.action) + task_counter = int(match.group(1)) elif state.action == DEV_TROUBLESHOOT.format(task_counter): if state.iterations is not None and len(state.iterations) > 0: From 0bbe64580539821b4a272832de91b0cf820e121d Mon Sep 17 00:00:00 2001 From: mijauexe Date: Tue, 1 Apr 2025 14:43:04 +0200 Subject: [PATCH 23/43] Limit query rows of project states --- core/cli/main.py | 2 +- core/db/models/project_state.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/cli/main.py b/core/cli/main.py index fdbc856711..a6a8c134bd 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -212,7 +212,7 @@ async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): """ if args.project or args.branch or args.step: - convo = (await load_convo(sm, args.project, args.branch))[-50:] + convo = await load_convo(sm, args.project, args.branch) await print_convo(ui, convo) telemetry.set("is_continuation", True) diff --git a/core/db/models/project_state.py b/core/db/models/project_state.py index 7b85c041e7..4b8cf23d86 100644 --- a/core/db/models/project_state.py +++ b/core/db/models/project_state.py @@ -225,6 +225,7 @@ class ProjectState(Base): from core.db.models import Branch, ProjectState branch = None + limit = 100 if branch_id: branch = await session.execute(select(Branch).where(Branch.id == branch_id)) @@ -234,9 +235,8 @@ class ProjectState(Base): branch = branch.scalar_one_or_none() if branch: - project_states_result = await session.execute( - select(ProjectState).where(ProjectState.branch_id == branch.id) - ) + query = select(ProjectState).where(ProjectState.branch_id == branch.id).limit(limit) + project_states_result = await session.execute(query) return project_states_result.scalars().all() return [] From 66a8c4c3c36d84e9adb1509cdb0bf61a56f0774c Mon Sep 17 00:00:00 2001 From: mijauexe Date: Tue, 1 Apr 2025 15:03:37 +0200 Subject: [PATCH 24/43] Remove unnecessary parsing in question --- core/cli/helpers.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 74797ce361..c0518c3473 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -462,22 +462,16 @@ async def load_convo( if si is not None: if si.get("bug_reproduction_description", None) is not None: convo_el["bh_testing_instructions"] = si["bug_reproduction_description"] - if si.get("description", None) is not None: ###### - convo_el["bh_breakdown"] = si["description"] ###### + if si.get("description", None) is not None: + convo_el["bh_breakdown"] = si["description"] if ui.question == TS_APP_WORKING: - if len(state.iterations) > 0: - si = state.iterations[-1] - if si is not None: - if si.get("bug_reproduction_description", None) is not None: - convo_el["bh_testing_instructions"] = si["bug_reproduction_description"] - else: - task = find_first_todo_task(state.tasks) - if task: - if task.get("test_instructions", None) is not None: - convo_el["test_instructions"] = task["test_instructions"] - if task.get("instructions", None) is not None: - convo_el["task_breakdown"] = task["instructions"] + task = find_first_todo_task(state.tasks) + if task: + if task.get("test_instructions", None) is not None: + convo_el["test_instructions"] = task["test_instructions"] + if task.get("instructions", None) is not None: + convo_el["task_breakdown"] = task["instructions"] answer = trim_logs(ui.answer_text) if ui.answer_text is not None else ui.answer_button if answer == "bug": From 1360109f101454842a1130a2ecc53dabc4b1842c Mon Sep 17 00:00:00 2001 From: mijauexe Date: Tue, 1 Apr 2025 16:22:57 +0200 Subject: [PATCH 25/43] Change pydoc for loading convo function --- core/cli/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index c0518c3473..4722316c21 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -412,7 +412,8 @@ async def load_convo( branch_id: Optional[UUID] = None, ) -> list: """ - List all projects in the database. + Loads the conversation from an existing project. + returns: list of dictionaries with the conversation history """ convo = [] From 3e01ba8ec894651bf856a17054f8fd639d55aabc Mon Sep 17 00:00:00 2001 From: mijauexe Date: Tue, 1 Apr 2025 16:23:36 +0200 Subject: [PATCH 26/43] Load convo after loading project --- core/cli/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/cli/main.py b/core/cli/main.py index a6a8c134bd..828b7305f9 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -212,14 +212,14 @@ async def run_pythagora_session(sm: StateManager, ui: UIBase, args: Namespace): """ if args.project or args.branch or args.step: - convo = await load_convo(sm, args.project, args.branch) - await print_convo(ui, convo) - telemetry.set("is_continuation", True) success = await load_project(sm, args.project, args.branch, args.step) if not success: return False + convo = await load_convo(sm, args.project, args.branch) + await print_convo(ui, convo) + else: success = await start_new_project(sm, ui) if not success: From c6459f6a1dbb3ba464e2448fc9306ac70212df21 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Wed, 2 Apr 2025 09:09:00 +0200 Subject: [PATCH 27/43] Remove task breakdown when asking user to check if the app is working --- core/cli/helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 4722316c21..8206c260b2 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -471,8 +471,6 @@ async def load_convo( if task: if task.get("test_instructions", None) is not None: convo_el["test_instructions"] = task["test_instructions"] - if task.get("instructions", None) is not None: - convo_el["task_breakdown"] = task["instructions"] answer = trim_logs(ui.answer_text) if ui.answer_text is not None else ui.answer_button if answer == "bug": From ea1e634c3a4a654008c3391425ae382cf4b8ecc0 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Wed, 2 Apr 2025 10:25:22 +0200 Subject: [PATCH 28/43] Write continue as continue, print task description appropriately --- core/cli/helpers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 8206c260b2..80fbaa0aa8 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -19,6 +19,7 @@ from core.config.actions import ( BH_STARTING_PAIR_PROGRAMMING, BH_WAIT_BUG_REP_INSTRUCTIONS, CM_UPDATE_FILES, + DEV_EXECUTE_TASK, DEV_TASK_BREAKDOWN, DEV_TASK_START, DEV_TROUBLESHOOT, @@ -472,11 +473,15 @@ async def load_convo( if task.get("test_instructions", None) is not None: convo_el["test_instructions"] = task["test_instructions"] + if ui.question == DEV_EXECUTE_TASK: + task = find_first_todo_task(state.tasks) + if task: + if task.get("description", None) is not None: + convo_el["task_description"] = f"Task #{task_counter} - " + task["description"] + answer = trim_logs(ui.answer_text) if ui.answer_text is not None else ui.answer_button if answer == "bug": answer = "There is an issue" - elif answer == "continue": - answer = "Everything works" elif answer == "change": answer = "I want to make a change" convo_el["user_inputs"].append({"question": ui.question, "answer": answer}) From 0be7b4a01755ec5601899c01c84ca885569e56d5 Mon Sep 17 00:00:00 2001 From: LeonOstrez Date: Wed, 2 Apr 2025 10:46:11 +0200 Subject: [PATCH 29/43] fix clearing database at loading of the project. Now we delete user_inputs and file_contents. --- core/db/models/project_state.py | 31 ++++++++++++++++++++++++++++--- core/db/models/user_input.py | 11 ++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/core/db/models/project_state.py b/core/db/models/project_state.py index 4b8cf23d86..46dcd4097d 100644 --- a/core/db/models/project_state.py +++ b/core/db/models/project_state.py @@ -459,18 +459,43 @@ class ProjectState(Base): async def delete_after(self): """ - Delete all states in the branch after this one. + Delete all states in the branch after this one, along with related data. + + This includes: + - ProjectState records after this one + - Related UserInput records (including those for the current state) + - Related File records + - Orphaned FileContent records """ + from core.db.models import FileContent, UserInput session: AsyncSession = inspect(self).async_session log.debug(f"Deleting all project states in branch {self.branch_id} after {self.id}") - await session.execute( - delete(ProjectState).where( + + # Get all project states to be deleted + states_to_delete = await session.execute( + select(ProjectState).where( ProjectState.branch_id == self.branch_id, ProjectState.step_index > self.step_index, ) ) + states_to_delete = states_to_delete.scalars().all() + state_ids = [state.id for state in states_to_delete] + + # Delete user inputs for the current state + await session.execute(delete(UserInput).where(UserInput.project_state_id == self.id)) + + if state_ids: + # Delete related user inputs for states to be deleted + await session.execute(delete(UserInput).where(UserInput.project_state_id.in_(state_ids))) + + # Delete project states + await session.execute(delete(ProjectState).where(ProjectState.id.in_(state_ids))) + + # Clean up orphaned file content and user inputs + await FileContent.delete_orphans(session) + await UserInput.delete_orphans(session) def get_last_iteration_steps(self) -> list: """ diff --git a/core/db/models/user_input.py b/core/db/models/user_input.py index d823712a76..d68012b1c6 100644 --- a/core/db/models/user_input.py +++ b/core/db/models/user_input.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Optional from uuid import UUID -from sqlalchemy import ForeignKey, and_, inspect, select +from sqlalchemy import ForeignKey, and_, delete, inspect, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.sql import func @@ -70,3 +70,12 @@ class UserInput(Base): ) user_input = user_input.scalars().all() return user_input if len(user_input) > 0 else [] + + @classmethod + async def delete_orphans(cls, session: AsyncSession): + """ + Delete UserInput objects that have no associated ProjectState. + + :param session: The database session. + """ + await session.execute(delete(UserInput).where(UserInput.project_state_id.is_(None))) From 58c96fe82d4c5b59f0b23a1e148c092c4ce06433 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Wed, 2 Apr 2025 10:47:00 +0200 Subject: [PATCH 30/43] Take last 100 states, not the first 100 --- core/db/models/project_state.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/db/models/project_state.py b/core/db/models/project_state.py index 46dcd4097d..e7fd968a58 100644 --- a/core/db/models/project_state.py +++ b/core/db/models/project_state.py @@ -235,9 +235,16 @@ class ProjectState(Base): branch = branch.scalar_one_or_none() if branch: - query = select(ProjectState).where(ProjectState.branch_id == branch.id).limit(limit) + query = ( + select(ProjectState) + .where(ProjectState.branch_id == branch.id) + .order_by(ProjectState.step_index.desc()) # Get the latest 100 states + .limit(limit) + ) + project_states_result = await session.execute(query) - return project_states_result.scalars().all() + project_states = project_states_result.scalars().all() + return sorted(project_states, key=lambda x: x.step_index) return [] From ac49799e4a421ea645afc5b7f086689a40ff1486 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Wed, 2 Apr 2025 13:30:28 +0200 Subject: [PATCH 31/43] Skip printing messages during breakdown convo --- core/cli/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 80fbaa0aa8..89f1c3f7ab 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -457,6 +457,8 @@ async def load_convo( convo_el["test_instructions"] = task["test_instructions"] if task.get("instructions", None) is not None: convo_el["task_breakdown"] = task["instructions"] + # skip parsing that questions and its answers due to the fact that we do not keep states inside breakdown convo + break if ui.question == BH_HUMAN_TEST_AGAIN: if len(state.iterations) > 0: From e0b7b59c2aa79b273a1ca8c17a3533e69f248f4d Mon Sep 17 00:00:00 2001 From: mijauexe Date: Wed, 2 Apr 2025 13:31:37 +0200 Subject: [PATCH 32/43] Do not print bug hunter breakdown in some cases --- core/cli/helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 89f1c3f7ab..29e1f2b577 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -466,8 +466,6 @@ async def load_convo( if si is not None: if si.get("bug_reproduction_description", None) is not None: convo_el["bh_testing_instructions"] = si["bug_reproduction_description"] - if si.get("description", None) is not None: - convo_el["bh_breakdown"] = si["description"] if ui.question == TS_APP_WORKING: task = find_first_todo_task(state.tasks) From 7ececa8d648105eae713ab5084b251495e6e1c5f Mon Sep 17 00:00:00 2001 From: mijauexe Date: Wed, 2 Apr 2025 14:59:22 +0200 Subject: [PATCH 33/43] Fix re-entering database session --- core/state/state_manager.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/core/state/state_manager.py b/core/state/state_manager.py index 437b7eb1ca..5acb8ecbf2 100644 --- a/core/state/state_manager.py +++ b/core/state/state_manager.py @@ -84,20 +84,16 @@ class StateManager: return await Project.get_all_projects_old(session) async def get_project_states(self, project_id: Optional[UUID], branch_id: Optional[UUID]) -> list[ProjectState]: - async with self.session_manager as session: - return await ProjectState.get_project_states(session, project_id, branch_id) + return await ProjectState.get_project_states(self.current_session, project_id, branch_id) async def get_branches_for_project_id(self, project_id: UUID) -> list[Branch]: - async with self.session_manager as session: - return await Project.get_branches_for_project_id(session, project_id) + return await Project.get_branches_for_project_id(self.current_session, project_id) async def find_user_input(self, project_state, branch_id) -> Optional[list["UserInput"]]: - async with self.session_manager as session: - return await UserInput.find_user_inputs(session, project_state, branch_id) + return await UserInput.find_user_inputs(self.current_session, project_state, branch_id) async def get_file_for_project(self, state_id: UUID, path: str): - async with self.session_manager as session: - return await Project.get_file_for_project(session, state_id, path) + return await Project.get_file_for_project(self.current_session, state_id, path) async def create_project(self, name: str, folder_name: Optional[str] = None) -> Project: """ From 2addede4a3766662e2657eedc1ef06918be17bba Mon Sep 17 00:00:00 2001 From: mijauexe Date: Wed, 2 Apr 2025 17:03:06 +0200 Subject: [PATCH 34/43] Fix task counter --- core/cli/helpers.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 29e1f2b577..7439231fd1 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -1,7 +1,6 @@ import json import os import os.path -import re import sys from argparse import ArgumentParser, ArgumentTypeError, Namespace from difflib import unified_diff @@ -274,9 +273,13 @@ def find_first_todo_task(tasks): :param tasks: List of task objects :return: First task with status 'todo', or None if not found """ + if not tasks: + return None + for task in tasks: if task.get("status") == "todo": return task + return None @@ -435,6 +438,10 @@ async def load_convo( convo_el["id"] = str(state.id) user_inputs = await sm.find_user_input(state, branch_id) + todo_task = find_first_todo_task(state.tasks) + if todo_task: + task_counter = state.tasks.index(todo_task) + 1 + if user_inputs: convo_el["user_inputs"] = [] for ui in user_inputs: @@ -487,11 +494,7 @@ async def load_convo( convo_el["user_inputs"].append({"question": ui.question, "answer": answer}) if state.action is not None: - if "Task" in state.action and "start" in state.action: - match = re.search(r"Task\s+#\s*(?:\{)?(\d+)(?:\})?", state.action) - task_counter = int(match.group(1)) - - elif state.action == DEV_TROUBLESHOOT.format(task_counter): + if state.action == DEV_TROUBLESHOOT.format(task_counter): if state.iterations is not None and len(state.iterations) > 0: si = state.iterations[-1] if si is not None: @@ -503,10 +506,10 @@ async def load_convo( elif state.action == DEV_TASK_BREAKDOWN.format(task_counter): task = state.tasks[task_counter - 1] if task.get("description", None) is not None: - convo_el["description"] = task["description"] + convo_el["task_description"] = f"Task #{task_counter} - " + task["description"] if task.get("instructions", None) is not None: - convo_el["instructions"] = task["instructions"] + convo_el["task_breakdown"] = task["instructions"] elif state.action == TC_TASK_DONE.format(task_counter): if state.tasks: From c5cb9a296da09be5a6de6fa9b8fe8f9daa6851cb Mon Sep 17 00:00:00 2001 From: LeonOstrez Date: Thu, 3 Apr 2025 10:27:26 +0200 Subject: [PATCH 35/43] limit pythagora.log to 20k of lines --- core/config/__init__.py | 6 ++++ core/config/constants.py | 1 + core/log/__init__.py | 63 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/core/config/__init__.py b/core/config/__init__.py index f6f8bf4fec..a3519a83cf 100644 --- a/core/config/__init__.py +++ b/core/config/__init__.py @@ -5,6 +5,8 @@ from typing import Any, Literal, Optional, Union from pydantic import BaseModel, ConfigDict, Field, field_validator from typing_extensions import Annotated +from core.config.constants import LOGS_LINE_LIMIT + ROOT_DIR = abspath(join(dirname(__file__), "..", "..")) DEFAULT_IGNORE_PATHS = [ ".git", @@ -218,6 +220,10 @@ class LogConfig(_StrictModel): "pythagora.log", description="Output file for logs (if not specified, logs are printed to stderr)", ) + max_lines: int = Field( + LOGS_LINE_LIMIT, + description="Maximum number of lines to keep in the log file", + ) class DBConfig(_StrictModel): diff --git a/core/config/constants.py b/core/config/constants.py index 38880b120b..aaeeb0399e 100644 --- a/core/config/constants.py +++ b/core/config/constants.py @@ -1 +1,2 @@ CONVO_ITERATIONS_LIMIT = 8 +LOGS_LINE_LIMIT = 20000 diff --git a/core/log/__init__.py b/core/log/__init__.py index 67eb2ffcca..4737cdf363 100644 --- a/core/log/__init__.py +++ b/core/log/__init__.py @@ -1,6 +1,65 @@ +import os +from collections import deque from logging import FileHandler, Formatter, Logger, StreamHandler, getLogger from core.config import LogConfig +from core.config.constants import LOGS_LINE_LIMIT + + +class LineCountLimitedFileHandler(FileHandler): + """ + A file handler that limits the number of lines in the log file. + It keeps a fixed number of the most recent log lines. + """ + + def __init__(self, filename, max_lines=LOGS_LINE_LIMIT, mode="a", encoding=None, delay=False): + """ + Initialize the handler with the file and max lines. + + :param filename: Log file path + :param max_lines: Maximum number of lines to keep in the file + :param mode: File open mode + :param encoding: File encoding + :param delay: Delay file opening until first emit + """ + super().__init__(filename, mode, encoding, delay) + self.max_lines = max_lines + self.line_buffer = deque(maxlen=max_lines) + self._load_existing_lines() + + def _load_existing_lines(self): + """Load existing lines from the file into the buffer if the file exists.""" + if os.path.exists(self.baseFilename): + try: + with open(self.baseFilename, "r", encoding=self.encoding) as f: + for line in f: + if len(self.line_buffer) < self.max_lines: + self.line_buffer.append(line) + else: + self.line_buffer.popleft() + self.line_buffer.append(line) + except Exception: + # If there's an error reading the file, we'll just start with an empty buffer + self.line_buffer.clear() + + def emit(self, record): + """ + Emit a record and maintain the line count limit. + + :param record: Log record to emit + """ + try: + msg = self.format(record) + line = msg + self.terminator + self.line_buffer.append(line) + + # Rewrite the entire file with the current buffer + with open(self.baseFilename, "w", encoding=self.encoding) as f: + f.writelines(self.line_buffer) + + self.flush() + except Exception: + self.handleError(record) def setup(config: LogConfig, force: bool = False): @@ -27,7 +86,9 @@ def setup(config: LogConfig, force: bool = False): formatter = Formatter(config.format) if config.output: - handler = FileHandler(config.output, encoding="utf-8") + # Use our custom handler that limits line count + max_lines = getattr(config, "max_lines", LOGS_LINE_LIMIT) + handler = LineCountLimitedFileHandler(config.output, max_lines=max_lines, encoding="utf-8") else: handler = StreamHandler() From 0d242e98faa19f07b4570fb6bb4188636779f32f Mon Sep 17 00:00:00 2001 From: mijauexe Date: Thu, 3 Apr 2025 15:02:49 +0200 Subject: [PATCH 36/43] Remove source from send_test_instructions It is not necessary, was used for testing only --- core/cli/helpers.py | 2 -- core/ui/base.py | 4 +--- core/ui/console.py | 4 +--- core/ui/virtual.py | 4 +--- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 7439231fd1..b23457241b 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -374,14 +374,12 @@ async def print_convo( if "test_instructions" in msg: await ui.send_test_instructions( msg["test_instructions"], - source=get_source_for_history("test_instructions"), project_state_id=msg["id"], ) if "bh_testing_instructions" in msg: await ui.send_test_instructions( msg["bh_testing_instructions"], - source=get_source_for_history("test_instructions"), project_state_id=msg["id"], ) diff --git a/core/ui/base.py b/core/ui/base.py index 360973f331..42b9cd51da 100644 --- a/core/ui/base.py +++ b/core/ui/base.py @@ -383,9 +383,7 @@ class UIBase: """ raise NotImplementedError() - async def send_test_instructions( - self, test_instructions: str, project_state_id: Optional[str] = None, source: Optional[UISource] = None - ): + async def send_test_instructions(self, test_instructions: str, project_state_id: Optional[str] = None): """ Send test instructions. diff --git a/core/ui/console.py b/core/ui/console.py index 3c0c63e8a3..5bb85984da 100644 --- a/core/ui/console.py +++ b/core/ui/console.py @@ -180,9 +180,7 @@ class PlainConsoleUI(UIBase): async def send_project_stats(self, stats: dict): pass - async def send_test_instructions( - self, test_instructions: str, project_state_id: Optional[str] = None, source: Optional[UISource] = None - ): + async def send_test_instructions(self, test_instructions: str, project_state_id: Optional[str] = None): pass async def knowledge_base_update(self, knowledge_base: dict): diff --git a/core/ui/virtual.py b/core/ui/virtual.py index 2165d92451..dfda5239c2 100644 --- a/core/ui/virtual.py +++ b/core/ui/virtual.py @@ -176,9 +176,7 @@ class VirtualUI(UIBase): async def send_project_stats(self, stats: dict): pass - async def send_test_instructions( - self, test_instructions: str, project_state_id: Optional[str] = None, source: Optional[UISource] = None - ): + async def send_test_instructions(self, test_instructions: str, project_state_id: Optional[str] = None): pass async def knowledge_base_update(self, knowledge_base: dict): From cfa1d4fb013e42fcd139a23862c92efffc605f20 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Thu, 3 Apr 2025 15:04:11 +0200 Subject: [PATCH 37/43] Remove unnecessary variable It was used during early iterations of testing this PR, no longer necessary --- core/cli/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/cli/main.py b/core/cli/main.py index 828b7305f9..d92b6e998c 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -42,7 +42,6 @@ from core.ui.base import ( log = get_logger(__name__) telemetry_sent = False -source_alt = True def init_sentry(): From fdc81afb3fd7b3c19abbcd16ee03ed2cb8d029fc Mon Sep 17 00:00:00 2001 From: mijauexe Date: Thu, 3 Apr 2025 15:29:45 +0200 Subject: [PATCH 38/43] Make parsing old and new file contents more robust This will handle creating new files and deleting files with proper positive and negative diffs --- core/cli/helpers.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index b23457241b..1015f8ad97 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -531,18 +531,16 @@ async def load_convo( current_file = await sm.get_file_for_project(state.id, path) prev_file = await sm.get_file_for_project(prev_state.id, path) - if not current_file or not prev_file: - continue + old_content = prev_file.content.content if prev_file and prev_file.content else "" + new_content = current_file.content.content if current_file and current_file.content else "" file["diff"] = get_line_changes( - old_content=prev_file.content.content if prev_file else "", - new_content=current_file.content.content, + old_content=old_content, + new_content=new_content, ) - file["old_content"] = prev_file.content.content if prev_file else "" - file["new_content"] = current_file.content.content + file["old_content"] = old_content + file["new_content"] = new_content - # hack that works fine because state.steps.completed is false until file is updated, but - # if we don't do this, we would have to compare to previous state which is complicated if file["diff"] != (0, 0): files.append(file) From ac9e7498b5d0988504b43634392900658ba02397 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Thu, 3 Apr 2025 15:39:21 +0200 Subject: [PATCH 39/43] Prevent null error when parsing previous state file content --- core/cli/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 1015f8ad97..2a5681bb27 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -529,7 +529,7 @@ async def load_convo( file["path"] = path current_file = await sm.get_file_for_project(state.id, path) - prev_file = await sm.get_file_for_project(prev_state.id, path) + prev_file = await sm.get_file_for_project(prev_state.id, path) if prev_state else None old_content = prev_file.content.content if prev_file and prev_file.content else "" new_content = current_file.content.content if current_file and current_file.content else "" From 0cdd556659d2b5e6720a0eaec30eca8b2caf2bc7 Mon Sep 17 00:00:00 2001 From: mijauexe Date: Thu, 3 Apr 2025 15:40:22 +0200 Subject: [PATCH 40/43] Rename function to be clearer --- core/cli/helpers.py | 15 +++++++++++---- core/cli/main.py | 4 ++-- core/db/models/project.py | 2 +- core/state/state_manager.py | 4 ++-- tests/cli/test_cli.py | 8 ++++---- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/core/cli/helpers.py b/core/cli/helpers.py index 2a5681bb27..84117ee7b6 100644 --- a/core/cli/helpers.py +++ b/core/cli/helpers.py @@ -585,12 +585,12 @@ async def load_convo( return convo -async def list_projects_old(db: SessionManager): +async def list_projects_branches_states(db: SessionManager): """ - List all projects in the database. + List all projects in the database, including their branches and project states """ sm = StateManager(db) - projects = await sm.list_projects_old() + projects = await sm.list_projects_with_branches_states() print(f"Available projects ({len(projects)}):") for project in projects: @@ -686,4 +686,11 @@ def init() -> tuple[UIBase, SessionManager, Namespace]: return (ui, db, args) -__all__ = ["parse_arguments", "load_config", "list_projects_json", "list_projects_old", "load_project", "init"] +__all__ = [ + "parse_arguments", + "load_config", + "list_projects_json", + "list_projects_branches_states", + "load_project", + "init", +] diff --git a/core/cli/main.py b/core/cli/main.py index d92b6e998c..4a5cb5b3a3 100644 --- a/core/cli/main.py +++ b/core/cli/main.py @@ -17,8 +17,8 @@ from core.agents.orchestrator import Orchestrator from core.cli.helpers import ( delete_project, init, + list_projects_branches_states, list_projects_json, - list_projects_old, load_convo, load_project, print_convo, @@ -243,7 +243,7 @@ async def async_main( global telemetry_sent if args.list: - await list_projects_old(db) + await list_projects_branches_states(db) return True elif args.list_json: await list_projects_json(db) diff --git a/core/db/models/project.py b/core/db/models/project.py index 8974451d73..7627b21de3 100644 --- a/core/db/models/project.py +++ b/core/db/models/project.py @@ -88,7 +88,7 @@ class Project(Base): return result.fetchall() @staticmethod - async def get_all_projects_old(session: "AsyncSession") -> list["Project"]: + async def get_all_projects_with_branches_states(session: "AsyncSession") -> list["Project"]: """ Get all projects. diff --git a/core/state/state_manager.py b/core/state/state_manager.py index 5acb8ecbf2..5179ce5b97 100644 --- a/core/state/state_manager.py +++ b/core/state/state_manager.py @@ -76,12 +76,12 @@ class StateManager: async with self.session_manager as session: return await Project.get_all_projects(session) - async def list_projects_old(self) -> list[Project]: + async def list_projects_with_branches_states(self) -> list[Project]: """ :return: List of projects with branches and states (old) - for debugging """ async with self.session_manager as session: - return await Project.get_all_projects_old(session) + return await Project.get_all_projects_with_branches_states(session) async def get_project_states(self, project_id: Optional[UUID], branch_id: Optional[UUID]) -> list[ProjectState]: return await ProjectState.get_project_states(self.current_session, project_id, branch_id) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 3732be7627..ad23de6796 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -7,8 +7,8 @@ import pytest from core.cli.helpers import ( init, + list_projects_branches_states, list_projects_json, - list_projects_old, load_config, load_project, parse_arguments, @@ -213,11 +213,11 @@ async def test_list_projects(mock_StateManager, capsys): ) project.name = "project1" - sm.list_projects_old = AsyncMock(return_value=[project]) - await list_projects_old(None) + sm.list_projects_with_branches_states = AsyncMock(return_value=[project]) + await list_projects_branches_states(None) mock_StateManager.assert_called_once_with(None) - sm.list_projects_old.assert_awaited_once_with() + sm.list_projects_with_branches_states.assert_awaited_once_with() data = capsys.readouterr().out From b29e7c41f7ae763c432d068bdb66d8e76b5f1c60 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 4 Apr 2025 13:26:27 +0000 Subject: [PATCH 41/43] version --- CHANGELOG.md | 4 ++++ pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..5751cf9938 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,4 @@ +# (2025-04-04) + + + diff --git a/pyproject.toml b/pyproject.toml index 9f6c93e55a..34dc857cc3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pythagora-core" -version = "1.3.7" +version = "1.3.8" description = "Build complete apps using AI agents" authors = ["Leon Ostrez "] license = "FSL-1.1-MIT" From b61fc113f3a63ecae79a5f6cd533a668a2e83412 Mon Sep 17 00:00:00 2001 From: Zvonimir Sabljic Date: Mon, 7 Apr 2025 10:55:24 +0200 Subject: [PATCH 42/43] Added sonnet to the FE agent plus a small FE prompt change --- core/config/__init__.py | 2 +- core/prompts/frontend/build_frontend.prompt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/config/__init__.py b/core/config/__init__.py index a3519a83cf..af227b38c3 100644 --- a/core/config/__init__.py +++ b/core/config/__init__.py @@ -353,7 +353,7 @@ class Config(_StrictModel): ), FRONTEND_AGENT_NAME: AgentLLMConfig( provider=LLMProvider.OPENAI, - model="claude-3-5-sonnet-20241022", + model="claude-3-7-sonnet-20250219", temperature=0.0, ), GET_RELEVANT_FILES_AGENT_NAME: AgentLLMConfig( diff --git a/core/prompts/frontend/build_frontend.prompt b/core/prompts/frontend/build_frontend.prompt index bfc50586ba..3c6a338b4a 100644 --- a/core/prompts/frontend/build_frontend.prompt +++ b/core/prompts/frontend/build_frontend.prompt @@ -19,6 +19,7 @@ Use material design and nice icons for the design to be appealing and modern. Us 3. Heroicons: For a set of sleek, customizable icons that integrate well with modern designs. 4. React Hook Form: For efficient form handling with minimal re-rendering, ensuring a smooth user experience in form-heavy applications. 5. Use Tailwind built-in animations to enhance the visual appeal of the app + 6. Make the app look colorful and modern but also have the colors be subtle. Choose a flat color palette and make sure that the text is readable and follow design best practices to make the text readable. Also, Implement these design features onto the page - gradient background, frosted glass effects, rounded corner, buttons need to be in the brand colors, and interactive feedback on hover and focus. From 86b399a8661a19f72ef9daf479a34c32e1e348e1 Mon Sep 17 00:00:00 2001 From: LeonOstrez Date: Mon, 7 Apr 2025 12:13:12 +0200 Subject: [PATCH 43/43] remove vsc-dl-x64 from archive --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..175a2b0d8d --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +vsc-dl-x64 export-ignore