From bb2d856a65a8d4ec2c448a7b1b5ecfd368bde839 Mon Sep 17 00:00:00 2001 From: LeonOstrez Date: Wed, 6 Dec 2023 11:44:43 +0000 Subject: [PATCH 1/6] add buttons to extension --- pilot/helpers/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pilot/helpers/cli.py b/pilot/helpers/cli.py index 24c6927a14..2b861bb79a 100644 --- a/pilot/helpers/cli.py +++ b/pilot/helpers/cli.py @@ -209,6 +209,7 @@ def execute_command(project, command, timeout=None, success_message=None, comman else: question += '?' + print('yes/no', type='button') logger.info('--------- EXECUTE COMMAND ---------- : %s', question) answer = ask_user(project, 'If yes, just press ENTER. Otherwise, type "no" but it will be processed as ' 'successfully executed.', False, hint=question) From 87c29393693c8657a33315d539fc75d08adfcc32 Mon Sep 17 00:00:00 2001 From: Zvonimir Sabljic Date: Wed, 6 Dec 2023 14:40:04 +0100 Subject: [PATCH 2/6] Improved prompts --- pilot/const/function_calls.py | 4 ++-- pilot/prompts/development/feature_plan.prompt | 2 +- pilot/prompts/development/get_fully_coded_file.prompt | 5 ++++- pilot/prompts/development/implement_changes.prompt | 11 ++++++++++- pilot/prompts/development/task/breakdown.prompt | 2 +- .../development/task/step/write_automated_test.prompt | 2 +- pilot/prompts/development/task/step_code.prompt | 2 +- pilot/prompts/development/task/update_task.prompt | 4 ++-- 8 files changed, 22 insertions(+), 10 deletions(-) diff --git a/pilot/const/function_calls.py b/pilot/const/function_calls.py index 787335d3e3..17c7a45766 100644 --- a/pilot/const/function_calls.py +++ b/pilot/const/function_calls.py @@ -218,7 +218,7 @@ IMPLEMENT_TASK = { }, 'content': { 'type': 'string', - 'description': 'Full content of the file that needs to be implemented. It is mandatory that no parts of file content are skipped or assumed to be there. It must be full content.', + 'description': 'Full content of the file that needs to be implemented. **IMPORTANT**When you want to add a comment that tells the user to add the previous implementation at that place, make sure that the comment starts with `[OLD CODE]` and add a description of what old code should be inserted here. For example, `[OLD CODE] Login route`.', }, }, 'required': ['name', 'path', 'content'], @@ -478,7 +478,7 @@ IMPLEMENT_CHANGES = { }, 'content': { 'type': 'string', - 'description': 'Full content of the file that needs to be saved on the disk. It is mandatory that no parts of file content are skipped or assumed to be there. It must be full content.', + 'description': 'Full content of the file that needs to be saved on the disk. **IMPORTANT**When you want to add a comment that tells the user to add the previous implementation at that place, make sure that the comment starts with `[OLD CODE]` and add a description of what old code should be inserted here. For example, `[OLD CODE] Login route`.', }, 'description': { 'type': 'string', diff --git a/pilot/prompts/development/feature_plan.prompt b/pilot/prompts/development/feature_plan.prompt index e9887b5a7b..dd63bd56dd 100644 --- a/pilot/prompts/development/feature_plan.prompt +++ b/pilot/prompts/development/feature_plan.prompt @@ -76,4 +76,4 @@ Each task needs to be related only to the development of this feature and nothin For each task, there must be a way for human developer to check if the task is done or not. Write how should the developer check if the task is done. -Now, based on the project details provided and new feature description, think task by task and create the entire development plan for new feature. For each task, write a description and a user-review goal. \ No newline at end of file +Now, based on the project details provided and new feature description, think task by task and create the entire development plan for new feature. For each task, write a description and a user-review goal. diff --git a/pilot/prompts/development/get_fully_coded_file.prompt b/pilot/prompts/development/get_fully_coded_file.prompt index a2929f7761..030d99b8fa 100644 --- a/pilot/prompts/development/get_fully_coded_file.prompt +++ b/pilot/prompts/development/get_fully_coded_file.prompt @@ -3,13 +3,16 @@ Here is how the file `{{ file.path }}` looks like right now: {{ file.content }} ``` -And is the new implementation for the same file: +And here is the new implementation for the same file: ``` {{ new_file.content }} ``` Now, implement the new changes into the previously implemented file and return the entirely coded file. Do not add any lines of code that are not in the previously coded file or the new implementation but only combine those two. +**IMPORTANT** +All lines of code in the new implementation should be present. From the old implementation, only take the ones that are replaced with the comment `[OLD CODE]` in the new implementation + **IMPORTANT** Make sure that you respond with all lines of code that are replaced by these comments, including all control structures, error handling, and any other relevant logic that was in the original code. Under no circumstances should you ever leave any part of the code snippet unwritten. Every single line of code that exists in the place where the comment lives right now should be replaced. Do not include any code that is above or below the comment but only the code that should be in the position of the comment. diff --git a/pilot/prompts/development/implement_changes.prompt b/pilot/prompts/development/implement_changes.prompt index 1566d44fba..de88721eff 100644 --- a/pilot/prompts/development/implement_changes.prompt +++ b/pilot/prompts/development/implement_changes.prompt @@ -5,7 +5,16 @@ {{ file.content }} ``` -{% endfor %}{% endif %}Now, think step by step and apply the needed changes for step #{{ step_index }} - `{{ step_description }}`. +{% endfor %}{% endif -%} +Here is a description of changes for a task you're currently working on: +``` +{{ task_description }} +``` + +This task is broken down into steps. Please tell me the changes needed to implement the step #{{ step_index }} that is described like this: +``` +{{ step_description }} +``` Within the file modifications, anything needs to be written by the user, add the comment in the same line as the code that starts with `// INPUT_REQUIRED {input_description}` where `input_description` is a description of what needs to be added here by the user. Finally, you can save the modified files on the disk by calling `save_files` function. diff --git a/pilot/prompts/development/task/breakdown.prompt b/pilot/prompts/development/task/breakdown.prompt index df831fb9ea..a383398ffd 100644 --- a/pilot/prompts/development/task/breakdown.prompt +++ b/pilot/prompts/development/task/breakdown.prompt @@ -46,7 +46,7 @@ You are currently working on task {{ current_task_index + 1 }} with the followin {{ development_tasks[current_task_index]['description'] }} ``` {% if current_task_index != 0 %}All previous tasks are finished and you don't have to work on them.{% endif %} -After all the code is finished, a human developer will check it works this way - {{ development_tasks[current_task_index]['user_review_goal'] }} +After all the code is finished, a human developer will check if the app works this way - `{{ development_tasks[current_task_index]['user_review_goal'] }}` Now, tell me all the code that needs to be written to implement ONLY this task and have it fully working and all commands that need to be run to implement this task. diff --git a/pilot/prompts/development/task/step/write_automated_test.prompt b/pilot/prompts/development/task/step/write_automated_test.prompt index 0c9b26f93c..e7ff69c344 100644 --- a/pilot/prompts/development/task/step/write_automated_test.prompt +++ b/pilot/prompts/development/task/step/write_automated_test.prompt @@ -26,4 +26,4 @@ NEW_FILES: {new_files_array} `name` - file name with the full path relative to the root of the project `description` - a thorough description of what this file is meant to contain so that we can know in the future if any new code needs to be put in that file. Do not describe what is currently implemented in this file but rather a description so that anyone who looks at this description knows if they should put the new code in it or not. -You can write code in multiple files and keep in mind that you also need to write test (or tests) that will programmatically verify that your task is complete. If you need to run any commands, you can do that now but make sure that the command is not contained in any other steps outlined above. \ No newline at end of file +You can write code in multiple files and keep in mind that you also need to write test (or tests) that will programmatically verify that your task is complete. If you need to run any commands, you can do that now but make sure that the command is not contained in any other steps outlined above. diff --git a/pilot/prompts/development/task/step_code.prompt b/pilot/prompts/development/task/step_code.prompt index 06f2ce05b3..d525f5702c 100644 --- a/pilot/prompts/development/task/step_code.prompt +++ b/pilot/prompts/development/task/step_code.prompt @@ -26,4 +26,4 @@ You can write code in multiple files and keep in mind that you also need to writ If you need to run any commands, you can do that now but make sure that the command is not contained in any other steps outlined above. -If there is nothing needed to be done, respond with only `DONE` and nothing else. \ No newline at end of file +If there is nothing needed to be done, respond with only `DONE` and nothing else. diff --git a/pilot/prompts/development/task/update_task.prompt b/pilot/prompts/development/task/update_task.prompt index 607b8e2406..4ada21d1ad 100644 --- a/pilot/prompts/development/task/update_task.prompt +++ b/pilot/prompts/development/task/update_task.prompt @@ -1,5 +1,5 @@ {% if completed_steps -%} -Here is a list of completed steps: +We got a plan for how to debug this and we've already done some steps: ```{% for step in completed_steps %} step #{{ loop.index - 1 }}: step_type: {{ step.type }}{% if 'command' in step %} @@ -26,8 +26,8 @@ command_to_run: `{{ step.command.command }}`{% elif step.type == 'code_change' % code_change_description: `{{ step.code_change_description }}`{% endif %}{% endfor %} ``` -Now, think step by step and create a new list of task steps to complete this task based on the output we got from running `{{ current_step.command.command }}`. It will either be the same as it is now or any of the Next Task Steps may or may not be needed anymore. {% endif %} +Now, think step by step and create a new list of task steps to complete this task based on the output we got from running `{{ current_step.command.command }}`. It will either be the same as it is now or any of the Next Task Steps may or may not be needed anymore. **IMPORTANT** When you think about in which file should the new code go to, always try to make files as small as possible and put code in more smaller files rather than in one big file. Whenever a file becomes too large (more than 50 lines of code) split it into smaller files. From ff8575624bf1f8512133415bc5e7a431841d0f31 Mon Sep 17 00:00:00 2001 From: Zvonimir Sabljic Date: Wed, 6 Dec 2023 16:04:13 +0100 Subject: [PATCH 3/6] Fix replacing file content in messages --- pilot/helpers/AgentConvo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pilot/helpers/AgentConvo.py b/pilot/helpers/AgentConvo.py index 1ae670ab1a..fe434ab756 100644 --- a/pilot/helpers/AgentConvo.py +++ b/pilot/helpers/AgentConvo.py @@ -187,7 +187,7 @@ class AgentConvo: for msg in self.messages: if msg['role'] == 'user': for file in files: - self.replace_file_content(msg['content'], file['path'], file['content']) + msg['content'] = self.replace_file_content(msg['content'], file['path'], file['content']) def escape_specials(self, s): s = s.replace("\\", "\\\\") @@ -213,12 +213,12 @@ class AgentConvo: def replace_file_content(self, message, file_path, new_content): escaped_file_path = re.escape(file_path) - pattern = rf'\*\*{{ {escaped_file_path} }}\*\*\n```\n(.*?)\n```' + pattern = rf'\*\*{escaped_file_path}\*\*\n```\n(.*?)\n```' # Escape special characters in new_content for the sake of regex replacement new_content_escaped = self.escape_specials(new_content) - new_section_content = f'**{{ {file_path} }}**\n```\n{new_content_escaped}\n```' + new_section_content = f'**{file_path}**\n```\n{new_content_escaped}\n```' updated_message, num_replacements = re.subn(pattern, new_section_content, message, flags=re.DOTALL) From 935ac21f91069789fbe483336d88e18af0fde616 Mon Sep 17 00:00:00 2001 From: Zvonimir Sabljic Date: Wed, 6 Dec 2023 16:24:07 +0100 Subject: [PATCH 4/6] Enable .env to be sent to the LLM --- pilot/const/common.py | 1 - pilot/helpers/Debugger.py | 19 ++++++++++++++++--- pilot/helpers/agents/CodeMonkey.py | 3 ++- pilot/helpers/agents/Developer.py | 13 +++++++------ 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/pilot/const/common.py b/pilot/const/common.py index 9707ad1ab8..0b5374b6d0 100644 --- a/pilot/const/common.py +++ b/pilot/const/common.py @@ -30,7 +30,6 @@ IGNORE_FOLDERS = [ '.idea', '.vscode', '.DS_Store', - '.env', '__pycache__', 'node_modules', 'package-lock.json', diff --git a/pilot/helpers/Debugger.py b/pilot/helpers/Debugger.py index 83d8c33731..b185797ffb 100644 --- a/pilot/helpers/Debugger.py +++ b/pilot/helpers/Debugger.py @@ -1,5 +1,6 @@ import platform import uuid +import re from const.code_execution import MAX_COMMAND_DEBUG_TRIES, MAX_RECUSION_LAYER from const.function_calls import DEBUG_STEPS_BREAKDOWN @@ -41,8 +42,6 @@ class Debugger: if success: break - convo.load_branch(function_uuid) - llm_response = convo.send_message('dev_ops/debug.prompt', { 'command': command['command'] if command is not None else None, @@ -64,6 +63,7 @@ class Debugger: # TODO refactor to nicely get the developer agent result = self.agent.project.developer.execute_task( convo, + f"Thoughts: {llm_response['thoughts']}\n\nReasoning: {llm_response['reasoning']}", steps, test_command=command, test_after_code_changes=True, @@ -72,6 +72,7 @@ class Debugger: continue_from_step=len(completed_steps) ) + # in case one step failed or llm wants to see the output to determine the next steps if 'step_index' in result: result['os'] = platform.system() step_index = result['step_index'] @@ -88,6 +89,16 @@ class Debugger: DEBUG_STEPS_BREAKDOWN) else: success = result['success'] + if not success: + convo.load_branch(function_uuid) + if 'cli_response' in result: + user_input = result['cli_response'] + convo.messages[-2]['content'] = re.sub( + r'(?<=The output was:\n\n).*?(?=\n\nThink about this output)', + result['cli_response'], + convo.messages[-2]['content'], + flags=re.DOTALL + ) break except TokenLimitError as e: @@ -95,8 +106,10 @@ class Debugger: self.recursion_layer -= 1 raise e else: + if not success: + convo.load_branch(function_uuid) continue - + # if not success: # # TODO explain better how should the user approach debugging # # we can copy the entire convo to clipboard so they can paste it in the playground diff --git a/pilot/helpers/agents/CodeMonkey.py b/pilot/helpers/agents/CodeMonkey.py index ad83d9e0b8..4164cfb5b7 100644 --- a/pilot/helpers/agents/CodeMonkey.py +++ b/pilot/helpers/agents/CodeMonkey.py @@ -8,7 +8,7 @@ class CodeMonkey(Agent): super().__init__('code_monkey', project) self.developer = developer - def implement_code_changes(self, convo, code_changes_description, step, step_index=0): + def implement_code_changes(self, convo, task_description, code_changes_description, step, step_index=0): if convo is None: convo = AgentConvo(self) @@ -22,6 +22,7 @@ class CodeMonkey(Agent): llm_response = convo.send_message('development/implement_changes.prompt', { "step_description": code_changes_description, "step": step, + "task_description": task_description, "step_index": step_index, # todo remove step_index because in debugging we reset steps and it looks confusing in convo "directory_tree": self.project.get_directory_tree(True), "files": self.project.get_all_coded_files() # self.project.get_files(files_needed), diff --git a/pilot/helpers/agents/Developer.py b/pilot/helpers/agents/Developer.py index e00069196e..3e594c9b10 100644 --- a/pilot/helpers/agents/Developer.py +++ b/pilot/helpers/agents/Developer.py @@ -99,6 +99,7 @@ class Developer(Agent): while True: result = self.execute_task(convo_dev_task, + development_task['description'], steps, development_task=development_task, continue_development=True, @@ -141,12 +142,12 @@ class Developer(Agent): return files_with_comments - def step_code_change(self, convo, step, i, test_after_code_changes): + def step_code_change(self, convo, task_description, step, i, test_after_code_changes): if 'code_change_description' in step: # TODO this should be refactored so it always uses the same function call print(f'Implementing code changes for `{step["code_change_description"]}`') code_monkey = CodeMonkey(self.project, self) - updated_convo = code_monkey.implement_code_changes(convo, step['code_change_description'], step, i) + updated_convo = code_monkey.implement_code_changes(convo, task_description, step['code_change_description'], step, i) if test_after_code_changes: return self.test_code_changes(code_monkey, updated_convo) else: @@ -358,7 +359,7 @@ class Developer(Agent): return { "success": True, "user_input": answer } - def execute_task(self, convo, task_steps, test_command=None, reset_convo=True, + def execute_task(self, convo, task_description, task_steps, test_command=None, reset_convo=True, test_after_code_changes=True, continue_development=False, development_task=None, is_root_task=False, continue_from_step=0): function_uuid = str(uuid.uuid4()) @@ -385,7 +386,7 @@ class Developer(Agent): # result['user_input'] = result['cli_response'] elif step['type'] == 'code_change': - result = self.step_code_change(convo, step, i, test_after_code_changes) + result = self.step_code_change(convo, task_description, step, i, test_after_code_changes) elif step['type'] == 'human_intervention': result = self.step_human_intervention(convo, step) @@ -471,7 +472,7 @@ class Developer(Agent): if user_feedback is not None: iteration_convo = AgentConvo(self) - iteration_convo.send_message('development/iteration.prompt', { + iteration_description = iteration_convo.send_message('development/iteration.prompt', { "name": self.project.args['name'], "app_type": self.project.args['app_type'], "app_summary": self.project.project_description, @@ -496,7 +497,7 @@ class Developer(Agent): iteration_convo.remove_last_x_messages(2) task_steps = llm_response['tasks'] - self.execute_task(iteration_convo, task_steps, is_root_task=True) + self.execute_task(iteration_convo, iteration_description, task_steps, is_root_task=True) def set_up_environment(self): From d252c518ec52c81149728f6d69de6d7736f37499 Mon Sep 17 00:00:00 2001 From: Zvonimir Sabljic Date: Wed, 6 Dec 2023 16:25:44 +0100 Subject: [PATCH 5/6] Disable testing of code changes for now --- pilot/helpers/agents/Developer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pilot/helpers/agents/Developer.py b/pilot/helpers/agents/Developer.py index 3e594c9b10..83affe354e 100644 --- a/pilot/helpers/agents/Developer.py +++ b/pilot/helpers/agents/Developer.py @@ -592,6 +592,7 @@ class Developer(Agent): return llm_response def test_code_changes(self, code_monkey, convo): + return { "success": True } logger.info('Testing code changes...') llm_response = convo.send_message('development/task/step_check.prompt', {}, GET_TEST_TYPE) test_type = llm_response['type'] From a074b60077f931d02a232de477c3b380b028dd55 Mon Sep 17 00:00:00 2001 From: Zvonimir Sabljic Date: Wed, 6 Dec 2023 16:37:20 +0100 Subject: [PATCH 6/6] Fixed tests --- pilot/helpers/agents/test_CodeMonkey.py | 10 ++++++---- pilot/helpers/agents/test_Developer.py | 13 +++++-------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/pilot/helpers/agents/test_CodeMonkey.py b/pilot/helpers/agents/test_CodeMonkey.py index a7641b2934..55ba96dd07 100644 --- a/pilot/helpers/agents/test_CodeMonkey.py +++ b/pilot/helpers/agents/test_CodeMonkey.py @@ -46,6 +46,7 @@ class TestCodeMonkey: @patch.object(File, 'insert') def test_implement_code_changes(self, mock_get_dev, mock_save_dev, mock_file_insert): # Given + task_description = "High level description of the task" code_changes_description = "Write the word 'Washington' to a .txt file" self.project.get_all_coded_files = lambda: [] @@ -65,12 +66,12 @@ class TestCodeMonkey: convo.send_message.side_effect = mock_responses if WRITE_TO_FILE: - self.codeMonkey.implement_code_changes(convo, code_changes_description, {}) + self.codeMonkey.implement_code_changes(convo, task_description, code_changes_description, {}) else: # don't write the file, just with patch.object(Project, 'save_file') as mock_save_file: # When - self.codeMonkey.implement_code_changes(convo, code_changes_description, {}) + self.codeMonkey.implement_code_changes(convo, task_description, code_changes_description, {}) # Then mock_save_file.assert_called_once() @@ -85,6 +86,7 @@ class TestCodeMonkey: @patch.object(File, 'insert') def test_implement_code_changes_with_read(self, mock_get_dev, mock_save_dev, mock_file_insert): # Given + task_description = "High level description of the task" code_changes_description = "Read the file called file_to_read.txt and write its content to a file called output.txt" workspace = self.project.root_path update_file(os.path.join(workspace, 'file_to_read.txt'), 'Hello World!\n') @@ -106,11 +108,11 @@ class TestCodeMonkey: convo.send_message.side_effect = mock_responses if WRITE_TO_FILE: - self.codeMonkey.implement_code_changes(convo, code_changes_description, {}) + self.codeMonkey.implement_code_changes(convo, task_description, code_changes_description, {}) else: with patch.object(Project, 'save_file') as mock_save_file: # When - self.codeMonkey.implement_code_changes(convo, code_changes_description, {}) + self.codeMonkey.implement_code_changes(convo, task_description, code_changes_description, {}) # Then clear_directory(workspace) diff --git a/pilot/helpers/agents/test_Developer.py b/pilot/helpers/agents/test_Developer.py index 5564bf9224..bda533c668 100644 --- a/pilot/helpers/agents/test_Developer.py +++ b/pilot/helpers/agents/test_Developer.py @@ -74,7 +74,7 @@ class TestDeveloper: # Then we parse the response correctly and send list of steps to execute_task() assert developer.execute_task.call_count == 1 - assert developer.execute_task.call_args[0][1] == [{'command': 'ls -al'}] + assert developer.execute_task.call_args[0][2] == [{'command': 'ls -al'}] @patch('helpers.AgentConvo.get_saved_development_step') @patch('helpers.AgentConvo.save_development_step') @@ -132,7 +132,7 @@ class TestDeveloper: result = self.developer.test_code_changes(monkey, convo) # Then - assert result == {'success': True, 'cli_response': 'stdout:\n```\n\n```'} + assert result == {'success': True} @patch('helpers.AgentConvo.get_saved_development_step') @patch('helpers.AgentConvo.save_development_step') @@ -150,7 +150,7 @@ class TestDeveloper: result = self.developer.test_code_changes(monkey, convo) # Then - assert result == {'success': True, 'user_input': 'continue'} + assert result == {'success': True} @pytest.mark.skip("endless loop in questionary") @patch('helpers.AgentConvo.get_saved_development_step') @@ -234,8 +234,5 @@ class TestDeveloper: result = self.developer.test_code_changes(monkey, convo) # Then - assert result == {'success': True, 'cli_response': 'stdout:\n```\n\n```'} - assert mock_requests_post.call_count == 3 - assert "The JSON is invalid at $.type - 'command' is not one of " \ - "['command_test', 'manual_test', 'no_test']" in json_received[1][3] - assert mock_execute.call_count == 1 + assert result == {'success': True} + assert mock_requests_post.call_count == 0