You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
gpt-pilot/pilot/helpers/test_Debugger.py

152 lines
5.6 KiB
Python

import builtins
import json
import pytest
from unittest.mock import patch, MagicMock
from dotenv import load_dotenv
load_dotenv()
from utils.custom_print import get_custom_print
from helpers.agents.Developer import Developer
from helpers.AgentConvo import AgentConvo
from helpers.Debugger import Debugger
from helpers.test_Project import create_project
from test.mock_questionary import MockQuestionary
################## NOTE: this test needs to be ran in debug with breakpoints ##################
@pytest.mark.uses_tokens
@patch('pilot.helpers.AgentConvo.get_saved_development_step')
@patch('pilot.helpers.AgentConvo.save_development_step')
@patch('utils.questionary.get_saved_user_input')
@patch('utils.questionary.save_user_input')
@patch('helpers.cli.get_saved_command_run')
@patch('helpers.cli.run_command')
@patch('helpers.cli.save_command_run')
# @patch('pilot.helpers.cli.execute_command', return_value=('', 'DONE', 0))
def test_debug(
# mock_execute_command,
mock_save_command, mock_run_command, mock_get_saved_command,
mock_save_input, mock_user_input, mock_save_step, mock_get_saved_step):
# Given
builtins.print, ipc_client_instance = get_custom_print({})
project = create_project()
project.current_step = 'coding'
developer = Developer(project)
project.developer = developer
convo = AgentConvo(developer)
convo.load_branch = lambda x: None
debugger = Debugger(developer)
# TODO: mock agent.project.developer.execute_task
# convo.messages.append()
convo.construct_and_add_message_from_prompt('dev_ops/ran_command.prompt', {
'cli_response': '''
stderr:
```
node:internal/modules/cjs/loader:1080
throw err;
^
Error: Cannot find module 'mime'
Require stack:
- /workspace/chat_app/node_modules/send/index.js
- /workspace/chat_app/node_modules/express/lib/utils.js
- /workspace/chat_app/node_modules/express/lib/application.js
- /workspace/chat_app/node_modules/express/lib/express.js
- /workspace/chat_app/node_modules/express/index.js
- /workspace/chat_app/server.js
at Module._resolveFilename (node:internal/modules/cjs/loader:1077:15)
at Module._load (node:internal/modules/cjs/loader:922:27)
at Module.require (node:internal/modules/cjs/loader:1143:19)
at require (node:internal/modules/cjs/helpers:121:18)
at Object.<anonymous> (/workspace/chat_app/node_modules/send/index.js:24:12)
at Module._compile (node:internal/modules/cjs/loader:1256:14)
at Module._extensions..js (node:internal/modules/cjs/loader:1310:10)
at Module.load (node:internal/modules/cjs/loader:1119:32)
at Module._load (node:internal/modules/cjs/loader:960:12)
```
stdout:
```
> chat_app@1.0.0 start
> node server.js
```
'''
})
mock_questionary = MockQuestionary(['', ''])
with patch('utils.questionary.questionary', mock_questionary):
# When
result = debugger.debug(convo, command={'command': 'npm run start'}, is_root_task=True)
# Then
assert result == {'success': True}
@patch('helpers.AgentConvo.get_saved_development_step')
@patch('helpers.AgentConvo.create_gpt_chat_completion')
@patch('helpers.AgentConvo.save_development_step')
def test_debug_need_to_see_output(mock_save_step, mock_get_completion, mock_get_step):
# Given
builtins.print, ipc_client_instance = get_custom_print({})
project = create_project()
project.current_step = 'coding'
developer = Developer(project)
# hard-wired command output
developer.step_command_run = MagicMock()
developer.step_command_run.side_effect = [
{
'cli_response': 'stdout:\n```\n' + json.dumps({'dependencies': {'something': '0.1.2'}}) + '\n```\n',
'success': True,
}, {
'cli_response': 'app is running...',
'success': True,
}
]
developer.step_test = MagicMock()
developer.step_test.return_value = {'success': True}
project.developer = developer
debugger = Debugger(developer)
convo = AgentConvo(developer)
convo.load_branch = MagicMock()
convo.replace_files = MagicMock()
# hard-wired LLM responses, 1st response asks to see output
mock_get_completion.side_effect = [{'text': json.dumps(response)} for response in [{
'thoughts': 'Hmmm, testing',
'reasoning': 'I need to see the output of the command',
'steps': [
{
'type': 'command',
'command': 'cat package.json',
'need_to_see_output': True
}, {
'type': 'command',
'command': 'npm install something',
}
]
}, {
'thoughts': 'It is already installed',
'reasoning': 'I installed it earlier',
'steps': [
{
'type': 'command',
'command': 'npm start',
'command_id': 'app',
}
]
}]]
# When
debugger.debug(convo, command={'command': 'npm run start'}, is_root_task=True)
# Then we call the LLM twice, second time to show the output
assert mock_get_completion.call_count == 2
prompt = mock_get_completion.call_args_list[1].args[0][2]['content']
assert prompt.startswith('{"thoughts": "It is already installed", "reasoning": "I installed it earlier", "steps": [{"type": "command", "command": "npm start", "command_id": "app"}]}'.lstrip())
# And eventually we start the app
assert developer.step_command_run.call_count == 2
assert developer.step_command_run.call_args_list[1].args[1]['command'] == 'npm start'