You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
582 lines
22 KiB
Python
582 lines
22 KiB
Python
import json
|
|
import os
|
|
import os.path
|
|
import sys
|
|
from argparse import ArgumentParser, ArgumentTypeError, Namespace
|
|
from difflib import unified_diff
|
|
from typing import Optional
|
|
from urllib.parse import urlparse
|
|
from uuid import UUID
|
|
|
|
from core.config import Config, LLMProvider, LocalIPCConfig, ProviderConfig, UIAdapter, get_config, loader
|
|
from core.config.actions import (
|
|
BH_START_BUG_HUNT,
|
|
BH_START_USER_TEST,
|
|
BH_STARTING_PAIR_PROGRAMMING,
|
|
BH_WAIT_BUG_REP_INSTRUCTIONS,
|
|
CM_UPDATE_FILES,
|
|
DEV_TASK_BREAKDOWN,
|
|
DEV_TROUBLESHOOT,
|
|
MIX_BREAKDOWN_CHAT_PROMPT,
|
|
TC_TASK_DONE,
|
|
)
|
|
from core.config.env_importer import import_from_dotenv
|
|
from core.config.version import get_version
|
|
from core.db.session import SessionManager
|
|
from core.db.setup import run_migrations
|
|
from core.log import get_logger, setup
|
|
from core.state.state_manager import StateManager
|
|
from core.ui.base import UIBase
|
|
from core.ui.console import PlainConsoleUI
|
|
from core.ui.ipc_client import IPCClientUI
|
|
from core.ui.virtual import VirtualUI
|
|
|
|
log = get_logger(__name__)
|
|
|
|
|
|
def parse_llm_endpoint(value: str) -> Optional[tuple[LLMProvider, str]]:
|
|
"""
|
|
Parse --llm-endpoint command-line option.
|
|
|
|
Option syntax is: --llm-endpoint <provider>:<url>
|
|
|
|
:param value: Argument value.
|
|
:return: Tuple with LLM provider and URL, or None if the option wasn't provided.
|
|
"""
|
|
if not value:
|
|
return None
|
|
|
|
parts = value.split(":", 1)
|
|
if len(parts) != 2:
|
|
raise ArgumentTypeError("Invalid LLM endpoint format; expected 'provider:url'")
|
|
|
|
try:
|
|
provider = LLMProvider(parts[0])
|
|
except ValueError as err:
|
|
raise ArgumentTypeError(f"Unsupported LLM provider: {err}")
|
|
url = urlparse(parts[1])
|
|
if url.scheme not in ("http", "https"):
|
|
raise ArgumentTypeError(f"Invalid LLM endpoint URL: {parts[1]}")
|
|
|
|
return provider, url.geturl()
|
|
|
|
|
|
def get_line_changes(old_content: str, new_content: str) -> tuple[int, int]:
|
|
"""
|
|
Get the number of added and deleted lines between two files.
|
|
|
|
This uses Python difflib to produce a unified diff, then counts
|
|
the number of added and deleted lines.
|
|
|
|
:param old_content: old file content
|
|
:param new_content: new file content
|
|
:return: a tuple (added_lines, deleted_lines)
|
|
"""
|
|
|
|
from_lines = old_content.splitlines(keepends=True)
|
|
to_lines = new_content.splitlines(keepends=True)
|
|
|
|
diff_gen = unified_diff(from_lines, to_lines)
|
|
|
|
added_lines = 0
|
|
deleted_lines = 0
|
|
|
|
for line in diff_gen:
|
|
if line.startswith("+") and not line.startswith("+++"): # Exclude the file headers
|
|
added_lines += 1
|
|
elif line.startswith("-") and not line.startswith("---"): # Exclude the file headers
|
|
deleted_lines += 1
|
|
|
|
return added_lines, deleted_lines
|
|
|
|
|
|
def parse_llm_key(value: str) -> Optional[tuple[LLMProvider, str]]:
|
|
"""
|
|
Parse --llm-key command-line option.
|
|
|
|
Option syntax is: --llm-key <provider>:<key>
|
|
|
|
:param value: Argument value.
|
|
:return: Tuple with LLM provider and key, or None if if the option wasn't provided.
|
|
"""
|
|
if not value:
|
|
return None
|
|
|
|
parts = value.split(":", 1)
|
|
if len(parts) != 2:
|
|
raise ArgumentTypeError("Invalid LLM endpoint format; expected 'provider:key'")
|
|
|
|
try:
|
|
provider = LLMProvider(parts[0])
|
|
except ValueError as err:
|
|
raise ArgumentTypeError(f"Unsupported LLM provider: {err}")
|
|
|
|
return provider, parts[1]
|
|
|
|
|
|
def parse_arguments() -> Namespace:
|
|
"""
|
|
Parse command-line arguments.
|
|
|
|
Available arguments:
|
|
--help: Show the help message
|
|
--config: Path to the configuration file
|
|
--show-config: Output the default configuration to stdout
|
|
--default-config: Output the configuration to stdout
|
|
--level: Log level (debug,info,warning,error,critical)
|
|
--database: Database URL
|
|
--local-ipc-port: Local IPC port to connect to
|
|
--local-ipc-host: Local IPC host to connect to
|
|
--version: Show the version and exit
|
|
--list: List all projects
|
|
--list-json: List all projects in JSON format
|
|
--project: Load a specific project
|
|
--branch: Load a specific branch
|
|
--step: Load a specific step in a project/branch
|
|
--llm-endpoint: Use specific API endpoint for the given provider
|
|
--llm-key: Use specific LLM key for the given provider
|
|
--import-v0: Import data from a v0 (gpt-pilot) database with the given path
|
|
--email: User's email address, if provided
|
|
--extension-version: Version of the VSCode extension, if used
|
|
--use-git: Use Git for version control
|
|
--access-token: Access token
|
|
:return: Parsed arguments object.
|
|
"""
|
|
version = get_version()
|
|
|
|
parser = ArgumentParser()
|
|
parser.add_argument("--config", help="Path to the configuration file", default="config.json")
|
|
parser.add_argument("--show-config", help="Output the default configuration to stdout", action="store_true")
|
|
parser.add_argument("--level", help="Log level (debug,info,warning,error,critical)", required=False)
|
|
parser.add_argument("--database", help="Database URL", required=False)
|
|
parser.add_argument("--local-ipc-port", help="Local IPC port to connect to", type=int, required=False)
|
|
parser.add_argument("--local-ipc-host", help="Local IPC host to connect to", default="localhost", required=False)
|
|
parser.add_argument("--version", action="version", version=version)
|
|
parser.add_argument("--list", help="List all projects", action="store_true")
|
|
parser.add_argument("--list-json", help="List all projects in JSON format", action="store_true")
|
|
parser.add_argument("--project", help="Load a specific project", type=UUID, required=False)
|
|
parser.add_argument("--branch", help="Load a specific branch", type=UUID, required=False)
|
|
parser.add_argument("--step", help="Load a specific step in a project/branch", type=int, required=False)
|
|
parser.add_argument("--delete", help="Delete a specific project", type=UUID, required=False)
|
|
parser.add_argument(
|
|
"--llm-endpoint",
|
|
help="Use specific API endpoint for the given provider",
|
|
type=parse_llm_endpoint,
|
|
action="append",
|
|
required=False,
|
|
)
|
|
parser.add_argument(
|
|
"--llm-key",
|
|
help="Use specific LLM key for the given provider",
|
|
type=parse_llm_key,
|
|
action="append",
|
|
required=False,
|
|
)
|
|
parser.add_argument(
|
|
"--import-v0",
|
|
help="Import data from a v0 (gpt-pilot) database with the given path",
|
|
required=False,
|
|
)
|
|
parser.add_argument("--email", help="User's email address", required=False)
|
|
parser.add_argument("--extension-version", help="Version of the VSCode extension", required=False)
|
|
parser.add_argument("--use-git", help="Use Git for version control", action="store_true", required=False)
|
|
parser.add_argument("--access-token", help="Access token", required=False)
|
|
return parser.parse_args()
|
|
|
|
|
|
def load_config(args: Namespace) -> Optional[Config]:
|
|
"""
|
|
Load Pythagora JSON configuration file and apply command-line arguments.
|
|
|
|
:param args: Command-line arguments (at least `config` must be present).
|
|
:return: Configuration object, or None if config couldn't be loaded.
|
|
"""
|
|
if not os.path.isfile(args.config):
|
|
imported = import_from_dotenv(args.config)
|
|
if not imported:
|
|
print(f"Configuration file not found: {args.config}; using default", file=sys.stderr)
|
|
return get_config()
|
|
|
|
try:
|
|
config = loader.load(args.config)
|
|
except ValueError as err:
|
|
print(f"Error parsing config file {args.config}: {err}", file=sys.stderr)
|
|
return None
|
|
|
|
if args.level:
|
|
config.log.level = args.level.upper()
|
|
|
|
if args.database:
|
|
config.db.url = args.database
|
|
|
|
if args.local_ipc_port:
|
|
config.ui = LocalIPCConfig(port=args.local_ipc_port, host=args.local_ipc_host)
|
|
|
|
if args.llm_endpoint:
|
|
for provider, endpoint in args.llm_endpoint:
|
|
if provider not in config.llm:
|
|
config.llm[provider] = ProviderConfig()
|
|
config.llm[provider].base_url = endpoint
|
|
|
|
if args.llm_key:
|
|
for provider, key in args.llm_key:
|
|
if provider not in config.llm:
|
|
config.llm[provider] = ProviderConfig()
|
|
config.llm[provider].api_key = key
|
|
|
|
try:
|
|
Config.model_validate(config)
|
|
except ValueError as err:
|
|
print(f"Configuration error: {err}", file=sys.stderr)
|
|
return None
|
|
|
|
return config
|
|
|
|
|
|
async def list_projects_json(db: SessionManager):
|
|
"""
|
|
List all projects in the database in JSON format.
|
|
"""
|
|
sm = StateManager(db)
|
|
projects = await sm.list_projects()
|
|
|
|
data = []
|
|
for project in projects:
|
|
last_updated = None
|
|
p = {
|
|
"name": project.name,
|
|
"id": project.id.hex,
|
|
"branches": [],
|
|
}
|
|
for branch in project.branches:
|
|
b = {
|
|
"name": branch.name,
|
|
"id": branch.id.hex,
|
|
"steps": [],
|
|
}
|
|
for state in branch.states:
|
|
if not last_updated or state.created_at > last_updated:
|
|
last_updated = state.created_at
|
|
s = {
|
|
"name": state.action or f"Step #{state.step_index}",
|
|
"step": state.step_index,
|
|
}
|
|
b["steps"].append(s)
|
|
if b["steps"]:
|
|
b["steps"][-1]["name"] = "Latest step"
|
|
p["branches"].append(b)
|
|
p["updated_at"] = last_updated.isoformat() if last_updated else None
|
|
data.append(p)
|
|
|
|
print(json.dumps(data, indent=2))
|
|
|
|
|
|
def find_first_todo_task(tasks):
|
|
"""
|
|
Find the first task with status 'todo' from a list of tasks.
|
|
|
|
:param tasks: List of task objects
|
|
:return: First task with status 'todo', or None if not found
|
|
"""
|
|
for task in tasks:
|
|
if task.get("status") == "todo":
|
|
return task
|
|
return None
|
|
|
|
|
|
def trim_logs(logs: str) -> str:
|
|
"""
|
|
Trim logs by removing everything after specific marker phrases.
|
|
|
|
This function cuts off the string at the first occurrence of
|
|
"Here are the backend logs" or "Here are the frontend logs".
|
|
|
|
:param logs: Log text to trim
|
|
:return: Trimmed log text with the marker phrase removed
|
|
"""
|
|
if not logs:
|
|
return ""
|
|
|
|
# Define marker phrases
|
|
markers = ["Here are the backend logs", "Here are the frontend logs"]
|
|
|
|
# Find the first occurrence of any marker
|
|
index = float("inf")
|
|
for marker in markers:
|
|
pos = logs.find(marker)
|
|
if pos != -1 and pos < index:
|
|
index = pos
|
|
|
|
# If a marker was found, trim the string
|
|
if index != float("inf"):
|
|
return logs[:index]
|
|
|
|
return logs
|
|
|
|
|
|
async def load_convo(
|
|
sm: StateManager,
|
|
project_id: Optional[UUID] = None,
|
|
step_index: Optional[int] = None,
|
|
) -> list:
|
|
"""
|
|
List all projects in the database.
|
|
"""
|
|
convo = []
|
|
|
|
project_states = await sm.get_project_states(project_id)
|
|
project_states = [state for state in project_states if 0 <= state.step_index <= step_index]
|
|
|
|
branches = await sm.get_branches_for_project_id(project_id)
|
|
branch_id = branches[0].id
|
|
|
|
task_counter = 1
|
|
|
|
for i, state in enumerate(project_states):
|
|
prev_state = project_states[i - 1] if i > 0 else None
|
|
|
|
convo_el = {}
|
|
convo_el["id"] = str(state.id)
|
|
user_inputs = await sm.find_user_input(state, branch_id)
|
|
|
|
if user_inputs:
|
|
convo_el["user_inputs"] = []
|
|
for ui in user_inputs:
|
|
if ui.question:
|
|
answer = trim_logs(ui.answer_text) if ui.answer_text is not None else ui.answer_button
|
|
convo_el["user_inputs"].append({"question": ui.question, "answer": answer})
|
|
|
|
if state.action is not None:
|
|
if "Task" in state.action and "start" in state.action:
|
|
task_counter = int(state.action.split("#")[1].split()[0])
|
|
|
|
elif state.action == DEV_TROUBLESHOOT.format(task_counter):
|
|
if state.iterations is not None:
|
|
si = state.iterations[-1]
|
|
if si is not None:
|
|
if si["user_feedback"] is not None:
|
|
convo_el["user_feedback"] = si["user_feedback"]
|
|
if si["description"] is not None:
|
|
convo_el["description"] = si["description"]
|
|
|
|
elif state.action == DEV_TASK_BREAKDOWN.format(task_counter):
|
|
task = state.tasks[task_counter - 1]
|
|
if "description" in task and task["description"] is not None:
|
|
convo_el["description"] = task["description"]
|
|
|
|
if "instructions" in task and task["instructions"] is not None:
|
|
convo_el["instructions"] = task["instructions"]
|
|
|
|
elif state.action == TC_TASK_DONE.format(task_counter):
|
|
if state.tasks:
|
|
# find the next task description and print it in question
|
|
next_task = find_first_todo_task(state.tasks)
|
|
# next_task_alt = find_first_todo_task(prev_state.tasks)
|
|
if next_task:
|
|
convo_el["task_description"] = next_task["description"]
|
|
task = state.tasks[task_counter - 1]
|
|
if "test_instructions" in task and task["test_instructions"] is not None:
|
|
convo_el["test_instructions"] = task["test_instructions"]
|
|
|
|
elif state.action == CM_UPDATE_FILES:
|
|
files = []
|
|
for steps in state.steps:
|
|
file = {}
|
|
if "save_file" in steps and "path" in steps["save_file"]:
|
|
path = steps["save_file"]["path"]
|
|
file["path"] = path
|
|
|
|
current_file = await sm.get_file_for_project(state.id, path)
|
|
prev_file = await sm.get_file_for_project(prev_state.id, path)
|
|
|
|
if current_file and prev_file:
|
|
file["diff"] = get_line_changes(
|
|
old_content=prev_file.content.content, new_content=current_file.content.content
|
|
)
|
|
files.append(file)
|
|
|
|
convo_el["files"] = files
|
|
|
|
elif state.action == BH_START_BUG_HUNT.format(task_counter):
|
|
si = state.iterations[-1]
|
|
if si is not None:
|
|
if "user_feedback" in si and si["user_feedback"] is not None:
|
|
convo_el["user_feedback"] = si["user_feedback"]
|
|
|
|
if "description" in si and si["description"] is not None:
|
|
convo_el["description"] = si["description"]
|
|
|
|
elif state.action == BH_WAIT_BUG_REP_INSTRUCTIONS.format(task_counter):
|
|
if state.iterations is not None:
|
|
for si in state.iterations:
|
|
if "bug_reproduction_description" in si and si["bug_reproduction_description"] is not None:
|
|
convo_el["bug_reproduction_description"] = si["bug_reproduction_description"]
|
|
|
|
elif state.action == BH_START_USER_TEST.format(task_counter):
|
|
si = state.iterations[-1]
|
|
if si is not None:
|
|
if "bug_hunting_cycles" in si and si["bug_hunting_cycles"] is not None:
|
|
cycle = si["bug_hunting_cycles"][-1]
|
|
if cycle is not None:
|
|
if "user_feedback" in cycle and cycle["user_feedback"] is not None:
|
|
convo_el["user_feedback"] = cycle["user_feedback"]
|
|
if (
|
|
"human_readable_instructions" in cycle
|
|
and cycle["human_readable_instructions"] is not None
|
|
):
|
|
convo_el["human_readable_instructions"] = cycle["human_readable_instructions"]
|
|
|
|
elif state.action == BH_STARTING_PAIR_PROGRAMMING.format(task_counter):
|
|
si = state.iterations[-1]
|
|
if si is not None:
|
|
if "user_feedback" in si and si["user_feedback"] is not None:
|
|
convo_el["user_feedback"] = si["user_feedback"]
|
|
if "initial_explanation" in si and si["initial_explanation"] is not None:
|
|
convo_el["initial_explanation"] = si["initial_explanation"]
|
|
|
|
if convo_el.get("user_inputs", None) is not None or convo_el.get("files", None) is not None:
|
|
for j, ui in enumerate(user_inputs):
|
|
if ui.question and ui.question == MIX_BREAKDOWN_CHAT_PROMPT:
|
|
if len(state.tasks) == 1:
|
|
if state.tasks[0]["instructions"] is not None:
|
|
# convo_el["user_inputs"].insert(j, {'question': state.tasks[task_counter - 1]["instructions"]})
|
|
convo_el["breakdown"] = state.tasks[task_counter - 1]["instructions"]
|
|
break
|
|
elif (
|
|
state.tasks[task_counter - 1] is not None
|
|
and state.tasks[task_counter - 1]["instructions"] is not None
|
|
):
|
|
# convo_el["user_inputs"].insert(j, {'question': state.tasks[task_counter - 1]["instructions"]})
|
|
convo_el["breakdown"] = state.tasks[task_counter - 1]["instructions"]
|
|
break
|
|
elif state.iterations:
|
|
if (
|
|
state.iterations[-1].get("bug_hunting_cycles", None) is not None
|
|
and state.iterations[-1]["bug_hunting_cycles"]
|
|
and state.iterations[-1]["bug_hunting_cycles"][-1].get("human_readable_instructions", None)
|
|
is not None
|
|
):
|
|
# convo_el["user_inputs"].insert(j, {'question': state.iterations[-1]["bug_hunting_cycles"][-1]["human_readable_instructions"]})
|
|
convo_el["breakdown"] = state.iterations[-1]["bug_hunting_cycles"][-1][
|
|
"human_readable_instructions"
|
|
]
|
|
break
|
|
else:
|
|
# try to get the next state's instructions from the task
|
|
next_state = project_states[i + 1]
|
|
if next_state.tasks is not None:
|
|
next_task = find_first_todo_task(next_state.tasks)
|
|
if next_task:
|
|
# convo_el["user_inputs"].insert(j, {'question': next_task["description"]})
|
|
convo_el["task_description"] = next_task["description"]
|
|
break
|
|
|
|
convo_el["action"] = state.action
|
|
convo.append(convo_el)
|
|
|
|
return convo
|
|
|
|
|
|
async def list_projects(db: SessionManager):
|
|
"""
|
|
List all projects in the database.
|
|
"""
|
|
sm = StateManager(db)
|
|
projects = await sm.list_projects()
|
|
|
|
print(f"Available projects ({len(projects)}):")
|
|
for project in projects:
|
|
print(f"* {project.name} ({project.id})")
|
|
for branch in project.branches:
|
|
last_step = max(state.step_index for state in branch.states)
|
|
print(f" - {branch.name} ({branch.id}) - last step: {last_step}")
|
|
|
|
|
|
async def load_project(
|
|
sm: StateManager,
|
|
project_id: Optional[UUID] = None,
|
|
branch_id: Optional[UUID] = None,
|
|
step_index: Optional[int] = None,
|
|
) -> bool:
|
|
"""
|
|
Load a project from the database.
|
|
|
|
:param sm: State manager.
|
|
:param project_id: Project ID (optional, loads the last step in the main branch).
|
|
:param branch_id: Branch ID (optional, loads the last step in the branch).
|
|
:param step_index: Step index (optional, loads the state at the given step).
|
|
:return: True if the project was loaded successfully, False otherwise.
|
|
"""
|
|
step_txt = f" step {step_index}" if step_index else ""
|
|
|
|
if branch_id:
|
|
project_state = await sm.load_project(branch_id=branch_id, step_index=step_index)
|
|
if project_state:
|
|
return True
|
|
else:
|
|
print(f"Branch {branch_id}{step_txt} not found; use --list to list all projects", file=sys.stderr)
|
|
return False
|
|
|
|
elif project_id:
|
|
project_state = await sm.load_project(project_id=project_id, step_index=step_index)
|
|
if project_state:
|
|
return True
|
|
else:
|
|
print(f"Project {project_id}{step_txt} not found; use --list to list all projects", file=sys.stderr)
|
|
return False
|
|
|
|
return False
|
|
|
|
|
|
async def delete_project(db: SessionManager, project_id: UUID) -> bool:
|
|
"""
|
|
Delete a project from a database.
|
|
|
|
:param sm: State manager.
|
|
:param project_id: Project ID.
|
|
:return: True if project was deleted, False otherwise.
|
|
"""
|
|
|
|
sm = StateManager(db)
|
|
return await sm.delete_project(project_id)
|
|
|
|
|
|
def show_config():
|
|
"""
|
|
Print the current configuration to stdout.
|
|
"""
|
|
cfg = get_config()
|
|
print(cfg.model_dump_json(indent=2))
|
|
|
|
|
|
def init() -> tuple[UIBase, SessionManager, Namespace]:
|
|
"""
|
|
Initialize the application.
|
|
|
|
Loads configuration, sets up logging and UI, initializes the database
|
|
and runs database migrations.
|
|
|
|
:return: Tuple with UI, db session manager, file manager, and command-line arguments.
|
|
"""
|
|
args = parse_arguments()
|
|
config = load_config(args)
|
|
if not config:
|
|
return (None, None, args)
|
|
|
|
setup(config.log, force=True)
|
|
|
|
if config.ui.type == UIAdapter.IPC_CLIENT:
|
|
ui = IPCClientUI(config.ui)
|
|
elif config.ui.type == UIAdapter.VIRTUAL:
|
|
ui = VirtualUI(config.ui.inputs)
|
|
else:
|
|
ui = PlainConsoleUI()
|
|
|
|
run_migrations(config.db)
|
|
db = SessionManager(config.db)
|
|
|
|
return (ui, db, args)
|
|
|
|
|
|
__all__ = ["parse_arguments", "load_config", "list_projects_json", "list_projects", "load_project", "init"]
|