You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/gateway/update/runner.go

625 lines
20 KiB
Go

package update
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
)
const maxCommandOutput = 2 << 20
var commitPattern = regexp.MustCompile(`^[0-9a-f]{40,64}$`)
type CommandExecutor interface {
Run(ctx context.Context, directory string, environment []string, command string, arguments ...string) ([]byte, error)
}
type OSCommandExecutor struct{}
func (OSCommandExecutor) Run(ctx context.Context, directory string, environment []string, command string, arguments ...string) ([]byte, error) {
if command == "git" {
arguments = append([]string{"-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", "-c", "credential.helper="}, arguments...)
}
cmd := exec.CommandContext(ctx, command, arguments...)
cmd.Dir = directory
cmd.Env = commandEnvironment(environment)
var output bytes.Buffer
writer := &limitedWriter{writer: &output, remaining: maxCommandOutput}
cmd.Stdout = writer
cmd.Stderr = writer
err := cmd.Run()
return output.Bytes(), err
}
func commandEnvironment(overrides []string) []string {
values := make(map[string]string, len(os.Environ())+len(overrides))
for _, entry := range os.Environ() {
key, value, ok := strings.Cut(entry, "=")
if !ok || unsafeCommandEnvironmentKey(key) {
continue
}
values[key] = value
}
for _, entry := range overrides {
key, value, ok := strings.Cut(entry, "=")
if ok && key != "" {
values[key] = value
}
}
result := make([]string, 0, len(values))
for key, value := range values {
result = append(result, key+"="+value)
}
return result
}
func unsafeCommandEnvironmentKey(key string) bool {
if strings.HasPrefix(key, "GIT_CONFIG_KEY_") || strings.HasPrefix(key, "GIT_CONFIG_VALUE_") ||
strings.HasPrefix(key, "DYLD_") {
return true
}
switch key {
case "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY",
"GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_CONFIG", "GIT_CONFIG_COUNT",
"GIT_SSH", "GIT_SSH_COMMAND", "GIT_PROXY_COMMAND", "LD_PRELOAD":
return true
default:
return false
}
}
type limitedWriter struct {
mu sync.Mutex
writer *bytes.Buffer
remaining int
}
func (w *limitedWriter) Write(value []byte) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()
original := len(value)
if w.remaining > 0 {
amount := len(value)
if amount > w.remaining {
amount = w.remaining
}
_, _ = w.writer.Write(value[:amount])
w.remaining -= amount
}
return original, nil
}
type CheckResult struct {
Repository string `json:"repository"`
CurrentCommit string `json:"current_commit"`
RemoteCommit string `json:"remote_commit"`
UpdateAvailable bool `json:"update_available"`
Status string `json:"status"`
Branch string `json:"branch"`
MaintenanceLock bool `json:"maintenance_lock"`
}
type SettingsResult struct {
Repository string `json:"repository"`
Branch string `json:"branch"`
CurrentCommit string `json:"current_commit"`
CheckoutReady bool `json:"checkout_ready"`
CheckoutError string `json:"checkout_error,omitempty"`
TokenConfigured bool `json:"token_configured"`
ActiveJobID string `json:"active_job_id,omitempty"`
LastJobID string `json:"last_job_id,omitempty"`
MaintenanceLock bool `json:"maintenance_lock"`
}
type Manager struct {
config Config
state *StateStore
executor CommandExecutor
timeout time.Duration
healthTimeout time.Duration
healthPollInterval time.Duration
}
func NewManager(config Config, state *StateStore, executor CommandExecutor) (*Manager, error) {
if err := config.Validate(); err != nil {
return nil, err
}
if state == nil || executor == nil {
return nil, errors.New("update state and command executor are required")
}
return &Manager{
config: config,
state: state,
executor: executor,
timeout: 10 * time.Minute,
healthTimeout: 60 * time.Second,
healthPollInterval: time.Second,
}, nil
}
func (m *Manager) Check(ctx context.Context) (CheckResult, error) {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
current, err := m.validateLocalRepository(ctx)
if err != nil {
return CheckResult{}, err
}
remote, err := m.remoteCommit(ctx)
if err != nil {
return CheckResult{}, err
}
status := "current"
if current != remote {
status = "update_available"
}
return CheckResult{Repository: RequiredRemoteURL, CurrentCommit: current, RemoteCommit: remote, UpdateAvailable: current != remote, Status: status,
Branch: RequiredBranch, MaintenanceLock: m.state.MaintenanceLocked()}, nil
}
func (m *Manager) Settings(ctx context.Context) SettingsResult {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
active, last := m.state.JobPointers()
result := SettingsResult{Repository: RequiredRemoteURL, Branch: RequiredBranch,
TokenConfigured: m.state.TokenConfigured(), ActiveJobID: active, LastJobID: last,
MaintenanceLock: m.state.MaintenanceLocked()}
commit, err := m.validateLocalRepository(ctx)
if err != nil {
result.CheckoutError = "仓库分支、远端或工作树状态不符合更新要求"
return result
}
result.CurrentCommit = commit
result.CheckoutReady = true
return result
}
func (m *Manager) StartApply(expectedCurrent, expectedRemote string) (Job, error) {
if !commitPattern.MatchString(expectedCurrent) || !commitPattern.MatchString(expectedRemote) {
return Job{}, errors.New("expected commits must be lowercase full object IDs")
}
if expectedCurrent == expectedRemote {
return Job{}, errors.New("remote commit does not contain an update")
}
id, err := randomID()
if err != nil {
return Job{}, err
}
job := Job{ID: id, ExpectedCurrentCommit: expectedCurrent, ExpectedRemoteCommit: expectedRemote}
if err := m.state.CreateJob(job); err != nil {
return Job{}, err
}
job, _ = m.state.Job(id)
go m.runApply(id)
return job, nil
}
func (m *Manager) runApply(id string) {
ctx, cancel := context.WithTimeout(context.Background(), m.timeout)
defer cancel()
job, ok := m.state.Job(id)
if !ok {
return
}
failSafe := func(code string) {
if err := m.state.UpdateJob(id, func(value *Job) {
value.Status = "failed"
value.ErrorCode = code
}); err != nil {
m.state.ForceManual(id, "state_persistence_failed")
}
}
failUncertain := func(code string) {
if err := m.state.UpdateJob(id, func(value *Job) {
value.Status = "manual_intervention"
value.ErrorCode = code
}); err != nil {
m.state.ForceManual(id, "state_persistence_failed")
}
}
checkpoint := func(name string) error {
err := m.state.UpdateJob(id, func(value *Job) {
value.Status = "running"
value.Checkpoint = name
})
if err != nil {
m.state.ForceManual(id, "state_persistence_failed")
}
return err
}
if err := checkpoint("validating_repository"); err != nil {
return
}
current, err := m.validateLocalRepository(ctx)
if err != nil || current != job.ExpectedCurrentCommit {
failSafe("local_repository_precondition_failed")
return
}
if err := checkpoint("fetching_expected_commit"); err != nil {
return
}
if err := m.fetch(ctx); err != nil {
failSafe("remote_fetch_failed")
return
}
remote, err := m.remoteTrackingCommit(ctx)
if err != nil || remote != job.ExpectedRemoteCommit {
failSafe("remote_commit_changed")
return
}
if err := m.ensureFastForward(ctx, current, remote); err != nil {
failSafe("non_fast_forward_update_rejected")
return
}
changed, err := m.changedPaths(ctx, current, remote)
if err != nil {
failSafe("change_inspection_failed")
return
}
if deniedChangedPath(changed) {
failSafe("manual_upgrade_required")
return
}
if err := checkpoint("preflight"); err != nil {
return
}
if err := m.preflight(ctx, remote); err != nil {
failSafe("host_preflight_failed")
return
}
if err := m.state.UpdateJob(id, func(value *Job) { value.PreviousCommit = current }); err != nil {
m.state.ForceManual(id, "state_persistence_failed")
return
}
// Every checkpoint below precedes a host mutation. A crash or any error is
// deliberately considered uncertain and holds the maintenance lock.
if err := checkpoint("stopping_services"); err != nil {
return
}
if err := m.stopServices(ctx); err != nil {
failUncertain("service_stop_requires_reconciliation")
return
}
if err := checkpoint("updating_source"); err != nil {
return
}
if _, err := m.run(ctx, nil, "git", "merge", "--ff-only", "refs/remotes/origin/"+RequiredBranch); err != nil {
failUncertain("source_update_requires_reconciliation")
return
}
if err := m.state.UpdateJob(id, func(value *Job) {
value.Status = "running"
value.Checkpoint = "source_updated"
value.AppliedCommit = remote
}); err != nil {
m.state.ForceManual(id, "state_persistence_failed")
return
}
if err := checkpoint("schema_init_started"); err != nil {
return
}
webDirectory, err := m.webDirectory()
if err != nil {
failUncertain("schema_init_result_uncertain")
return
}
if _, err := m.runAt(ctx, webDirectory, nil, m.config.PHPCommand, "xcat", "cloudInit"); err != nil {
failUncertain("schema_init_result_uncertain")
return
}
if err := checkpoint("schema_init_completed"); err != nil {
return
}
if err := checkpoint("starting_services"); err != nil {
return
}
if err := m.startServices(ctx); err != nil {
failUncertain("service_start_requires_reconciliation")
return
}
if err := checkpoint("health_checking"); err != nil {
return
}
if err := m.healthCheck(ctx); err != nil {
failUncertain("post_update_health_check_failed")
return
}
if err := m.state.UpdateJob(id, func(value *Job) {
value.Status = "succeeded"
value.Checkpoint = "completed"
value.ErrorCode = ""
}); err != nil {
m.state.ForceManual(id, "state_persistence_failed")
}
}
func (m *Manager) validateLocalRepository(ctx context.Context) (string, error) {
repository, err := filepath.EvalSymlinks(m.config.RepositoryDir)
if err != nil {
return "", errors.New("repository is unavailable")
}
root, err := m.gitLine(ctx, "rev-parse", "--show-toplevel")
if err != nil {
return "", errors.New("repository cannot be identified")
}
resolvedRoot, err := filepath.EvalSymlinks(root)
if err != nil || resolvedRoot != repository {
return "", errors.New("repository path does not match its Git root")
}
branch, err := m.gitLine(ctx, "symbolic-ref", "--short", "HEAD")
if err != nil || branch != RequiredBranch {
return "", errors.New("repository is not on the go branch")
}
remote, err := m.gitLine(ctx, "remote", "get-url", "origin")
if err != nil || remote != RequiredRemoteURL {
return "", errors.New("repository origin is not approved")
}
statePath, err := filepath.EvalSymlinks(m.config.StatePath)
if err != nil {
return "", errors.New("update state is unavailable")
}
stateRelative, err := filepath.Rel(repository, statePath)
if err != nil || strings.HasPrefix(stateRelative, ".."+string(filepath.Separator)) {
return "", errors.New("update state is outside the repository runtime directory")
}
if _, err := m.run(ctx, nil, "git", "check-ignore", "-q", "--", filepath.ToSlash(stateRelative)); err != nil {
return "", errors.New("update state path is not ignored by Git")
}
status, err := m.run(ctx, nil, "git", "status", "--porcelain=v1", "--untracked-files=normal")
if err != nil || len(status) != 0 {
return "", errors.New("repository worktree is not clean")
}
commit, err := m.gitLine(ctx, "rev-parse", "HEAD")
if err != nil || !commitPattern.MatchString(commit) {
return "", errors.New("current commit is invalid")
}
return commit, nil
}
func (m *Manager) remoteCommit(ctx context.Context) (string, error) {
environment, cleanup, err := m.gitEnvironment()
if err != nil {
return "", err
}
defer cleanup()
output, err := m.run(ctx, environment, "git", "ls-remote", "--exit-code", "--heads", RequiredRemoteURL, "refs/heads/"+RequiredBranch)
if err != nil {
return "", errors.New("remote branch check failed")
}
fields := strings.Fields(string(output))
if len(fields) != 2 || fields[1] != "refs/heads/"+RequiredBranch || !commitPattern.MatchString(fields[0]) {
return "", errors.New("remote branch response was invalid")
}
return fields[0], nil
}
func (m *Manager) fetch(ctx context.Context) error {
environment, cleanup, err := m.gitEnvironment()
if err != nil {
return err
}
defer cleanup()
_, err = m.run(ctx, environment, "git", "fetch", "--no-tags", "--prune", "origin",
"+refs/heads/"+RequiredBranch+":refs/remotes/origin/"+RequiredBranch)
return err
}
func (m *Manager) gitEnvironment() ([]string, func(), error) {
directory := filepath.Dir(m.config.StatePath)
if err := os.MkdirAll(directory, 0o700); err != nil {
return nil, func() {}, errors.New("cannot prepare Git credential helper")
}
file, err := os.CreateTemp(directory, ".git-askpass-*.sh")
if err != nil {
return nil, func() {}, errors.New("cannot prepare Git credential helper")
}
path := file.Name()
cleanup := func() { _ = os.Remove(path) }
script := "#!/bin/sh\ncase \"$1\" in\n*sername*) printf '%s\\n' 'oauth2' ;;\n*) printf '%s\\n' \"$VIRT_CLOUD_UPDATE_TOKEN\" ;;\nesac\n"
if err := file.Chmod(0o700); err != nil {
file.Close()
cleanup()
return nil, func() {}, errors.New("cannot protect Git credential helper")
}
if _, err := file.WriteString(script); err != nil {
file.Close()
cleanup()
return nil, func() {}, errors.New("cannot write Git credential helper")
}
if err := file.Close(); err != nil {
cleanup()
return nil, func() {}, errors.New("cannot finish Git credential helper")
}
return []string{"GIT_TERMINAL_PROMPT=0", "GIT_ASKPASS=" + path, "VIRT_CLOUD_UPDATE_TOKEN=" + m.state.AccessToken()}, cleanup, nil
}
func (m *Manager) remoteTrackingCommit(ctx context.Context) (string, error) {
return m.gitLine(ctx, "rev-parse", "refs/remotes/origin/"+RequiredBranch)
}
func (m *Manager) ensureFastForward(ctx context.Context, current, remote string) error {
_, err := m.run(ctx, nil, "git", "merge-base", "--is-ancestor", current, remote)
return err
}
func (m *Manager) changedPaths(ctx context.Context, current, remote string) ([]string, error) {
output, err := m.run(ctx, nil, "git", "diff", "--name-only", "-z", current+".."+remote)
if err != nil {
return nil, err
}
parts := strings.Split(string(output), "\x00")
if len(parts) > 0 && parts[len(parts)-1] == "" {
parts = parts[:len(parts)-1]
}
return parts, nil
}
func deniedChangedPath(paths []string) bool {
for _, path := range paths {
clean := filepath.ToSlash(filepath.Clean(path))
if clean == "." || clean == ".gitignore" || clean != path || strings.HasPrefix(clean, "../") || strings.HasPrefix(clean, "deploy/") {
return true
}
base := strings.ToLower(filepath.Base(clean))
if base == "go.mod" || base == "go.sum" || base == "composer.json" || base == "composer.lock" ||
base == "package.json" || base == "package-lock.json" || base == "yarn.lock" || base == "pnpm-lock.yaml" ||
base == "cargo.toml" || base == "cargo.lock" || strings.HasPrefix(base, "requirements") && strings.HasSuffix(base, ".txt") {
return true
}
}
return false
}
func (m *Manager) preflight(ctx context.Context, targetCommit string) error {
if _, err := m.webDirectory(); err != nil {
return err
}
if err := m.targetHasSchemaInitCommand(ctx, targetCommit); err != nil {
return err
}
if _, err := m.run(ctx, nil, m.config.PHPCommand, "--version"); err != nil {
return err
}
for _, unit := range append([]string{m.config.PHPFPMUnit}, m.config.ServiceUnits...) {
output, err := m.run(ctx, nil, "systemctl", "show", "--property=LoadState", "--property=ActiveState", unit)
if err != nil {
return err
}
properties := parseSystemdProperties(output)
if properties["LoadState"] != "loaded" || properties["ActiveState"] != "active" {
return errors.New("required service is not loaded and active")
}
}
return nil
}
func (m *Manager) webDirectory() (string, error) {
repository, err := filepath.EvalSymlinks(m.config.RepositoryDir)
if err != nil {
return "", errors.New("repository is unavailable")
}
webDirectory, err := filepath.EvalSymlinks(filepath.Join(m.config.RepositoryDir, "web"))
if err != nil || !pathWithin(repository, webDirectory) {
return "", errors.New("web directory is unavailable or escapes the repository")
}
xcatPath := filepath.Join(webDirectory, "xcat")
info, err := os.Lstat(xcatPath)
if err != nil || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
return "", errors.New("web/xcat must be a regular file")
}
return webDirectory, nil
}
func (m *Manager) targetHasSchemaInitCommand(ctx context.Context, targetCommit string) error {
output, err := m.run(ctx, nil, "git", "ls-tree", "-z", targetCommit, "--", "web/xcat")
if err != nil {
return err
}
line := strings.TrimSuffix(string(output), "\x00")
metadata, path, ok := strings.Cut(line, "\t")
fields := strings.Fields(metadata)
if !ok || path != "web/xcat" || len(fields) != 3 ||
(fields[0] != "100644" && fields[0] != "100755") || fields[1] != "blob" || !commitPattern.MatchString(fields[2]) {
return errors.New("target does not contain a regular web/xcat command")
}
return nil
}
func parseSystemdProperties(output []byte) map[string]string {
properties := make(map[string]string)
for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") {
key, value, ok := strings.Cut(strings.TrimSpace(line), "=")
if ok && key != "" {
properties[key] = value
}
}
return properties
}
func (m *Manager) stopServices(ctx context.Context) error {
for _, unit := range append(append([]string(nil), m.config.ServiceUnits...), m.config.PHPFPMUnit) {
if _, err := m.run(ctx, nil, "systemctl", "stop", unit); err != nil {
return err
}
}
return nil
}
func (m *Manager) startServices(ctx context.Context) error {
if _, err := m.run(ctx, nil, "systemctl", "start", m.config.PHPFPMUnit); err != nil {
return err
}
if err := m.waitUnitsActive(ctx, []string{m.config.PHPFPMUnit}); err != nil {
return err
}
for _, unit := range m.config.ServiceUnits {
if _, err := m.run(ctx, nil, "systemctl", "start", unit); err != nil {
return err
}
}
return nil
}
func (m *Manager) healthCheck(ctx context.Context) error {
return m.waitUnitsActive(ctx, append([]string{m.config.PHPFPMUnit}, m.config.ServiceUnits...))
}
func (m *Manager) waitUnitsActive(ctx context.Context, units []string) error {
deadline := time.Now().Add(m.healthTimeout)
for {
allActive := true
for _, unit := range units {
if _, err := m.run(ctx, nil, "systemctl", "is-active", "--quiet", unit); err != nil {
allActive = false
break
}
}
if allActive {
return nil
}
if !time.Now().Before(deadline) {
return errors.New("service health deadline exceeded")
}
timer := time.NewTimer(m.healthPollInterval)
select {
case <-ctx.Done():
timer.Stop()
return ctx.Err()
case <-timer.C:
}
}
}
func (m *Manager) gitLine(ctx context.Context, arguments ...string) (string, error) {
output, err := m.run(ctx, nil, "git", arguments...)
if err != nil {
return "", err
}
value := strings.TrimSpace(string(output))
if value == "" || strings.ContainsAny(value, "\x00\r\n") {
return "", errors.New("Git returned a non-canonical value")
}
return value, nil
}
func (m *Manager) run(ctx context.Context, environment []string, command string, arguments ...string) ([]byte, error) {
return m.runAt(ctx, m.config.RepositoryDir, environment, command, arguments...)
}
func (m *Manager) runAt(ctx context.Context, directory string, environment []string, command string, arguments ...string) ([]byte, error) {
output, err := m.executor.Run(ctx, directory, environment, command, arguments...)
if err != nil {
return nil, fmt.Errorf("update command failed")
}
return output, nil
}