From 966048cb5c39145090bb62e537196eecb15d69b1 Mon Sep 17 00:00:00 2001 From: codex Date: Wed, 28 Oct 2026 07:14:25 +0800 Subject: [PATCH] refactor: enforce migration recovery contracts --- .../Cloud/CloudInstanceMigrationService.php | 20 ++- .../Cloud/CloudInstanceMigrationWorkflow.php | 148 +++++++++++++++++- .../CloudInstanceMigrationWorkflowTest.php | 56 +++++++ 3 files changed, 215 insertions(+), 9 deletions(-) diff --git a/web/app/Services/Cloud/CloudInstanceMigrationService.php b/web/app/Services/Cloud/CloudInstanceMigrationService.php index 3ef54d3d..b5fbde42 100644 --- a/web/app/Services/Cloud/CloudInstanceMigrationService.php +++ b/web/app/Services/Cloud/CloudInstanceMigrationService.php @@ -1205,7 +1205,10 @@ final class CloudInstanceMigrationService $restore->updated_at = $now; $restore->save(); self::setState($migration, 'attention_required', - 'peer_transfer_failed', 'retry_transfer'); + 'peer_transfer_failed', + (string) CloudInstanceMigrationWorkflow::failureAction( + 'transfer' + )); return; } if (!is_array($result) @@ -1646,7 +1649,10 @@ final class CloudInstanceMigrationService } if ($status === 'failed') { self::setState($migration, 'attention_required', - 'migration_source_rollback_failed', 'retry_rollback'); + 'migration_source_rollback_failed', + (string) CloudInstanceMigrationWorkflow::failureAction( + 'rollback' + )); return; } if (!is_array($result) @@ -1702,7 +1708,10 @@ final class CloudInstanceMigrationService } if ($status === 'failed') { self::setState($migration, 'attention_required', - 'firewall_apply_failed', 'retry_firewall'); + 'firewall_apply_failed', + (string) CloudInstanceMigrationWorkflow::failureAction( + 'firewall' + )); return; } $policy = CloudInstanceFirewall::where( @@ -1887,7 +1896,10 @@ final class CloudInstanceMigrationService } if ($status === 'failed') { self::setState($migration, 'attention_required', - 'migration_target_activation_failed', 'retry_activation'); + 'migration_target_activation_failed', + (string) CloudInstanceMigrationWorkflow::failureAction( + 'activation' + )); return; } if (!is_array($result) diff --git a/web/app/Services/Cloud/CloudInstanceMigrationWorkflow.php b/web/app/Services/Cloud/CloudInstanceMigrationWorkflow.php index 2185aa8f..23777dc4 100644 --- a/web/app/Services/Cloud/CloudInstanceMigrationWorkflow.php +++ b/web/app/Services/Cloud/CloudInstanceMigrationWorkflow.php @@ -24,6 +24,9 @@ final class CloudInstanceMigrationWorkflow 'task_key' => 'prepare_task_id', 'finish_handler' => 'finishPrepareLocked', 'reconcile_action' => 'reconcile_prepare', + 'failure_policy' => 'conditional_abort', + 'failure_action' => null, + 'required_recovery_actions' => ['abort_prepare'], 'recovery_actions' => [ 'abort_prepare' => [ 'label' => '中止迁移并恢复源实例', @@ -45,6 +48,11 @@ final class CloudInstanceMigrationWorkflow 'task_key' => 'transfer_task_id', 'finish_handler' => 'finishTransferLocked', 'reconcile_action' => 'reconcile_transfer', + 'failure_policy' => 'manual_recovery', + 'failure_action' => 'retry_transfer', + 'required_recovery_actions' => [ + 'retry_transfer', 'abort_transfer', + ], 'recovery_actions' => [ 'retry_transfer' => [ 'label' => '继续分块传输', @@ -71,6 +79,9 @@ final class CloudInstanceMigrationWorkflow 'task_key' => null, 'finish_handler' => null, 'reconcile_action' => null, + 'failure_policy' => 'inline_transaction', + 'failure_action' => null, + 'required_recovery_actions' => [], 'recovery_actions' => [], ], 'firewall' => [ @@ -82,6 +93,9 @@ final class CloudInstanceMigrationWorkflow 'task_key' => 'firewall_task_id', 'finish_handler' => 'finishFirewallLocked', 'reconcile_action' => 'reconcile_firewall', + 'failure_policy' => 'manual_recovery', + 'failure_action' => 'retry_firewall', + 'required_recovery_actions' => ['retry_firewall'], 'recovery_actions' => [ 'retry_firewall' => [ 'label' => '重试目标防火墙', @@ -101,6 +115,9 @@ final class CloudInstanceMigrationWorkflow 'task_key' => 'activation_task_id', 'finish_handler' => 'finishActivationLocked', 'reconcile_action' => 'reconcile_activation', + 'failure_policy' => 'manual_recovery', + 'failure_action' => 'retry_activation', + 'required_recovery_actions' => ['retry_activation'], 'recovery_actions' => [ 'retry_activation' => [ 'label' => '重试目标状态恢复', @@ -120,6 +137,9 @@ final class CloudInstanceMigrationWorkflow 'task_key' => 'cleanup_task_id', 'finish_handler' => 'finishCleanupLocked', 'reconcile_action' => 'reconcile_cleanup', + 'failure_policy' => 'source_retained', + 'failure_action' => null, + 'required_recovery_actions' => [], 'recovery_actions' => [], ], 'rollback' => [ @@ -131,6 +151,9 @@ final class CloudInstanceMigrationWorkflow 'task_key' => 'rollback_task_id', 'finish_handler' => 'finishRollbackLocked', 'reconcile_action' => 'reconcile_rollback', + 'failure_policy' => 'manual_recovery', + 'failure_action' => 'retry_rollback', + 'required_recovery_actions' => ['retry_rollback'], 'recovery_actions' => [ 'retry_rollback' => [ 'label' => '重试恢复源实例', @@ -199,6 +222,18 @@ final class CloudInstanceMigrationWorkflow return $definition === null ? null : (string) $definition['handler']; } + public static function failureAction(string $stageKey): ?string + { + self::assertIntegrity(); + if (!isset(self::STAGES[$stageKey])) { + throw new \LogicException( + 'Unknown maintenance-migration stage: ' . $stageKey + ); + } + $action = self::STAGES[$stageKey]['failure_action']; + return $action === null ? null : (string) $action; + } + /** * @return list */ @@ -399,10 +434,22 @@ final class CloudInstanceMigrationWorkflow if ($checked) { return; } + self::assertDefinitions(self::STAGES); + $checked = true; + } + + /** @param array> $stages */ + public static function assertDefinitions(array $stages): void + { $required = [ 'position', 'label', 'running_state', 'job_kind', 'job_field', 'task_key', 'finish_handler', 'reconcile_action', - 'recovery_actions', + 'failure_policy', 'failure_action', + 'required_recovery_actions', 'recovery_actions', + ]; + $failurePolicies = [ + 'conditional_abort', 'manual_recovery', 'inline_transaction', + 'source_retained', ]; $positions = []; $states = []; @@ -411,11 +458,37 @@ final class CloudInstanceMigrationWorkflow $taskKeys = []; $reconcileActions = []; $actions = []; - foreach (self::STAGES as $key => $stage) { - if (array_diff($required, array_keys($stage)) !== [] + if ($stages === []) { + throw new \LogicException( + 'Maintenance-migration workflow has no stages' + ); + } + foreach ($stages as $key => $stage) { + if (!is_string($key) || $key === '' || !is_array($stage) + || array_diff($required, array_keys($stage)) !== [] || !is_string($stage['label']) || $stage['label'] === '' || !is_string($stage['running_state']) || $stage['running_state'] === '' + || ($stage['position'] !== null + && !is_int($stage['position'])) + || !is_string($stage['failure_policy']) + || !in_array( + $stage['failure_policy'], $failurePolicies, true + ) + || ($stage['failure_action'] !== null + && (!is_string($stage['failure_action']) + || $stage['failure_action'] === '')) + || !is_array($stage['required_recovery_actions']) + || !array_is_list($stage['required_recovery_actions']) + || array_filter( + $stage['required_recovery_actions'], + static fn ($action): bool => + !is_string($action) || $action === '' + ) !== [] + || count($stage['required_recovery_actions']) + !== count(array_unique( + $stage['required_recovery_actions'] + )) || !is_array($stage['recovery_actions'])) { throw new \LogicException( 'Incomplete maintenance-migration stage: ' . $key @@ -490,6 +563,72 @@ final class CloudInstanceMigrationWorkflow } $actions[$action] = true; } + $definedActions = array_keys($stage['recovery_actions']); + $requiredActions = $stage['required_recovery_actions']; + sort($definedActions); + sort($requiredActions); + if ($definedActions !== $requiredActions) { + throw new \LogicException( + 'Migration stage recovery exits do not match its contract: ' + . $key + ); + } + $failureAction = $stage['failure_action']; + if ($stage['failure_policy'] === 'manual_recovery') { + if ($stage['job_kind'] === null || $failureAction === null + || !isset($stage['recovery_actions'][$failureAction])) { + throw new \LogicException( + 'Manual-recovery stage has no failure exit: ' . $key + ); + } + foreach ($stage['required_recovery_actions'] as $action) { + if (!in_array( + $failureAction, + $stage['recovery_actions'][$action]['recorded_actions'], + true + )) { + throw new \LogicException( + 'Recovery exit is unreachable from stage failure: ' + . $key + ); + } + } + } elseif ($stage['failure_policy'] === 'conditional_abort') { + if ($stage['job_kind'] === null || $failureAction !== null + || $stage['required_recovery_actions'] === []) { + throw new \LogicException( + 'Conditional-abort stage has an invalid failure contract: ' + . $key + ); + } + foreach ($stage['required_recovery_actions'] as $action) { + $definition = $stage['recovery_actions'][$action]; + if (!$definition['requires_abort_evidence'] + || !in_array( + $stage['reconcile_action'], + $definition['recorded_actions'], + true + )) { + throw new \LogicException( + 'Conditional abort is not evidence-gated: ' . $key + ); + } + } + } elseif ($stage['failure_policy'] === 'inline_transaction') { + if ($stage['job_kind'] !== null || $failureAction !== null + || $stage['required_recovery_actions'] !== []) { + throw new \LogicException( + 'Inline stage exposes an invalid recovery contract: ' + . $key + ); + } + } elseif ($stage['job_kind'] === null || $failureAction !== null + || $stage['required_recovery_actions'] !== []) { + throw new \LogicException( + 'Source-retained stage exposes an invalid recovery contract: ' + . $key + ); + } } ksort($positions); if (array_keys($positions) !== range(1, count($positions))) { @@ -497,7 +636,7 @@ final class CloudInstanceMigrationWorkflow 'Maintenance-migration stage positions are not contiguous' ); } - foreach (self::STAGES as $stage) { + foreach ($stages as $stage) { foreach ($stage['recovery_actions'] as $definition) { foreach ($definition['recorded_actions'] as $recorded) { if (!isset($actions[$recorded]) @@ -509,7 +648,6 @@ final class CloudInstanceMigrationWorkflow } } } - $checked = true; } /** @return array|null */ diff --git a/web/tests/CloudInstanceMigrationWorkflowTest.php b/web/tests/CloudInstanceMigrationWorkflowTest.php index bf21ed1f..eef7d822 100644 --- a/web/tests/CloudInstanceMigrationWorkflowTest.php +++ b/web/tests/CloudInstanceMigrationWorkflowTest.php @@ -43,6 +43,50 @@ $assert(array_keys($stages) === [ 'prepare', 'transfer', 'cutover', 'firewall', 'activation', 'cleanup', 'rollback', ], 'The executable migration-stage registry is incomplete or reordered.'); +$assert(method_exists( + CloudInstanceMigrationWorkflow::class, + 'assertDefinitions' + ), + 'Migration workflow definitions cannot be mutation-tested.'); + +$expectedFailureContracts = [ + 'prepare' => ['conditional_abort', null, ['abort_prepare']], + 'transfer' => [ + 'manual_recovery', 'retry_transfer', + ['retry_transfer', 'abort_transfer'], + ], + 'cutover' => ['inline_transaction', null, []], + 'firewall' => [ + 'manual_recovery', 'retry_firewall', ['retry_firewall'], + ], + 'activation' => [ + 'manual_recovery', 'retry_activation', ['retry_activation'], + ], + 'cleanup' => ['source_retained', null, []], + 'rollback' => [ + 'manual_recovery', 'retry_rollback', ['retry_rollback'], + ], +]; +foreach ($expectedFailureContracts as $stageKey => $contract) { + [$policy, $failureAction, $requiredActions] = $contract; + $assert(($stages[$stageKey]['failure_policy'] ?? null) === $policy + && ($stages[$stageKey]['failure_action'] ?? null) === $failureAction + && ($stages[$stageKey]['required_recovery_actions'] ?? null) + === $requiredActions, + "Stage {$stageKey} lost its explicit failure contract."); + foreach ($requiredActions as $requiredAction) { + $mutated = $stages; + unset($mutated[$stageKey]['recovery_actions'][$requiredAction]); + $missingExitRejected = false; + try { + CloudInstanceMigrationWorkflow::assertDefinitions($mutated); + } catch (LogicException $error) { + $missingExitRejected = true; + } + $assert($missingExitRejected, + "Stage {$stageKey} accepted a missing recovery exit."); + } +} $service = new ReflectionClass(CloudInstanceMigrationService::class); $jobKinds = []; @@ -80,6 +124,18 @@ $assert(CloudInstanceMigrationWorkflow::recoveryActions( false ) === ['retry_firewall'], 'Firewall recovery policy is incomplete.'); +$activationFailure = $migration('attention_required', 'retry_activation'); +$assert(CloudInstanceMigrationWorkflow::recoveryActions( + $activationFailure, + false + ) === ['retry_activation'], + 'Activation recovery policy is incomplete.'); +$rollbackFailure = $migration('attention_required', 'retry_rollback'); +$assert(CloudInstanceMigrationWorkflow::recoveryActions( + $rollbackFailure, + false + ) === ['retry_rollback'], + 'Rollback recovery policy is incomplete.'); $prepareUnknown = $migration('attention_required', 'reconcile_prepare'); $assert(CloudInstanceMigrationWorkflow::recoveryActions( $prepareUnknown,