refactor: enforce migration recovery contracts

go
codex 3 weeks ago
parent cc43455191
commit 966048cb5c

@ -1205,7 +1205,10 @@ final class CloudInstanceMigrationService
$restore->updated_at = $now;
$restore->save();
self::setState($migration, 'attention_required',
'peer_transfer_failed', 'retry_transfer');
'peer_transfer_failed',
(string) CloudInstanceMigrationWorkflow::failureAction(
'transfer'
));
return;
}
if (!is_array($result)
@ -1646,7 +1649,10 @@ final class CloudInstanceMigrationService
}
if ($status === 'failed') {
self::setState($migration, 'attention_required',
'migration_source_rollback_failed', 'retry_rollback');
'migration_source_rollback_failed',
(string) CloudInstanceMigrationWorkflow::failureAction(
'rollback'
));
return;
}
if (!is_array($result)
@ -1702,7 +1708,10 @@ final class CloudInstanceMigrationService
}
if ($status === 'failed') {
self::setState($migration, 'attention_required',
'firewall_apply_failed', 'retry_firewall');
'firewall_apply_failed',
(string) CloudInstanceMigrationWorkflow::failureAction(
'firewall'
));
return;
}
$policy = CloudInstanceFirewall::where(
@ -1887,7 +1896,10 @@ final class CloudInstanceMigrationService
}
if ($status === 'failed') {
self::setState($migration, 'attention_required',
'migration_target_activation_failed', 'retry_activation');
'migration_target_activation_failed',
(string) CloudInstanceMigrationWorkflow::failureAction(
'activation'
));
return;
}
if (!is_array($result)

@ -24,6 +24,9 @@ final class CloudInstanceMigrationWorkflow
'task_key' => 'prepare_task_id',
'finish_handler' => 'finishPrepareLocked',
'reconcile_action' => 'reconcile_prepare',
'failure_policy' => 'conditional_abort',
'failure_action' => null,
'required_recovery_actions' => ['abort_prepare'],
'recovery_actions' => [
'abort_prepare' => [
'label' => '中止迁移并恢复源实例',
@ -45,6 +48,11 @@ final class CloudInstanceMigrationWorkflow
'task_key' => 'transfer_task_id',
'finish_handler' => 'finishTransferLocked',
'reconcile_action' => 'reconcile_transfer',
'failure_policy' => 'manual_recovery',
'failure_action' => 'retry_transfer',
'required_recovery_actions' => [
'retry_transfer', 'abort_transfer',
],
'recovery_actions' => [
'retry_transfer' => [
'label' => '继续分块传输',
@ -71,6 +79,9 @@ final class CloudInstanceMigrationWorkflow
'task_key' => null,
'finish_handler' => null,
'reconcile_action' => null,
'failure_policy' => 'inline_transaction',
'failure_action' => null,
'required_recovery_actions' => [],
'recovery_actions' => [],
],
'firewall' => [
@ -82,6 +93,9 @@ final class CloudInstanceMigrationWorkflow
'task_key' => 'firewall_task_id',
'finish_handler' => 'finishFirewallLocked',
'reconcile_action' => 'reconcile_firewall',
'failure_policy' => 'manual_recovery',
'failure_action' => 'retry_firewall',
'required_recovery_actions' => ['retry_firewall'],
'recovery_actions' => [
'retry_firewall' => [
'label' => '重试目标防火墙',
@ -101,6 +115,9 @@ final class CloudInstanceMigrationWorkflow
'task_key' => 'activation_task_id',
'finish_handler' => 'finishActivationLocked',
'reconcile_action' => 'reconcile_activation',
'failure_policy' => 'manual_recovery',
'failure_action' => 'retry_activation',
'required_recovery_actions' => ['retry_activation'],
'recovery_actions' => [
'retry_activation' => [
'label' => '重试目标状态恢复',
@ -120,6 +137,9 @@ final class CloudInstanceMigrationWorkflow
'task_key' => 'cleanup_task_id',
'finish_handler' => 'finishCleanupLocked',
'reconcile_action' => 'reconcile_cleanup',
'failure_policy' => 'source_retained',
'failure_action' => null,
'required_recovery_actions' => [],
'recovery_actions' => [],
],
'rollback' => [
@ -131,6 +151,9 @@ final class CloudInstanceMigrationWorkflow
'task_key' => 'rollback_task_id',
'finish_handler' => 'finishRollbackLocked',
'reconcile_action' => 'reconcile_rollback',
'failure_policy' => 'manual_recovery',
'failure_action' => 'retry_rollback',
'required_recovery_actions' => ['retry_rollback'],
'recovery_actions' => [
'retry_rollback' => [
'label' => '重试恢复源实例',
@ -199,6 +222,18 @@ final class CloudInstanceMigrationWorkflow
return $definition === null ? null : (string) $definition['handler'];
}
public static function failureAction(string $stageKey): ?string
{
self::assertIntegrity();
if (!isset(self::STAGES[$stageKey])) {
throw new \LogicException(
'Unknown maintenance-migration stage: ' . $stageKey
);
}
$action = self::STAGES[$stageKey]['failure_action'];
return $action === null ? null : (string) $action;
}
/**
* @return list<array{action:string,label:string,destructive:bool}>
*/
@ -399,10 +434,22 @@ final class CloudInstanceMigrationWorkflow
if ($checked) {
return;
}
self::assertDefinitions(self::STAGES);
$checked = true;
}
/** @param array<string,array<string,mixed>> $stages */
public static function assertDefinitions(array $stages): void
{
$required = [
'position', 'label', 'running_state', 'job_kind', 'job_field',
'task_key', 'finish_handler', 'reconcile_action',
'recovery_actions',
'failure_policy', 'failure_action',
'required_recovery_actions', 'recovery_actions',
];
$failurePolicies = [
'conditional_abort', 'manual_recovery', 'inline_transaction',
'source_retained',
];
$positions = [];
$states = [];
@ -411,11 +458,37 @@ final class CloudInstanceMigrationWorkflow
$taskKeys = [];
$reconcileActions = [];
$actions = [];
foreach (self::STAGES as $key => $stage) {
if (array_diff($required, array_keys($stage)) !== []
if ($stages === []) {
throw new \LogicException(
'Maintenance-migration workflow has no stages'
);
}
foreach ($stages as $key => $stage) {
if (!is_string($key) || $key === '' || !is_array($stage)
|| array_diff($required, array_keys($stage)) !== []
|| !is_string($stage['label']) || $stage['label'] === ''
|| !is_string($stage['running_state'])
|| $stage['running_state'] === ''
|| ($stage['position'] !== null
&& !is_int($stage['position']))
|| !is_string($stage['failure_policy'])
|| !in_array(
$stage['failure_policy'], $failurePolicies, true
)
|| ($stage['failure_action'] !== null
&& (!is_string($stage['failure_action'])
|| $stage['failure_action'] === ''))
|| !is_array($stage['required_recovery_actions'])
|| !array_is_list($stage['required_recovery_actions'])
|| array_filter(
$stage['required_recovery_actions'],
static fn ($action): bool =>
!is_string($action) || $action === ''
) !== []
|| count($stage['required_recovery_actions'])
!== count(array_unique(
$stage['required_recovery_actions']
))
|| !is_array($stage['recovery_actions'])) {
throw new \LogicException(
'Incomplete maintenance-migration stage: ' . $key
@ -490,6 +563,72 @@ final class CloudInstanceMigrationWorkflow
}
$actions[$action] = true;
}
$definedActions = array_keys($stage['recovery_actions']);
$requiredActions = $stage['required_recovery_actions'];
sort($definedActions);
sort($requiredActions);
if ($definedActions !== $requiredActions) {
throw new \LogicException(
'Migration stage recovery exits do not match its contract: '
. $key
);
}
$failureAction = $stage['failure_action'];
if ($stage['failure_policy'] === 'manual_recovery') {
if ($stage['job_kind'] === null || $failureAction === null
|| !isset($stage['recovery_actions'][$failureAction])) {
throw new \LogicException(
'Manual-recovery stage has no failure exit: ' . $key
);
}
foreach ($stage['required_recovery_actions'] as $action) {
if (!in_array(
$failureAction,
$stage['recovery_actions'][$action]['recorded_actions'],
true
)) {
throw new \LogicException(
'Recovery exit is unreachable from stage failure: '
. $key
);
}
}
} elseif ($stage['failure_policy'] === 'conditional_abort') {
if ($stage['job_kind'] === null || $failureAction !== null
|| $stage['required_recovery_actions'] === []) {
throw new \LogicException(
'Conditional-abort stage has an invalid failure contract: '
. $key
);
}
foreach ($stage['required_recovery_actions'] as $action) {
$definition = $stage['recovery_actions'][$action];
if (!$definition['requires_abort_evidence']
|| !in_array(
$stage['reconcile_action'],
$definition['recorded_actions'],
true
)) {
throw new \LogicException(
'Conditional abort is not evidence-gated: ' . $key
);
}
}
} elseif ($stage['failure_policy'] === 'inline_transaction') {
if ($stage['job_kind'] !== null || $failureAction !== null
|| $stage['required_recovery_actions'] !== []) {
throw new \LogicException(
'Inline stage exposes an invalid recovery contract: '
. $key
);
}
} elseif ($stage['job_kind'] === null || $failureAction !== null
|| $stage['required_recovery_actions'] !== []) {
throw new \LogicException(
'Source-retained stage exposes an invalid recovery contract: '
. $key
);
}
}
ksort($positions);
if (array_keys($positions) !== range(1, count($positions))) {
@ -497,7 +636,7 @@ final class CloudInstanceMigrationWorkflow
'Maintenance-migration stage positions are not contiguous'
);
}
foreach (self::STAGES as $stage) {
foreach ($stages as $stage) {
foreach ($stage['recovery_actions'] as $definition) {
foreach ($definition['recorded_actions'] as $recorded) {
if (!isset($actions[$recorded])
@ -509,7 +648,6 @@ final class CloudInstanceMigrationWorkflow
}
}
}
$checked = true;
}
/** @return array<string,mixed>|null */

@ -43,6 +43,50 @@ $assert(array_keys($stages) === [
'prepare', 'transfer', 'cutover', 'firewall', 'activation', 'cleanup',
'rollback',
], 'The executable migration-stage registry is incomplete or reordered.');
$assert(method_exists(
CloudInstanceMigrationWorkflow::class,
'assertDefinitions'
),
'Migration workflow definitions cannot be mutation-tested.');
$expectedFailureContracts = [
'prepare' => ['conditional_abort', null, ['abort_prepare']],
'transfer' => [
'manual_recovery', 'retry_transfer',
['retry_transfer', 'abort_transfer'],
],
'cutover' => ['inline_transaction', null, []],
'firewall' => [
'manual_recovery', 'retry_firewall', ['retry_firewall'],
],
'activation' => [
'manual_recovery', 'retry_activation', ['retry_activation'],
],
'cleanup' => ['source_retained', null, []],
'rollback' => [
'manual_recovery', 'retry_rollback', ['retry_rollback'],
],
];
foreach ($expectedFailureContracts as $stageKey => $contract) {
[$policy, $failureAction, $requiredActions] = $contract;
$assert(($stages[$stageKey]['failure_policy'] ?? null) === $policy
&& ($stages[$stageKey]['failure_action'] ?? null) === $failureAction
&& ($stages[$stageKey]['required_recovery_actions'] ?? null)
=== $requiredActions,
"Stage {$stageKey} lost its explicit failure contract.");
foreach ($requiredActions as $requiredAction) {
$mutated = $stages;
unset($mutated[$stageKey]['recovery_actions'][$requiredAction]);
$missingExitRejected = false;
try {
CloudInstanceMigrationWorkflow::assertDefinitions($mutated);
} catch (LogicException $error) {
$missingExitRejected = true;
}
$assert($missingExitRejected,
"Stage {$stageKey} accepted a missing recovery exit.");
}
}
$service = new ReflectionClass(CloudInstanceMigrationService::class);
$jobKinds = [];
@ -80,6 +124,18 @@ $assert(CloudInstanceMigrationWorkflow::recoveryActions(
false
) === ['retry_firewall'],
'Firewall recovery policy is incomplete.');
$activationFailure = $migration('attention_required', 'retry_activation');
$assert(CloudInstanceMigrationWorkflow::recoveryActions(
$activationFailure,
false
) === ['retry_activation'],
'Activation recovery policy is incomplete.');
$rollbackFailure = $migration('attention_required', 'retry_rollback');
$assert(CloudInstanceMigrationWorkflow::recoveryActions(
$rollbackFailure,
false
) === ['retry_rollback'],
'Rollback recovery policy is incomplete.');
$prepareUnknown = $migration('attention_required', 'reconcile_prepare');
$assert(CloudInstanceMigrationWorkflow::recoveryActions(
$prepareUnknown,

Loading…
Cancel
Save