From d166b56aa4dfba20663e8d26a6e13b59a5a88552 Mon Sep 17 00:00:00 2001 From: codex Date: Fri, 30 Oct 2026 19:13:50 +0800 Subject: [PATCH] feat(template): coordinate replica lifecycle --- agent/hypervisor/libvirt_catalog_features.go | 9 - ...CloudAutomaticOperationRecoveryService.php | 1 + .../Cloud/CloudOperationRecoveryIdentity.php | 12 +- .../Cloud/CloudTemplateArtifactManager.php | 5 +- .../Cloud/CloudTemplateCatalogService.php | 293 ++++++++++++------ .../Cloud/CloudTemplateReplicaCoordinator.php | 286 +++++++++++++++++ .../Cloud/CloudTemplateStateMachine.php | 164 ++++++++++ .../CloudAggregateRecoveryConvergenceTest.php | 6 +- ...TemplateReplicaCoordinatorContractTest.php | 57 ++++ .../CloudTemplateReplicaStateMachineTest.php | 77 +++++ ...plateStandaloneReplicaFinalizationTest.php | 199 ++++++++++++ web/tests/VirtualizationAssetsTest.php | 7 +- 12 files changed, 1007 insertions(+), 109 deletions(-) create mode 100644 web/app/Services/Cloud/CloudTemplateReplicaCoordinator.php create mode 100644 web/app/Services/Cloud/CloudTemplateStateMachine.php create mode 100644 web/tests/CloudTemplateReplicaCoordinatorContractTest.php create mode 100644 web/tests/CloudTemplateReplicaStateMachineTest.php create mode 100644 web/tests/CloudTemplateStandaloneReplicaFinalizationTest.php diff --git a/agent/hypervisor/libvirt_catalog_features.go b/agent/hypervisor/libvirt_catalog_features.go index dec56ac5..adecb875 100644 --- a/agent/hypervisor/libvirt_catalog_features.go +++ b/agent/hypervisor/libvirt_catalog_features.go @@ -121,15 +121,6 @@ func (l *Libvirt) ImportTemplate(ctx context.Context, request ImportTemplateRequ if _, err := validateRemoteSource(request.SourceURL); err != nil { return ImageTemplate{}, err } - templates, err := l.ListTemplates(ctx) - if err != nil { - return ImageTemplate{}, err - } - for _, item := range templates { - if strings.EqualFold(item.Name, request.Name) { - return ImageTemplate{}, NewError(ErrConflict, "template_name_exists", "a template with this name already exists on this host", nil) - } - } existing, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleTemplate), templateVolumeName(request.ID)) if err != nil { return ImageTemplate{}, err diff --git a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php index feb8c0e0..11cc4d4c 100644 --- a/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php +++ b/web/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php @@ -654,6 +654,7 @@ final class CloudAutomaticOperationRecoveryService } if (!$envelopeConflict && (string) $job->phase === 'terminal' + && (string) $job->execution_mode === 'aggregate' && (CloudTemplate::where( 'active_job_id', (int) $job->id diff --git a/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php b/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php index 590cb04c..982624c6 100644 --- a/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php +++ b/web/app/Services/Cloud/CloudOperationRecoveryIdentity.php @@ -1862,11 +1862,17 @@ final class CloudOperationRecoveryIdentity 'template_metadata_replica' => 'template_metadata_sync', 'template_delete_replica' => 'template_delete', ][(string) $job->kind]; + $standaloneImport = (string) $job->kind + === 'template_import_replica' + && $job->parent_job_id === null + && ($payload['replica_sync'] ?? null) === true; if (!is_string($id) || preg_match(self::TEMPLATE_ID_PATTERN, $id) !== 1 - || $template === null || $replica === null || $parent === null - || (string) $parent->kind !== $expectedParent - || (int) $template->active_job_id !== (int) $parent->id + || $template === null || $replica === null + || (!$standaloneImport && ($parent === null + || (string) $parent->kind !== $expectedParent)) + || (int) $template->active_job_id !== ($standaloneImport + ? (int) $job->id : (int) $parent->id) || (int) $replica->job_id !== (int) $job->id || !hash_equals((string) $replica->operation_id, (string) $job->operation_id)) { diff --git a/web/app/Services/Cloud/CloudTemplateArtifactManager.php b/web/app/Services/Cloud/CloudTemplateArtifactManager.php index 08835c79..0f7e7d96 100644 --- a/web/app/Services/Cloud/CloudTemplateArtifactManager.php +++ b/web/app/Services/Cloud/CloudTemplateArtifactManager.php @@ -186,7 +186,10 @@ final class CloudTemplateArtifactManager $artifact->error_code = substr($code, 0, 64); $artifact->updated_at = $now; $artifact->save(); - $template->state = $unknown ? 'source_unknown' : 'source_failed'; + CloudTemplateStateMachine::moveTemplate( + $template, + $unknown ? 'source_unknown' : 'source_failed' + ); $template->updated_at = $now; $template->save(); }, diff --git a/web/app/Services/Cloud/CloudTemplateCatalogService.php b/web/app/Services/Cloud/CloudTemplateCatalogService.php index 1b2a0015..0c297950 100644 --- a/web/app/Services/Cloud/CloudTemplateCatalogService.php +++ b/web/app/Services/Cloud/CloudTemplateCatalogService.php @@ -181,7 +181,7 @@ final class CloudTemplateCatalogService return $parent; } $now = time(); - $locked->state = 'importing'; + CloudTemplateStateMachine::moveTemplate($locked, 'importing'); $locked->active_job_id = (int) $parent->id; $locked->updated_at = $now; $locked->save(); @@ -215,7 +215,11 @@ final class CloudTemplateCatalogService $replica->revision = 1; $replica->generation_token = (string) $locked->generation_token; - $replica->state = 'importing'; + $replica->state = 'pending'; + CloudTemplateStateMachine::moveReplica( + $replica, + 'importing' + ); $replica->image_bytes = 0; $replica->sha256 = (string) $locked->sha256; $replica->operation_id = $operationId; @@ -290,15 +294,12 @@ final class CloudTemplateCatalogService 'template_id' => $id, ], (int) $parent->id, $operationId ); - CloudTemplateReplica::query() - ->where('host_id', (int) $replica->host_id) - ->where('template_id', $id)->update([ - 'state' => 'syncing', - 'operation_id' => $operationId, - 'job_id' => (int) $child->id, - 'error_code' => '', - 'updated_at' => time(), - ]); + CloudTemplateStateMachine::moveReplica($replica, 'syncing'); + $replica->operation_id = $operationId; + $replica->job_id = (int) $child->id; + $replica->error_code = ''; + $replica->updated_at = time(); + $replica->save(); } return $parent; }, 3); @@ -327,7 +328,7 @@ final class CloudTemplateCatalogService $template ); $partialCleanup = ($partialCleanupKind === 'template_import' - && (string) $template->state === 'error') + && (string) $template->state === 'degraded') || ($partialCleanupKind === 'template_delete' && (string) $template->state === 'delete_failed'); if ((string) $template->state !== 'active' && !$partialCleanup) { @@ -382,9 +383,11 @@ final class CloudTemplateCatalogService ->where('template_id', $id) ->where('state', 'delete_failed') ->lockForUpdate()->get() as $survivor) { - $survivor->state = (int) $survivor->revision - === (int) $template->revision - ? 'active' : 'stale'; + CloudTemplateStateMachine::moveReplica( + $survivor, + (int) $survivor->revision === (int) $template->revision + ? 'active' : 'stale' + ); $survivor->error_code = ''; $survivor->updated_at = time(); $survivor->save(); @@ -404,7 +407,7 @@ final class CloudTemplateCatalogService throw new CloudError('云镜像部分结果账本不完整,禁止删除', 409); } } - $template->state = 'deleting'; + CloudTemplateStateMachine::moveTemplate($template, 'deleting'); $template->active_job_id = (int) $parent->id; $template->updated_at = time(); $template->save(); @@ -418,15 +421,12 @@ final class CloudTemplateCatalogService 'template_id' => $id, ], (int) $parent->id, $operationId ); - CloudTemplateReplica::query() - ->where('host_id', (int) $replica->host_id) - ->where('template_id', $id)->update([ - 'state' => 'deleting', - 'operation_id' => $operationId, - 'job_id' => (int) $child->id, - 'error_code' => '', - 'updated_at' => time(), - ]); + CloudTemplateStateMachine::moveReplica($replica, 'deleting'); + $replica->operation_id = $operationId; + $replica->job_id = (int) $child->id; + $replica->error_code = ''; + $replica->updated_at = time(); + $replica->save(); } return $parent; }, 3); @@ -454,13 +454,21 @@ final class CloudTemplateCatalogService if ($template === null || $replica === null) { throw new CloudError('云镜像任务账本不存在', 409); } - // Only project this replica here. CloudWorkerService persists the - // child's terminal effect first, then reconciles the aggregate parent - // through settleRecoveredParentLocked(), where the complete child / - // replica topology and content can be verified atomically. + // Project the leaf first. Aggregate children are published only by the + // parent reconciler; standalone repair jobs publish availability here + // only after a definitive result and keep unknown effects fenced. if ((string) $job->kind === 'template_import_replica') { self::finishImportReplica($template, $replica, $status, $result, $operation, $recovery); + if ($job->parent_job_id === null) { + CloudTemplateReplicaCoordinator::settleStandaloneLocked( + $template, + $replica, + $job, + $status, + $operation + ); + } } elseif ((string) $job->kind === 'template_metadata_replica') { self::finishMetadataReplica($template, $replica, $status, $result, $operation, $recovery); @@ -634,7 +642,7 @@ final class CloudTemplateCatalogService if ($resolution === 'mixed' && $kind !== 'template_metadata_sync') { $expectedState = $kind === 'template_import' - ? 'error' : 'delete_failed'; + ? 'degraded' : 'delete_failed'; if ((string) $template->state !== $expectedState) { throw new CloudRecoveryInstanceStateConflict( '云镜像汇总任务的部分成功目录状态不一致', @@ -848,7 +856,7 @@ final class CloudTemplateCatalogService 'template_id', (string) $template->id )->delete(); - $template->state = 'retired'; + CloudTemplateStateMachine::moveTemplate($template, 'retired'); $template->updated_at = time(); $template->save(); CloudJobService::completeAggregateLocked( @@ -866,13 +874,16 @@ final class CloudTemplateCatalogService (string) $template->id )->orderBy('host_id')->lockForUpdate()->get(); foreach ($replicas as $replica) { - $replica->state = (int) $replica->revision - === (int) $template->revision ? 'active' : 'stale'; + CloudTemplateStateMachine::moveReplica( + $replica, + (int) $replica->revision === (int) $template->revision + ? 'active' : 'stale' + ); $replica->error_code = ''; $replica->updated_at = time(); $replica->save(); } - $template->state = 'active'; + CloudTemplateStateMachine::moveTemplate($template, 'active'); $template->updated_at = time(); $template->save(); CloudJobService::completeAggregateLocked( @@ -884,6 +895,10 @@ final class CloudTemplateCatalogService ); return; } + CloudTemplateStateMachine::assertReplicaTransition( + 'syncing', + 'stale' + ); CloudTemplateReplica::query()->where( 'template_id', (string) $template->id @@ -891,7 +906,7 @@ final class CloudTemplateCatalogService 'state' => 'stale', 'updated_at' => time(), ]); - $template->state = 'active'; + CloudTemplateStateMachine::moveTemplate($template, 'active'); $template->updated_at = time(); $template->save(); CloudJobService::completeAggregateLocked( @@ -940,13 +955,13 @@ final class CloudTemplateCatalogService $template->updated_at = time(); $template->save(); } - $replica->state = 'active'; + CloudTemplateStateMachine::moveReplica($replica, 'active'); $replica->revision = (int) $template->revision; $replica->image_bytes = (int) $bytes; $replica->sha256 = $digest; $replica->error_code = ''; } else { - $replica->state = 'failed'; + CloudTemplateStateMachine::moveReplica($replica, 'failed'); $replica->error_code = self::errorCode( $operation, 'template_import_failed' ); @@ -984,16 +999,34 @@ final class CloudTemplateCatalogService } $success = !$replicas->contains(static fn ($row): bool => (string) $row->state !== 'active'); - if ($success) { + $hostIds = $replicas->map( + static fn (CloudTemplateReplica $replica): int => + (int) $replica->host_id + )->all(); + $availability = CloudTemplateStateMachine::availability( + $template, + $replicas, + $hostIds + ); + if ($availability !== 'error') { CloudTemplate::query() ->where('catalog_key', (string) $template->catalog_key) ->where('id', '<>', (string) $template->id) ->where('is_default', 1) ->update(['is_default' => 0, 'updated_at' => time()]); - $template->state = 'active'; + CloudTemplateStateMachine::moveTemplate( + $template, + $availability + ); $template->is_default = 1; $template->updated_at = time(); $template->save(); + } else { + CloudTemplateStateMachine::moveTemplate($template, 'error'); + $template->updated_at = time(); + $template->save(); + } + if ($success) { CloudJobService::completeAggregateLocked($parent, 'succeeded', true, false, '', ['template_id' => (string) $template->id]); return; @@ -1003,9 +1036,6 @@ final class CloudTemplateCatalogService || ($operation['effect_may_have_applied'] ?? false) === true || CloudJob::where('parent_job_id', $parentId) ->where('effect_may_have_applied', 1)->exists(); - $template->state = 'error'; - $template->updated_at = time(); - $template->save(); CloudJobService::completeAggregateLocked($parent, 'failed', $effect, false, 'template_import_failed'); } @@ -1030,11 +1060,11 @@ final class CloudTemplateCatalogService !== (string) $template->generation_token) { throw new CloudError('Agent 元数据同步回执与请求不匹配', 502); } - $replica->state = 'active'; + CloudTemplateStateMachine::moveReplica($replica, 'active'); $replica->revision = (int) $template->revision; $replica->error_code = ''; } else { - $replica->state = 'stale'; + CloudTemplateStateMachine::moveReplica($replica, 'stale'); $replica->error_code = self::errorCode( $operation, 'template_metadata_sync_failed' ); @@ -1104,10 +1134,13 @@ final class CloudTemplateCatalogService )) { throw new CloudError('Agent 云镜像删除回执缺少清理证据', 502); } - $replica->state = 'delete_succeeded'; + CloudTemplateStateMachine::moveReplica( + $replica, + 'delete_succeeded' + ); $replica->error_code = ''; } else { - $replica->state = 'delete_failed'; + CloudTemplateStateMachine::moveReplica($replica, 'delete_failed'); $replica->error_code = self::errorCode( $operation, 'template_delete_failed' ); @@ -1146,7 +1179,7 @@ final class CloudTemplateCatalogService CloudTemplateReplica::query()->where( 'template_id', (string) $template->id )->delete(); - $template->state = 'retired'; + CloudTemplateStateMachine::moveTemplate($template, 'retired'); $template->revision = (int) $template->revision + 1; $template->updated_at = time(); $template->save(); @@ -1162,7 +1195,7 @@ final class CloudTemplateCatalogService || ($operation['effect_may_have_applied'] ?? false) === true || CloudJob::where('parent_job_id', $parentId) ->where('effect_may_have_applied', 1)->exists(); - $template->state = 'delete_failed'; + CloudTemplateStateMachine::moveTemplate($template, 'delete_failed'); $template->updated_at = time(); $template->save(); CloudJobService::completeAggregateLocked($parent, 'failed', $effect, @@ -1199,6 +1232,7 @@ final class CloudTemplateCatalogService 'template_metadata_sync' => 'stale', default => 'delete_failed', }; + CloudTemplateStateMachine::assertReplicaTransition($from, $to); CloudTemplateReplica::query() ->where('template_id', (string) $template->id) ->where('state', $from)->update([ @@ -1207,9 +1241,24 @@ final class CloudTemplateCatalogService 'updated_at' => time(), ]); if ((string) $parent->kind === 'template_import') { - $template->state = 'error'; + $replicas = self::lockedReplicas((string) $template->id); + $availability = CloudTemplateStateMachine::availability( + $template, + $replicas, + $replicas->map( + static fn (CloudTemplateReplica $replica): int => + (int) $replica->host_id + )->all() + ); + CloudTemplateStateMachine::moveTemplate( + $template, + $availability + ); } elseif ((string) $parent->kind === 'template_delete') { - $template->state = 'delete_failed'; + CloudTemplateStateMachine::moveTemplate( + $template, + 'delete_failed' + ); } $template->updated_at = time(); $template->save(); @@ -1371,6 +1420,17 @@ final class CloudTemplateCatalogService } } + /** + * Acquire the canonical mutation boundary for one host-replica sync. + * The coordinator still owns replica and job validation; exposing this + * narrow lock keeps the parent -> template lock order in one module. + */ + public static function lockForReplicaSync( + string $id + ): ?CloudTemplate { + return self::lockTemplateForMutation(self::templateId($id)); + } + /** * Discover the old parent without a row lock, then acquire the canonical * parent -> catalog order and reject a pointer that changed in between. @@ -1541,7 +1601,7 @@ final class CloudTemplateCatalogService ? 'applied' : $actualResolution; $projectionStateMatches = match ((string) $parent->kind) { 'template_import' => (string) $template->state - === ($actualResolution === 'mixed' ? 'error' : 'active'), + === ($actualResolution === 'mixed' ? 'degraded' : 'active'), 'template_metadata_sync' => (string) $template->state === 'active', 'template_delete' => (string) $template->state === ($actualResolution === 'mixed' @@ -1666,10 +1726,9 @@ final class CloudTemplateCatalogService } /** - * Treat a NULL/zero pointer as safe only when the current immutable - * replica/child chain proves that its one historical parent is fully - * settled. Any in-flight, incomplete, cross-wired or ambiguous chain - * remains fenced for the parent-first recovery sweep. + * Treat a NULL/zero pointer as safe only when every current replica owns a + * settled, identity-matched job. A catalog may legitimately contain a mix + * of historical aggregate children and newer standalone repair jobs. */ private static function missingPointerRecoveryPending( CloudTemplate $template @@ -1704,7 +1763,10 @@ final class CloudTemplateCatalogService if (CloudUserIsoReplica::query()->whereIn( 'job_id', $jobIds - )->exists()) { + )->exists() || CloudTemplateReplica::query()->whereIn( + 'job_id', + $jobIds + )->where('template_id', '<>', $templateId)->exists()) { return true; } $children = CloudJob::whereIn('id', $jobIds)->orderBy('id')->get(); @@ -1712,6 +1774,7 @@ final class CloudTemplateCatalogService return true; } $parentIds = []; + $hasStandalone = false; foreach ($children as $child) { $matches = $replicas->filter( static fn (CloudTemplateReplica $replica): bool => @@ -1720,7 +1783,6 @@ final class CloudTemplateCatalogService $replica = $matches->count() === 1 ? $matches->first() : null; $parentId = (int) ($child->parent_job_id ?? 0); if ($replica === null - || $parentId < 1 || (string) $child->execution_mode !== 'agent' || (int) $child->host_id < 1 || (int) $replica->host_id !== (int) $child->host_id @@ -1738,43 +1800,90 @@ final class CloudTemplateCatalogService )) { return true; } + if ($parentId < 1) { + $hasStandalone = true; + if (!self::standaloneReplicaJobSafelySettled( + $template, + $replica, + $child + )) { + return true; + } + continue; + } $parentIds[$parentId] = true; } - if (count($parentIds) !== 1) { - return true; + foreach (array_keys($parentIds) as $parentId) { + $parent = CloudJob::find((int) $parentId); + $expectedChildKind = match ((string) ($parent?->kind ?? '')) { + 'template_import' => 'template_import_replica', + 'template_metadata_sync' => 'template_metadata_replica', + 'template_delete' => 'template_delete_replica', + default => null, + }; + if ($parent === null + || $expectedChildKind === null + || (string) $parent->execution_mode !== 'aggregate' + || $parent->parent_job_id !== null + || !self::aggregateParentSafelySettled($parent, false) + || CloudTemplate::query() + ->where('active_job_id', (int) $parent->id) + ->where('id', '<>', $templateId)->exists()) { + return true; + } + foreach ($children->where('parent_job_id', (int) $parentId) + as $child) { + if ((string) $child->kind !== $expectedChildKind + || (int) $child->user_id !== (int) $parent->user_id) { + return true; + } + } } - $parent = CloudJob::find((int) array_key_first($parentIds)); - $expectedChildKind = match ((string) ($parent?->kind ?? '')) { - 'template_import' => 'template_import_replica', - 'template_metadata_sync' => 'template_metadata_replica', - 'template_delete' => 'template_delete_replica', - default => null, - }; - if ($parent === null - || $expectedChildKind === null - || (string) $parent->execution_mode !== 'aggregate' - || $parent->parent_job_id !== null - || CloudJob::where('parent_job_id', (int) $parent->id)->count() - !== $children->count() - || CloudTemplateReplica::query()->whereIn('job_id', $jobIds) - ->where('template_id', '<>', $templateId)->exists() - || CloudUserIsoReplica::query()->whereIn( - 'job_id', - $jobIds - )->exists() - || CloudTemplate::query() - ->where('active_job_id', (int) $parent->id) - ->where('id', '<>', $templateId)->exists() - || $children->contains(static fn (CloudJob $child): bool => - (string) $child->kind !== $expectedChildKind - || (int) $child->user_id !== (int) $parent->user_id)) { - return true; + if (!$hasStandalone && count($parentIds) === 1) { + $parent = CloudJob::find((int) array_key_first($parentIds)); + $allChildrenBelongToCurrentReplicas = $parent !== null + && CloudJob::where( + 'parent_job_id', + (int) $parent->id + )->count() === $children->count(); + if ($allChildrenBelongToCurrentReplicas + && !self::currentParentProjectionMatches( + $template, + $parent, + false + )) { + return true; + } } - return !self::aggregateParentSafelySettled($parent, false) - || !self::currentParentProjectionMatches( - $template, - $parent, - false + return false; + } + + private static function standaloneReplicaJobSafelySettled( + CloudTemplate $template, + CloudTemplateReplica $replica, + CloudJob $job + ): bool { + if ($job->parent_job_id !== null + || (string) $job->kind !== 'template_import_replica' + || (string) $job->phase !== 'terminal' + || !in_array((string) $job->effect_resolution, [ + 'applied', 'not_applied', + ], true) + || (bool) $job->finalization_incident + || CloudJobService::hasAgentRecoveryEnvelopeConflict($job)) { + return false; + } + if ((string) $job->effect_resolution === 'not_applied') { + return (bool) $job->safe_to_resubmit + && (string) $replica->state === 'failed' + && (int) $replica->image_bytes === 0; + } + return (string) $replica->state === 'active' + && (int) $replica->revision === (int) $template->revision + && (int) $replica->image_bytes === (int) $template->image_bytes + && hash_equals( + (string) $template->sha256, + (string) $replica->sha256 ); } @@ -1785,7 +1894,7 @@ final class CloudTemplateCatalogService if ((int) ($template->active_job_id ?? 0) > 0 || !in_array( (string) $template->state, - ['error', 'delete_failed'], + ['degraded', 'delete_failed'], true ) || self::missingPointerRecoveryPending($template)) { @@ -1882,7 +1991,7 @@ final class CloudTemplateCatalogService default => null, }; $expectedStates = match ($kind) { - 'template_import' => ['importing', 'error', 'active'], + 'template_import' => ['importing', 'error', 'degraded', 'active'], 'template_metadata_sync' => ['active'], 'template_delete' => ['deleting', 'delete_failed', 'active'], default => [], diff --git a/web/app/Services/Cloud/CloudTemplateReplicaCoordinator.php b/web/app/Services/Cloud/CloudTemplateReplicaCoordinator.php new file mode 100644 index 00000000..e57f065e --- /dev/null +++ b/web/app/Services/Cloud/CloudTemplateReplicaCoordinator.php @@ -0,0 +1,286 @@ +state, [ + 'active', 'degraded', 'error', + ], true)) { + throw new CloudError('云镜像主记录当前不能补齐宿主机副本', 409); + } + $source = CloudTemplateArtifactManager::downloadSource($snapshot); + $generation = (string) $snapshot->generation_token; + $revision = (int) $snapshot->revision; + + $job = $snapshot->getConnection()->transaction(static function () use ( + $templateId, + $hostId, + $actor, + $reason, + $eligibleHostIds, + $source, + $generation, + $revision + ): ?CloudJob { + $template = CloudTemplateCatalogService::lockForReplicaSync( + $templateId + ); + if ($template === null || !in_array((string) $template->state, [ + 'active', 'degraded', 'error', + ], true)) { + throw new CloudError( + '云镜像主记录当前不能补齐宿主机副本', + 409 + ); + } + if ((int) $template->revision !== $revision + || !hash_equals( + (string) $template->generation_token, + $generation + )) { + throw new CloudError('云镜像版本已变化,请重新同步', 409); + } + + $replica = CloudTemplateReplica::query() + ->where('host_id', $hostId) + ->where('template_id', $templateId) + ->lockForUpdate()->first(); + if ($replica !== null && self::isExact($template, $replica)) { + return null; + } + if ($replica !== null) { + if ((string) $replica->state !== 'failed') { + throw new CloudError( + '宿主机云镜像副本当前不能安全重新同步', + 409 + ); + } + self::assertRetryableFailure($template, $replica); + } + + $operationId = bin2hex(random_bytes(16)); + $body = [ + 'id' => (string) $template->id, + 'name' => (string) $template->name, + 'source_url' => $source['source_url'], + 'os_family' => (string) $template->os_family, + 'version' => (string) $template->os_version, + 'architecture' => 'x86_64', + 'sha256' => (string) $source['sha256'], + 'revision' => (int) $template->revision, + 'generation_token' => (string) $template->generation_token, + 'source_size_bytes' => (int) $source['source_size_bytes'], + ]; + $job = CloudJobService::reserveAgentMutation( + $actor, + $hostId, + 'template_import_replica', + 'POST', + '/v1/templates', + $body, + [ + 'can_manage' => true, + 'template_id' => (string) $template->id, + 'replica_sync' => true, + 'replica_sync_reason' => $reason, + ], + null, + $operationId + ); + $now = time(); + if ($replica === null) { + $replica = new CloudTemplateReplica(); + $replica->host_id = $hostId; + $replica->template_id = (string) $template->id; + $replica->created_at = $now; + $replica->state = 'pending'; + } + CloudTemplateStateMachine::moveReplica($replica, 'importing'); + $replica->revision = (int) $template->revision; + $replica->generation_token = + (string) $template->generation_token; + $replica->image_bytes = 0; + $replica->sha256 = (string) $template->sha256; + $replica->operation_id = $operationId; + $replica->job_id = (int) $job->id; + $replica->error_code = ''; + $replica->updated_at = $now; + $replica->save(); + + $replicas = CloudTemplateReplica::query() + ->where('template_id', (string) $template->id) + ->orderBy('host_id')->lockForUpdate()->get(); + $availability = CloudTemplateStateMachine::availability( + $template, + $replicas, + $eligibleHostIds + ); + CloudTemplateStateMachine::moveTemplate( + $template, + $availability + ); + $template->active_job_id = (int) $job->id; + $template->updated_at = $now; + $template->save(); + return $job; + }, 3); + + if ($job === null) { + return [ + 'scheduled' => false, + 'template_id' => $templateId, + 'host_id' => $hostId, + ]; + } + return ['scheduled' => true, 'template_id' => $templateId, + 'host_id' => $hostId] + CloudJobService::acknowledgement($job); + } + + /** Publish one standalone replica result while its job and rows are locked. */ + public static function settleStandaloneLocked( + CloudTemplate $template, + CloudTemplateReplica $replica, + CloudJob $job, + string $status, + array $operation + ): void { + if ($job->parent_job_id !== null + || (string) $job->kind !== 'template_import_replica' + || (int) $template->active_job_id !== (int) $job->id + || (int) $replica->job_id !== (int) $job->id) { + throw new CloudError('云镜像单宿主机同步账本不匹配', 409); + } + $settled = $status === 'succeeded' + || (($operation['effect_may_have_applied'] ?? null) === false + && ($operation['safe_to_resubmit'] ?? null) === true); + if (!$settled) { + return; + } + $eligibleHostIds = array_values(array_unique(array_map( + 'intval', + CloudAgentSettingsService::enabledHostIds() + ))); + sort($eligibleHostIds, SORT_NUMERIC); + $replicas = CloudTemplateReplica::query() + ->where('template_id', (string) $template->id) + ->orderBy('host_id')->lockForUpdate()->get(); + $availability = CloudTemplateStateMachine::availability( + $template, + $replicas, + $eligibleHostIds + ); + CloudTemplateStateMachine::moveTemplate($template, $availability); + if ($availability !== 'error') { + CloudTemplate::query() + ->where('catalog_key', (string) $template->catalog_key) + ->where('id', '<>', (string) $template->id) + ->where('is_default', 1) + ->update(['is_default' => 0, 'updated_at' => time()]); + $template->is_default = 1; + } + $template->active_job_id = null; + $template->updated_at = time(); + $template->save(); + } + + private static function assertRetryableFailure( + CloudTemplate $template, + CloudTemplateReplica $replica + ): void { + $job = $replica->job_id === null ? null + : CloudJob::query()->where('id', (int) $replica->job_id) + ->lockForUpdate()->first(); + if ($job === null + || (string) $job->kind !== 'template_import_replica' + || (int) $job->host_id !== (int) $replica->host_id + || (string) $job->execution_mode !== 'agent' + || (string) $job->phase !== 'terminal' + || (string) $job->outcome !== 'failed' + || (string) $job->status !== 'failed' + || (string) $job->effect_resolution !== 'not_applied' + || (bool) $job->effect_may_have_applied + || !(bool) $job->safe_to_resubmit + || (bool) $job->finalization_incident + || CloudJobService::hasAgentRecoveryEnvelopeConflict($job) + || !hash_equals( + (string) $job->operation_id, + (string) $replica->operation_id + ) + || !hash_equals( + (string) $template->generation_token, + (string) $replica->generation_token + )) { + throw new CloudError( + '宿主机副本失败结果尚未证明可安全重试', + 409 + ); + } + } + + private static function isExact( + CloudTemplate $template, + CloudTemplateReplica $replica + ): bool { + return (string) $replica->state === 'active' + && (int) $replica->revision === (int) $template->revision + && (int) $replica->image_bytes === (int) $template->image_bytes + && hash_equals( + (string) $template->generation_token, + (string) $replica->generation_token + ) + && hash_equals( + (string) $template->sha256, + (string) $replica->sha256 + ); + } + + private static function assertIdentity( + string $templateId, + int $hostId, + string $reason + ): void { + if (preg_match( + '/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D', + $templateId + ) !== 1 + || $hostId < 1 + || preg_match('/^[a-z][a-z0-9_]{1,47}$/D', $reason) !== 1) { + throw new CloudError('云镜像宿主机同步参数无效', 422); + } + } +} diff --git a/web/app/Services/Cloud/CloudTemplateStateMachine.php b/web/app/Services/Cloud/CloudTemplateStateMachine.php new file mode 100644 index 00000000..dd91890a --- /dev/null +++ b/web/app/Services/Cloud/CloudTemplateStateMachine.php @@ -0,0 +1,164 @@ +> */ + private const TEMPLATE_TRANSITIONS = [ + 'preparing' => ['importing', 'source_failed', 'source_unknown'], + 'source_failed' => ['retired'], + 'source_unknown' => ['retired'], + 'importing' => ['active', 'degraded', 'error', 'retired'], + 'active' => ['active', 'degraded', 'deleting'], + 'degraded' => ['active', 'degraded', 'error', 'deleting', 'retired'], + 'error' => ['active', 'degraded', 'error', 'deleting', 'retired'], + 'deleting' => ['active', 'degraded', 'delete_failed', 'retired'], + 'delete_failed' => ['active', 'degraded', 'deleting', 'retired'], + 'retired' => [], + ]; + + /** @var array> */ + private const REPLICA_TRANSITIONS = [ + 'pending' => ['importing'], + 'importing' => ['active', 'failed'], + 'failed' => ['importing', 'active', 'failed'], + 'active' => ['active', 'syncing', 'deleting'], + 'stale' => ['active', 'stale', 'syncing', 'deleting'], + 'syncing' => ['active', 'stale'], + 'deleting' => ['delete_succeeded', 'delete_failed'], + 'delete_failed' => [ + 'active', 'stale', 'deleting', 'delete_failed', 'delete_succeeded', + ], + 'delete_succeeded' => [], + ]; + + public static function moveTemplate( + CloudTemplate $template, + string $next + ): void { + self::assertTransition( + self::TEMPLATE_TRANSITIONS, + (string) $template->state, + $next, + '云镜像主记录' + ); + $template->state = $next; + } + + public static function moveReplica( + CloudTemplateReplica $replica, + string $next + ): void { + self::assertTransition( + self::REPLICA_TRANSITIONS, + (string) $replica->state, + $next, + '云镜像宿主机副本' + ); + $replica->state = $next; + } + + public static function assertTemplateTransition( + string $current, + string $next + ): void { + self::assertTransition( + self::TEMPLATE_TRANSITIONS, + $current, + $next, + '云镜像主记录' + ); + } + + public static function assertReplicaTransition( + string $current, + string $next + ): void { + self::assertTransition( + self::REPLICA_TRANSITIONS, + $current, + $next, + '云镜像宿主机副本' + ); + } + + /** + * Project catalog availability from exact replicas on currently eligible + * hosts. Replicas on disabled hosts do not make a template healthy. + * + * @param iterable $replicas + * @param list $eligibleHostIds + */ + public static function availability( + CloudTemplate $template, + iterable $replicas, + array $eligibleHostIds + ): string { + $eligible = []; + foreach ($eligibleHostIds as $hostId) { + $hostId = (int) $hostId; + if ($hostId > 0) { + $eligible[$hostId] = true; + } + } + if ($eligible === []) { + return 'error'; + } + + $ready = []; + foreach ($replicas as $replica) { + $hostId = (int) $replica->host_id; + if (!isset($eligible[$hostId]) + || (string) $replica->state !== 'active' + || (int) $replica->revision !== (int) $template->revision + || !hash_equals( + (string) $template->generation_token, + (string) $replica->generation_token + ) + || (int) $replica->image_bytes !== (int) $template->image_bytes + || !hash_equals( + (string) $template->sha256, + (string) $replica->sha256 + )) { + continue; + } + $ready[$hostId] = true; + } + if ($ready === []) { + return 'error'; + } + return count($ready) === count($eligible) ? 'active' : 'degraded'; + } + + /** @param array> $definition */ + private static function assertTransition( + array $definition, + string $current, + string $next, + string $label + ): void { + if (array_key_exists($current, $definition) && $current === $next) { + return; + } + if (!array_key_exists($current, $definition) + || !in_array($next, $definition[$current], true)) { + throw new CloudError(sprintf( + '%s状态不能从 %s 变更为 %s', + $label, + $current === '' ? '(empty)' : $current, + $next === '' ? '(empty)' : $next + ), 409); + } + } +} diff --git a/web/tests/CloudAggregateRecoveryConvergenceTest.php b/web/tests/CloudAggregateRecoveryConvergenceTest.php index 01527e2a..862f8ffc 100644 --- a/web/tests/CloudAggregateRecoveryConvergenceTest.php +++ b/web/tests/CloudAggregateRecoveryConvergenceTest.php @@ -384,7 +384,7 @@ namespace { $generation = md5('projection-generation:' . $parentId); $sha256 = hash('sha256', 'projection-sha:' . $parentId); $state = match ((string) $parent->kind) { - 'template_import' => 'error', + 'template_import' => 'degraded', 'template_metadata_sync' => 'active', 'template_delete' => 'delete_failed', }; @@ -589,7 +589,7 @@ namespace { && $partial->lease_token === null && (int) $partial->available_at === 0 && $partialTemplate !== null - && (string) $partialTemplate->state === 'error' + && (string) $partialTemplate->state === 'degraded' && $partialTemplate->active_job_id === null && $partialReplicaStates === ['active', 'failed'] && is_array($partialTemplateDto) @@ -2995,7 +2995,7 @@ namespace { ->where('id', $legacyPartialResourceId)->first(); $expectedState = str_ends_with($parentKind, '_import') ? (str_starts_with($parentKind, 'template_') - ? 'error' : 'import_failed') + ? 'degraded' : 'import_failed') : 'delete_failed'; $assert((string) $legacyPartial->effect_resolution === 'applied' && str_ends_with( diff --git a/web/tests/CloudTemplateReplicaCoordinatorContractTest.php b/web/tests/CloudTemplateReplicaCoordinatorContractTest.php new file mode 100644 index 00000000..a7e33a52 --- /dev/null +++ b/web/tests/CloudTemplateReplicaCoordinatorContractTest.php @@ -0,0 +1,57 @@ + true") + && str_contains($coordinator, + "\$job->effect_resolution !== 'not_applied'") + && str_contains($coordinator, + 'CloudTemplateStateMachine::availability(') + && str_contains($catalog, + 'CloudTemplateReplicaCoordinator::settleStandaloneLocked('), + 'Standalone replica synchronization is not bound to the immutable artifact and safe retry evidence.'); + +$assert(str_contains($identity, + '$standaloneImport = (string) $job->kind') + && str_contains($identity, + "&& (\$payload['replica_sync'] ?? null) === true") + && str_contains($identity, + '? (int) $job->id : (int) $parent->id)'), + 'Recovery identity cannot distinguish a standalone replica sync from a damaged aggregate child.'); + +$assert(str_contains($automatic, + "&& (string) \$job->execution_mode === 'aggregate'") + && !str_contains($agent, '"template_name_exists"'), + 'Standalone repair is misclassified as aggregate damage or template versions still conflict by display name.'); + +echo "Template standalone replica coordinator contract passed.\n"; diff --git a/web/tests/CloudTemplateReplicaStateMachineTest.php b/web/tests/CloudTemplateReplicaStateMachineTest.php new file mode 100644 index 00000000..dcc33ffb --- /dev/null +++ b/web/tests/CloudTemplateReplicaStateMachineTest.php @@ -0,0 +1,77 @@ +state = 'preparing'; +$template->revision = 3; +$template->generation_token = str_repeat('a', 32); +$template->image_bytes = 4096; +$template->sha256 = str_repeat('b', 64); +CloudTemplateStateMachine::moveTemplate($template, 'importing'); +CloudTemplateStateMachine::moveTemplate($template, 'degraded'); +$assert((string) $template->state === 'degraded', + 'Template state machine did not permit a partial initial import.'); + +$invalidRejected = false; +try { + CloudTemplateStateMachine::moveTemplate($template, 'source_failed'); +} catch (CloudError $error) { + $invalidRejected = $error->httpStatus === 409; +} +$assert($invalidRejected, + 'Template state machine accepted an impossible source transition.'); + +$ready = new CloudTemplateReplica(); +$ready->host_id = 11; +$ready->state = 'pending'; +$ready->revision = 3; +$ready->generation_token = str_repeat('a', 32); +$ready->image_bytes = 4096; +$ready->sha256 = str_repeat('b', 64); +CloudTemplateStateMachine::moveReplica($ready, 'importing'); +CloudTemplateStateMachine::moveReplica($ready, 'active'); + +$failed = new CloudTemplateReplica(); +$failed->host_id = 12; +$failed->state = 'pending'; +$failed->revision = 3; +$failed->generation_token = str_repeat('a', 32); +$failed->image_bytes = 0; +$failed->sha256 = str_repeat('b', 64); +CloudTemplateStateMachine::moveReplica($failed, 'importing'); +CloudTemplateStateMachine::moveReplica($failed, 'failed'); + +$assert(CloudTemplateStateMachine::availability( + $template, [$ready, $failed], [11, 12] +) === 'degraded', + 'One healthy and one failed eligible replica was not degraded.'); +$assert(CloudTemplateStateMachine::availability( + $template, [$ready, $failed], [11] +) === 'active', + 'A complete eligible-host projection was not active.'); +$assert(CloudTemplateStateMachine::availability( + $template, [$failed], [12] +) === 'error', + 'A catalog without any exact healthy replica was exposed as available.'); + +CloudTemplateStateMachine::moveReplica($failed, 'importing'); +CloudTemplateStateMachine::moveReplica($failed, 'active'); +$failed->image_bytes = 4096; +$assert(CloudTemplateStateMachine::availability( + $template, [$ready, $failed], [11, 12] +) === 'active', + 'A safely retried exact replica did not restore full availability.'); + +echo "Template replica state-machine behavior passed.\n"; diff --git a/web/tests/CloudTemplateStandaloneReplicaFinalizationTest.php b/web/tests/CloudTemplateStandaloneReplicaFinalizationTest.php new file mode 100644 index 00000000..75555d5d --- /dev/null +++ b/web/tests/CloudTemplateStandaloneReplicaFinalizationTest.php @@ -0,0 +1,199 @@ +addConnection([ + 'driver' => 'sqlite', 'database' => ':memory:', +], 'default'); +$capsule->setAsGlobal(); +$capsule->bootEloquent(); +$schema = $capsule->schema('default'); +$schema->create('cloud_hosts', static function (Blueprint $table): void { + $table->integer('id')->primary(); + $table->boolean('enabled'); + $table->string('default_network'); +}); +$schema->create('cloud_host_agents', static function (Blueprint $table): void { + $table->integer('host_id')->primary(); + $table->boolean('enabled'); + $table->string('last_test_status'); + $table->integer('tested_revision'); + $table->integer('revision'); +}); +$schema->create('cloud_host_storage_configurations', + static function (Blueprint $table): void { + $table->integer('host_id')->primary(); + $table->string('state'); + }); +$schema->create('cloud_templates', static function (Blueprint $table): void { + $table->string('id')->primary(); + $table->string('catalog_key'); + $table->string('version_label'); + $table->boolean('is_default'); + $table->string('name'); + $table->string('os_family'); + $table->string('os_version'); + $table->string('source_type'); + $table->text('sealed_source_url'); + $table->integer('image_bytes'); + $table->string('sha256'); + $table->text('startup_script'); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + $table->integer('active_job_id')->nullable(); + $table->integer('created_at'); + $table->integer('updated_at'); +}); +$schema->create('cloud_template_replicas', + static function (Blueprint $table): void { + $table->integer('host_id'); + $table->string('template_id'); + $table->integer('revision'); + $table->string('generation_token'); + $table->string('state'); + $table->integer('image_bytes'); + $table->string('sha256'); + $table->string('operation_id'); + $table->integer('job_id')->nullable(); + $table->string('error_code'); + $table->integer('created_at'); + $table->integer('updated_at'); + $table->primary(['host_id', 'template_id']); + }); + +foreach ([31, 32] as $hostId) { + Capsule::table('cloud_hosts')->insert([ + 'id' => $hostId, 'enabled' => 1, + 'default_network' => 'bridge:br0', + ]); + Capsule::table('cloud_host_agents')->insert([ + 'host_id' => $hostId, 'enabled' => 1, + 'last_test_status' => 'healthy', 'tested_revision' => 4, + 'revision' => 4, + ]); + Capsule::table('cloud_host_storage_configurations')->insert([ + 'host_id' => $hostId, 'state' => 'applied', + ]); +} +$now = time(); +$generation = str_repeat('a', 32); +$sha = str_repeat('b', 64); +Capsule::table('cloud_templates')->insert([ + 'id' => 'standalone-replica-template', + 'catalog_key' => 'debian-13', + 'version_label' => 'v2', + 'is_default' => 1, + 'name' => 'Debian 13', + 'os_family' => 'Debian', + 'os_version' => '13', + 'source_type' => 'object', + 'sealed_source_url' => 'test-only', + 'image_bytes' => 4096, + 'sha256' => $sha, + 'startup_script' => '', + 'revision' => 2, + 'generation_token' => $generation, + 'state' => 'degraded', + 'active_job_id' => 501, + 'created_at' => $now, + 'updated_at' => $now, +]); +Capsule::table('cloud_template_replicas')->insert([ + [ + 'host_id' => 31, 'template_id' => 'standalone-replica-template', + 'revision' => 2, 'generation_token' => $generation, + 'state' => 'active', 'image_bytes' => 4096, 'sha256' => $sha, + 'operation_id' => str_repeat('1', 32), 'job_id' => 500, + 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, + ], + [ + 'host_id' => 32, 'template_id' => 'standalone-replica-template', + 'revision' => 2, 'generation_token' => $generation, + 'state' => 'importing', 'image_bytes' => 0, 'sha256' => $sha, + 'operation_id' => str_repeat('2', 32), 'job_id' => 501, + 'error_code' => '', 'created_at' => $now, 'updated_at' => $now, + ], +]); + +$job = new CloudJob(); +$job->id = 501; +$job->host_id = 32; +$job->kind = 'template_import_replica'; +$job->parent_job_id = null; +$job->operation_id = str_repeat('2', 32); +$payload = ['template_id' => 'standalone-replica-template']; +$result = [ + 'id' => 'standalone-replica-template', + 'revision' => 2, + 'generation_token' => $generation, + 'size_bytes' => 4096, + 'sha256' => $sha, +]; +CloudTemplateCatalogService::finishJobLocked( + $job, + 'succeeded', + $result, + $payload, + ['effect_may_have_applied' => false, 'safe_to_resubmit' => false] +); +$template = CloudTemplate::findOrFail('standalone-replica-template'); +$assert((string) $template->state === 'active' + && $template->active_job_id === null + && (string) CloudTemplateReplica::query()->where('host_id', 32) + ->where('template_id', (string) $template->id)->value('state') + === 'active', + 'An exact standalone replica receipt did not restore full availability.'); + +Capsule::table('cloud_templates')->where('id', (string) $template->id) + ->update(['state' => 'degraded', 'active_job_id' => 502]); +Capsule::table('cloud_template_replicas')->where('host_id', 32) + ->where('template_id', (string) $template->id)->update([ + 'state' => 'importing', 'image_bytes' => 0, + 'job_id' => 502, 'operation_id' => str_repeat('3', 32), + ]); +$job->id = 502; +$job->operation_id = str_repeat('3', 32); +CloudTemplateCatalogService::finishJobLocked( + $job, + 'failed', + null, + $payload, + [ + 'effect_may_have_applied' => true, + 'safe_to_resubmit' => false, + 'error_code' => 'agent_timeout', + ] +); +$template->refresh(); +$assert((int) $template->active_job_id === 502 + && (string) $template->state === 'degraded', + 'An unknown standalone replica effect released the template fence.'); + +CloudTemplateCatalogService::finishJobLocked( + $job, + 'succeeded', + $result, + $payload, + ['effect_may_have_applied' => false, 'safe_to_resubmit' => false], + true +); +$template->refresh(); +$assert($template->active_job_id === null + && (string) $template->state === 'active', + 'Recovered standalone replica evidence did not release the original fence.'); + +echo "Template standalone replica finalization behavior passed.\n"; diff --git a/web/tests/VirtualizationAssetsTest.php b/web/tests/VirtualizationAssetsTest.php index 23bf647b..6abf2e33 100644 --- a/web/tests/VirtualizationAssetsTest.php +++ b/web/tests/VirtualizationAssetsTest.php @@ -83,6 +83,8 @@ $adminBillingStyles = $public . '/admin-billing.css'; $adminSharedStyles = file_get_contents($root . '/public/theme/venix/css/admin-home.css'); $templateCatalogService = file_get_contents( $root . '/app/Services/Cloud/CloudTemplateCatalogService.php'); +$templateStateMachine = file_get_contents( + $root . '/app/Services/Cloud/CloudTemplateStateMachine.php'); $assert(str_contains($cloudShell, "file='user/panel_sidebar.tpl'") && str_contains($userLayout, "file='user/panel_sidebar.tpl'"), 'Cloud and account pages must share one sidebar partial'); $assert(str_contains($cloudShell, "file='user/panel_topbar.tpl'") @@ -711,7 +713,10 @@ $assert(str_contains($cloudSchema, 'CREATE TABLE cloud_templates') && !str_contains($cloudSchema, 'cloud_template_retirements') && !str_contains($cloudSchema, 'cloud_template_catalog_locks') && str_contains($templateCatalogService, 'lockForUpdate()') - && str_contains($templateCatalogService, '$template->state = \'retired\'') + && str_contains($templateCatalogService, + "CloudTemplateStateMachine::moveTemplate(\$template, 'retired')") + && is_string($templateStateMachine) + && str_contains($templateStateMachine, 'TEMPLATE_TRANSITIONS') && str_contains($deploymentPageService, 'CloudTemplateCatalogService::publicCatalog()'), 'Template catalog must use the fresh PHP schema and CAS retirement boundary.'); $assert(str_contains($clientScript,