feat(template): coordinate replica lifecycle

go
codex 3 weeks ago
parent 59c3586e3a
commit d166b56aa4

@ -121,15 +121,6 @@ func (l *Libvirt) ImportTemplate(ctx context.Context, request ImportTemplateRequ
if _, err := validateRemoteSource(request.SourceURL); err != nil {
return ImageTemplate{}, err
}
templates, err := l.ListTemplates(ctx)
if err != nil {
return ImageTemplate{}, err
}
for _, item := range templates {
if strings.EqualFold(item.Name, request.Name) {
return ImageTemplate{}, NewError(ErrConflict, "template_name_exists", "a template with this name already exists on this host", nil)
}
}
existing, err := l.lookupOptionalManagedVolume(l.storagePoolName(StorageRoleTemplate), templateVolumeName(request.ID))
if err != nil {
return ImageTemplate{}, err

@ -654,6 +654,7 @@ final class CloudAutomaticOperationRecoveryService
}
if (!$envelopeConflict
&& (string) $job->phase === 'terminal'
&& (string) $job->execution_mode === 'aggregate'
&& (CloudTemplate::where(
'active_job_id',
(int) $job->id

@ -1862,11 +1862,17 @@ final class CloudOperationRecoveryIdentity
'template_metadata_replica' => 'template_metadata_sync',
'template_delete_replica' => 'template_delete',
][(string) $job->kind];
$standaloneImport = (string) $job->kind
=== 'template_import_replica'
&& $job->parent_job_id === null
&& ($payload['replica_sync'] ?? null) === true;
if (!is_string($id)
|| preg_match(self::TEMPLATE_ID_PATTERN, $id) !== 1
|| $template === null || $replica === null || $parent === null
|| (string) $parent->kind !== $expectedParent
|| (int) $template->active_job_id !== (int) $parent->id
|| $template === null || $replica === null
|| (!$standaloneImport && ($parent === null
|| (string) $parent->kind !== $expectedParent))
|| (int) $template->active_job_id !== ($standaloneImport
? (int) $job->id : (int) $parent->id)
|| (int) $replica->job_id !== (int) $job->id
|| !hash_equals((string) $replica->operation_id,
(string) $job->operation_id)) {

@ -186,7 +186,10 @@ final class CloudTemplateArtifactManager
$artifact->error_code = substr($code, 0, 64);
$artifact->updated_at = $now;
$artifact->save();
$template->state = $unknown ? 'source_unknown' : 'source_failed';
CloudTemplateStateMachine::moveTemplate(
$template,
$unknown ? 'source_unknown' : 'source_failed'
);
$template->updated_at = $now;
$template->save();
},

@ -181,7 +181,7 @@ final class CloudTemplateCatalogService
return $parent;
}
$now = time();
$locked->state = 'importing';
CloudTemplateStateMachine::moveTemplate($locked, 'importing');
$locked->active_job_id = (int) $parent->id;
$locked->updated_at = $now;
$locked->save();
@ -215,7 +215,11 @@ final class CloudTemplateCatalogService
$replica->revision = 1;
$replica->generation_token =
(string) $locked->generation_token;
$replica->state = 'importing';
$replica->state = 'pending';
CloudTemplateStateMachine::moveReplica(
$replica,
'importing'
);
$replica->image_bytes = 0;
$replica->sha256 = (string) $locked->sha256;
$replica->operation_id = $operationId;
@ -290,15 +294,12 @@ final class CloudTemplateCatalogService
'template_id' => $id,
], (int) $parent->id, $operationId
);
CloudTemplateReplica::query()
->where('host_id', (int) $replica->host_id)
->where('template_id', $id)->update([
'state' => 'syncing',
'operation_id' => $operationId,
'job_id' => (int) $child->id,
'error_code' => '',
'updated_at' => time(),
]);
CloudTemplateStateMachine::moveReplica($replica, 'syncing');
$replica->operation_id = $operationId;
$replica->job_id = (int) $child->id;
$replica->error_code = '';
$replica->updated_at = time();
$replica->save();
}
return $parent;
}, 3);
@ -327,7 +328,7 @@ final class CloudTemplateCatalogService
$template
);
$partialCleanup = ($partialCleanupKind === 'template_import'
&& (string) $template->state === 'error')
&& (string) $template->state === 'degraded')
|| ($partialCleanupKind === 'template_delete'
&& (string) $template->state === 'delete_failed');
if ((string) $template->state !== 'active' && !$partialCleanup) {
@ -382,9 +383,11 @@ final class CloudTemplateCatalogService
->where('template_id', $id)
->where('state', 'delete_failed')
->lockForUpdate()->get() as $survivor) {
$survivor->state = (int) $survivor->revision
=== (int) $template->revision
? 'active' : 'stale';
CloudTemplateStateMachine::moveReplica(
$survivor,
(int) $survivor->revision === (int) $template->revision
? 'active' : 'stale'
);
$survivor->error_code = '';
$survivor->updated_at = time();
$survivor->save();
@ -404,7 +407,7 @@ final class CloudTemplateCatalogService
throw new CloudError('云镜像部分结果账本不完整,禁止删除', 409);
}
}
$template->state = 'deleting';
CloudTemplateStateMachine::moveTemplate($template, 'deleting');
$template->active_job_id = (int) $parent->id;
$template->updated_at = time();
$template->save();
@ -418,15 +421,12 @@ final class CloudTemplateCatalogService
'template_id' => $id,
], (int) $parent->id, $operationId
);
CloudTemplateReplica::query()
->where('host_id', (int) $replica->host_id)
->where('template_id', $id)->update([
'state' => 'deleting',
'operation_id' => $operationId,
'job_id' => (int) $child->id,
'error_code' => '',
'updated_at' => time(),
]);
CloudTemplateStateMachine::moveReplica($replica, 'deleting');
$replica->operation_id = $operationId;
$replica->job_id = (int) $child->id;
$replica->error_code = '';
$replica->updated_at = time();
$replica->save();
}
return $parent;
}, 3);
@ -454,13 +454,21 @@ final class CloudTemplateCatalogService
if ($template === null || $replica === null) {
throw new CloudError('云镜像任务账本不存在', 409);
}
// Only project this replica here. CloudWorkerService persists the
// child's terminal effect first, then reconciles the aggregate parent
// through settleRecoveredParentLocked(), where the complete child /
// replica topology and content can be verified atomically.
// Project the leaf first. Aggregate children are published only by the
// parent reconciler; standalone repair jobs publish availability here
// only after a definitive result and keep unknown effects fenced.
if ((string) $job->kind === 'template_import_replica') {
self::finishImportReplica($template, $replica, $status, $result,
$operation, $recovery);
if ($job->parent_job_id === null) {
CloudTemplateReplicaCoordinator::settleStandaloneLocked(
$template,
$replica,
$job,
$status,
$operation
);
}
} elseif ((string) $job->kind === 'template_metadata_replica') {
self::finishMetadataReplica($template, $replica, $status, $result,
$operation, $recovery);
@ -634,7 +642,7 @@ final class CloudTemplateCatalogService
if ($resolution === 'mixed'
&& $kind !== 'template_metadata_sync') {
$expectedState = $kind === 'template_import'
? 'error' : 'delete_failed';
? 'degraded' : 'delete_failed';
if ((string) $template->state !== $expectedState) {
throw new CloudRecoveryInstanceStateConflict(
'云镜像汇总任务的部分成功目录状态不一致',
@ -848,7 +856,7 @@ final class CloudTemplateCatalogService
'template_id',
(string) $template->id
)->delete();
$template->state = 'retired';
CloudTemplateStateMachine::moveTemplate($template, 'retired');
$template->updated_at = time();
$template->save();
CloudJobService::completeAggregateLocked(
@ -866,13 +874,16 @@ final class CloudTemplateCatalogService
(string) $template->id
)->orderBy('host_id')->lockForUpdate()->get();
foreach ($replicas as $replica) {
$replica->state = (int) $replica->revision
=== (int) $template->revision ? 'active' : 'stale';
CloudTemplateStateMachine::moveReplica(
$replica,
(int) $replica->revision === (int) $template->revision
? 'active' : 'stale'
);
$replica->error_code = '';
$replica->updated_at = time();
$replica->save();
}
$template->state = 'active';
CloudTemplateStateMachine::moveTemplate($template, 'active');
$template->updated_at = time();
$template->save();
CloudJobService::completeAggregateLocked(
@ -884,6 +895,10 @@ final class CloudTemplateCatalogService
);
return;
}
CloudTemplateStateMachine::assertReplicaTransition(
'syncing',
'stale'
);
CloudTemplateReplica::query()->where(
'template_id',
(string) $template->id
@ -891,7 +906,7 @@ final class CloudTemplateCatalogService
'state' => 'stale',
'updated_at' => time(),
]);
$template->state = 'active';
CloudTemplateStateMachine::moveTemplate($template, 'active');
$template->updated_at = time();
$template->save();
CloudJobService::completeAggregateLocked(
@ -940,13 +955,13 @@ final class CloudTemplateCatalogService
$template->updated_at = time();
$template->save();
}
$replica->state = 'active';
CloudTemplateStateMachine::moveReplica($replica, 'active');
$replica->revision = (int) $template->revision;
$replica->image_bytes = (int) $bytes;
$replica->sha256 = $digest;
$replica->error_code = '';
} else {
$replica->state = 'failed';
CloudTemplateStateMachine::moveReplica($replica, 'failed');
$replica->error_code = self::errorCode(
$operation, 'template_import_failed'
);
@ -984,16 +999,34 @@ final class CloudTemplateCatalogService
}
$success = !$replicas->contains(static fn ($row): bool =>
(string) $row->state !== 'active');
if ($success) {
$hostIds = $replicas->map(
static fn (CloudTemplateReplica $replica): int =>
(int) $replica->host_id
)->all();
$availability = CloudTemplateStateMachine::availability(
$template,
$replicas,
$hostIds
);
if ($availability !== 'error') {
CloudTemplate::query()
->where('catalog_key', (string) $template->catalog_key)
->where('id', '<>', (string) $template->id)
->where('is_default', 1)
->update(['is_default' => 0, 'updated_at' => time()]);
$template->state = 'active';
CloudTemplateStateMachine::moveTemplate(
$template,
$availability
);
$template->is_default = 1;
$template->updated_at = time();
$template->save();
} else {
CloudTemplateStateMachine::moveTemplate($template, 'error');
$template->updated_at = time();
$template->save();
}
if ($success) {
CloudJobService::completeAggregateLocked($parent, 'succeeded',
true, false, '', ['template_id' => (string) $template->id]);
return;
@ -1003,9 +1036,6 @@ final class CloudTemplateCatalogService
|| ($operation['effect_may_have_applied'] ?? false) === true
|| CloudJob::where('parent_job_id', $parentId)
->where('effect_may_have_applied', 1)->exists();
$template->state = 'error';
$template->updated_at = time();
$template->save();
CloudJobService::completeAggregateLocked($parent, 'failed', $effect,
false, 'template_import_failed');
}
@ -1030,11 +1060,11 @@ final class CloudTemplateCatalogService
!== (string) $template->generation_token) {
throw new CloudError('Agent 元数据同步回执与请求不匹配', 502);
}
$replica->state = 'active';
CloudTemplateStateMachine::moveReplica($replica, 'active');
$replica->revision = (int) $template->revision;
$replica->error_code = '';
} else {
$replica->state = 'stale';
CloudTemplateStateMachine::moveReplica($replica, 'stale');
$replica->error_code = self::errorCode(
$operation, 'template_metadata_sync_failed'
);
@ -1104,10 +1134,13 @@ final class CloudTemplateCatalogService
)) {
throw new CloudError('Agent 云镜像删除回执缺少清理证据', 502);
}
$replica->state = 'delete_succeeded';
CloudTemplateStateMachine::moveReplica(
$replica,
'delete_succeeded'
);
$replica->error_code = '';
} else {
$replica->state = 'delete_failed';
CloudTemplateStateMachine::moveReplica($replica, 'delete_failed');
$replica->error_code = self::errorCode(
$operation, 'template_delete_failed'
);
@ -1146,7 +1179,7 @@ final class CloudTemplateCatalogService
CloudTemplateReplica::query()->where(
'template_id', (string) $template->id
)->delete();
$template->state = 'retired';
CloudTemplateStateMachine::moveTemplate($template, 'retired');
$template->revision = (int) $template->revision + 1;
$template->updated_at = time();
$template->save();
@ -1162,7 +1195,7 @@ final class CloudTemplateCatalogService
|| ($operation['effect_may_have_applied'] ?? false) === true
|| CloudJob::where('parent_job_id', $parentId)
->where('effect_may_have_applied', 1)->exists();
$template->state = 'delete_failed';
CloudTemplateStateMachine::moveTemplate($template, 'delete_failed');
$template->updated_at = time();
$template->save();
CloudJobService::completeAggregateLocked($parent, 'failed', $effect,
@ -1199,6 +1232,7 @@ final class CloudTemplateCatalogService
'template_metadata_sync' => 'stale',
default => 'delete_failed',
};
CloudTemplateStateMachine::assertReplicaTransition($from, $to);
CloudTemplateReplica::query()
->where('template_id', (string) $template->id)
->where('state', $from)->update([
@ -1207,9 +1241,24 @@ final class CloudTemplateCatalogService
'updated_at' => time(),
]);
if ((string) $parent->kind === 'template_import') {
$template->state = 'error';
$replicas = self::lockedReplicas((string) $template->id);
$availability = CloudTemplateStateMachine::availability(
$template,
$replicas,
$replicas->map(
static fn (CloudTemplateReplica $replica): int =>
(int) $replica->host_id
)->all()
);
CloudTemplateStateMachine::moveTemplate(
$template,
$availability
);
} elseif ((string) $parent->kind === 'template_delete') {
$template->state = 'delete_failed';
CloudTemplateStateMachine::moveTemplate(
$template,
'delete_failed'
);
}
$template->updated_at = time();
$template->save();
@ -1371,6 +1420,17 @@ final class CloudTemplateCatalogService
}
}
/**
* Acquire the canonical mutation boundary for one host-replica sync.
* The coordinator still owns replica and job validation; exposing this
* narrow lock keeps the parent -> template lock order in one module.
*/
public static function lockForReplicaSync(
string $id
): ?CloudTemplate {
return self::lockTemplateForMutation(self::templateId($id));
}
/**
* Discover the old parent without a row lock, then acquire the canonical
* parent -> catalog order and reject a pointer that changed in between.
@ -1541,7 +1601,7 @@ final class CloudTemplateCatalogService
? 'applied' : $actualResolution;
$projectionStateMatches = match ((string) $parent->kind) {
'template_import' => (string) $template->state
=== ($actualResolution === 'mixed' ? 'error' : 'active'),
=== ($actualResolution === 'mixed' ? 'degraded' : 'active'),
'template_metadata_sync' => (string) $template->state === 'active',
'template_delete' => (string) $template->state
=== ($actualResolution === 'mixed'
@ -1666,10 +1726,9 @@ final class CloudTemplateCatalogService
}
/**
* Treat a NULL/zero pointer as safe only when the current immutable
* replica/child chain proves that its one historical parent is fully
* settled. Any in-flight, incomplete, cross-wired or ambiguous chain
* remains fenced for the parent-first recovery sweep.
* Treat a NULL/zero pointer as safe only when every current replica owns a
* settled, identity-matched job. A catalog may legitimately contain a mix
* of historical aggregate children and newer standalone repair jobs.
*/
private static function missingPointerRecoveryPending(
CloudTemplate $template
@ -1704,7 +1763,10 @@ final class CloudTemplateCatalogService
if (CloudUserIsoReplica::query()->whereIn(
'job_id',
$jobIds
)->exists()) {
)->exists() || CloudTemplateReplica::query()->whereIn(
'job_id',
$jobIds
)->where('template_id', '<>', $templateId)->exists()) {
return true;
}
$children = CloudJob::whereIn('id', $jobIds)->orderBy('id')->get();
@ -1712,6 +1774,7 @@ final class CloudTemplateCatalogService
return true;
}
$parentIds = [];
$hasStandalone = false;
foreach ($children as $child) {
$matches = $replicas->filter(
static fn (CloudTemplateReplica $replica): bool =>
@ -1720,7 +1783,6 @@ final class CloudTemplateCatalogService
$replica = $matches->count() === 1 ? $matches->first() : null;
$parentId = (int) ($child->parent_job_id ?? 0);
if ($replica === null
|| $parentId < 1
|| (string) $child->execution_mode !== 'agent'
|| (int) $child->host_id < 1
|| (int) $replica->host_id !== (int) $child->host_id
@ -1738,43 +1800,90 @@ final class CloudTemplateCatalogService
)) {
return true;
}
if ($parentId < 1) {
$hasStandalone = true;
if (!self::standaloneReplicaJobSafelySettled(
$template,
$replica,
$child
)) {
return true;
}
continue;
}
$parentIds[$parentId] = true;
}
if (count($parentIds) !== 1) {
return true;
foreach (array_keys($parentIds) as $parentId) {
$parent = CloudJob::find((int) $parentId);
$expectedChildKind = match ((string) ($parent?->kind ?? '')) {
'template_import' => 'template_import_replica',
'template_metadata_sync' => 'template_metadata_replica',
'template_delete' => 'template_delete_replica',
default => null,
};
if ($parent === null
|| $expectedChildKind === null
|| (string) $parent->execution_mode !== 'aggregate'
|| $parent->parent_job_id !== null
|| !self::aggregateParentSafelySettled($parent, false)
|| CloudTemplate::query()
->where('active_job_id', (int) $parent->id)
->where('id', '<>', $templateId)->exists()) {
return true;
}
foreach ($children->where('parent_job_id', (int) $parentId)
as $child) {
if ((string) $child->kind !== $expectedChildKind
|| (int) $child->user_id !== (int) $parent->user_id) {
return true;
}
}
}
$parent = CloudJob::find((int) array_key_first($parentIds));
$expectedChildKind = match ((string) ($parent?->kind ?? '')) {
'template_import' => 'template_import_replica',
'template_metadata_sync' => 'template_metadata_replica',
'template_delete' => 'template_delete_replica',
default => null,
};
if ($parent === null
|| $expectedChildKind === null
|| (string) $parent->execution_mode !== 'aggregate'
|| $parent->parent_job_id !== null
|| CloudJob::where('parent_job_id', (int) $parent->id)->count()
!== $children->count()
|| CloudTemplateReplica::query()->whereIn('job_id', $jobIds)
->where('template_id', '<>', $templateId)->exists()
|| CloudUserIsoReplica::query()->whereIn(
'job_id',
$jobIds
)->exists()
|| CloudTemplate::query()
->where('active_job_id', (int) $parent->id)
->where('id', '<>', $templateId)->exists()
|| $children->contains(static fn (CloudJob $child): bool =>
(string) $child->kind !== $expectedChildKind
|| (int) $child->user_id !== (int) $parent->user_id)) {
return true;
if (!$hasStandalone && count($parentIds) === 1) {
$parent = CloudJob::find((int) array_key_first($parentIds));
$allChildrenBelongToCurrentReplicas = $parent !== null
&& CloudJob::where(
'parent_job_id',
(int) $parent->id
)->count() === $children->count();
if ($allChildrenBelongToCurrentReplicas
&& !self::currentParentProjectionMatches(
$template,
$parent,
false
)) {
return true;
}
}
return !self::aggregateParentSafelySettled($parent, false)
|| !self::currentParentProjectionMatches(
$template,
$parent,
false
return false;
}
private static function standaloneReplicaJobSafelySettled(
CloudTemplate $template,
CloudTemplateReplica $replica,
CloudJob $job
): bool {
if ($job->parent_job_id !== null
|| (string) $job->kind !== 'template_import_replica'
|| (string) $job->phase !== 'terminal'
|| !in_array((string) $job->effect_resolution, [
'applied', 'not_applied',
], true)
|| (bool) $job->finalization_incident
|| CloudJobService::hasAgentRecoveryEnvelopeConflict($job)) {
return false;
}
if ((string) $job->effect_resolution === 'not_applied') {
return (bool) $job->safe_to_resubmit
&& (string) $replica->state === 'failed'
&& (int) $replica->image_bytes === 0;
}
return (string) $replica->state === 'active'
&& (int) $replica->revision === (int) $template->revision
&& (int) $replica->image_bytes === (int) $template->image_bytes
&& hash_equals(
(string) $template->sha256,
(string) $replica->sha256
);
}
@ -1785,7 +1894,7 @@ final class CloudTemplateCatalogService
if ((int) ($template->active_job_id ?? 0) > 0
|| !in_array(
(string) $template->state,
['error', 'delete_failed'],
['degraded', 'delete_failed'],
true
)
|| self::missingPointerRecoveryPending($template)) {
@ -1882,7 +1991,7 @@ final class CloudTemplateCatalogService
default => null,
};
$expectedStates = match ($kind) {
'template_import' => ['importing', 'error', 'active'],
'template_import' => ['importing', 'error', 'degraded', 'active'],
'template_metadata_sync' => ['active'],
'template_delete' => ['deleting', 'delete_failed', 'active'],
default => [],

@ -0,0 +1,286 @@
<?php
namespace App\Services\Cloud;
use App\Models\CloudJob;
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
/**
* Coordinates one disposable host replica from the immutable master artifact.
*
* The coordinator never republishes a master object and never replays an
* ambiguous Agent operation. A failed replica is retryable only after its
* original job proves that no host effect occurred.
*/
final class CloudTemplateReplicaCoordinator
{
/**
* Ensure one eligible host receives the exact template generation.
*
* @return array{scheduled:bool,template_id:string,host_id:int,task_id?:string,status_url?:string}
*/
public static function syncHost(
string $templateId,
int $hostId,
array $actor,
string $reason = 'manual'
): array {
CloudAccess::manage($actor);
self::assertIdentity($templateId, $hostId, $reason);
$eligibleHostIds = array_values(array_unique(array_map(
'intval',
CloudAgentSettingsService::enabledHostIds()
)));
sort($eligibleHostIds, SORT_NUMERIC);
if (!in_array($hostId, $eligibleHostIds, true)) {
throw new CloudError('目标宿主机当前不能接收云镜像副本', 409);
}
$snapshot = CloudTemplate::find($templateId);
if ($snapshot === null || !in_array((string) $snapshot->state, [
'active', 'degraded', 'error',
], true)) {
throw new CloudError('云镜像主记录当前不能补齐宿主机副本', 409);
}
$source = CloudTemplateArtifactManager::downloadSource($snapshot);
$generation = (string) $snapshot->generation_token;
$revision = (int) $snapshot->revision;
$job = $snapshot->getConnection()->transaction(static function () use (
$templateId,
$hostId,
$actor,
$reason,
$eligibleHostIds,
$source,
$generation,
$revision
): ?CloudJob {
$template = CloudTemplateCatalogService::lockForReplicaSync(
$templateId
);
if ($template === null || !in_array((string) $template->state, [
'active', 'degraded', 'error',
], true)) {
throw new CloudError(
'云镜像主记录当前不能补齐宿主机副本',
409
);
}
if ((int) $template->revision !== $revision
|| !hash_equals(
(string) $template->generation_token,
$generation
)) {
throw new CloudError('云镜像版本已变化,请重新同步', 409);
}
$replica = CloudTemplateReplica::query()
->where('host_id', $hostId)
->where('template_id', $templateId)
->lockForUpdate()->first();
if ($replica !== null && self::isExact($template, $replica)) {
return null;
}
if ($replica !== null) {
if ((string) $replica->state !== 'failed') {
throw new CloudError(
'宿主机云镜像副本当前不能安全重新同步',
409
);
}
self::assertRetryableFailure($template, $replica);
}
$operationId = bin2hex(random_bytes(16));
$body = [
'id' => (string) $template->id,
'name' => (string) $template->name,
'source_url' => $source['source_url'],
'os_family' => (string) $template->os_family,
'version' => (string) $template->os_version,
'architecture' => 'x86_64',
'sha256' => (string) $source['sha256'],
'revision' => (int) $template->revision,
'generation_token' => (string) $template->generation_token,
'source_size_bytes' => (int) $source['source_size_bytes'],
];
$job = CloudJobService::reserveAgentMutation(
$actor,
$hostId,
'template_import_replica',
'POST',
'/v1/templates',
$body,
[
'can_manage' => true,
'template_id' => (string) $template->id,
'replica_sync' => true,
'replica_sync_reason' => $reason,
],
null,
$operationId
);
$now = time();
if ($replica === null) {
$replica = new CloudTemplateReplica();
$replica->host_id = $hostId;
$replica->template_id = (string) $template->id;
$replica->created_at = $now;
$replica->state = 'pending';
}
CloudTemplateStateMachine::moveReplica($replica, 'importing');
$replica->revision = (int) $template->revision;
$replica->generation_token =
(string) $template->generation_token;
$replica->image_bytes = 0;
$replica->sha256 = (string) $template->sha256;
$replica->operation_id = $operationId;
$replica->job_id = (int) $job->id;
$replica->error_code = '';
$replica->updated_at = $now;
$replica->save();
$replicas = CloudTemplateReplica::query()
->where('template_id', (string) $template->id)
->orderBy('host_id')->lockForUpdate()->get();
$availability = CloudTemplateStateMachine::availability(
$template,
$replicas,
$eligibleHostIds
);
CloudTemplateStateMachine::moveTemplate(
$template,
$availability
);
$template->active_job_id = (int) $job->id;
$template->updated_at = $now;
$template->save();
return $job;
}, 3);
if ($job === null) {
return [
'scheduled' => false,
'template_id' => $templateId,
'host_id' => $hostId,
];
}
return ['scheduled' => true, 'template_id' => $templateId,
'host_id' => $hostId] + CloudJobService::acknowledgement($job);
}
/** Publish one standalone replica result while its job and rows are locked. */
public static function settleStandaloneLocked(
CloudTemplate $template,
CloudTemplateReplica $replica,
CloudJob $job,
string $status,
array $operation
): void {
if ($job->parent_job_id !== null
|| (string) $job->kind !== 'template_import_replica'
|| (int) $template->active_job_id !== (int) $job->id
|| (int) $replica->job_id !== (int) $job->id) {
throw new CloudError('云镜像单宿主机同步账本不匹配', 409);
}
$settled = $status === 'succeeded'
|| (($operation['effect_may_have_applied'] ?? null) === false
&& ($operation['safe_to_resubmit'] ?? null) === true);
if (!$settled) {
return;
}
$eligibleHostIds = array_values(array_unique(array_map(
'intval',
CloudAgentSettingsService::enabledHostIds()
)));
sort($eligibleHostIds, SORT_NUMERIC);
$replicas = CloudTemplateReplica::query()
->where('template_id', (string) $template->id)
->orderBy('host_id')->lockForUpdate()->get();
$availability = CloudTemplateStateMachine::availability(
$template,
$replicas,
$eligibleHostIds
);
CloudTemplateStateMachine::moveTemplate($template, $availability);
if ($availability !== 'error') {
CloudTemplate::query()
->where('catalog_key', (string) $template->catalog_key)
->where('id', '<>', (string) $template->id)
->where('is_default', 1)
->update(['is_default' => 0, 'updated_at' => time()]);
$template->is_default = 1;
}
$template->active_job_id = null;
$template->updated_at = time();
$template->save();
}
private static function assertRetryableFailure(
CloudTemplate $template,
CloudTemplateReplica $replica
): void {
$job = $replica->job_id === null ? null
: CloudJob::query()->where('id', (int) $replica->job_id)
->lockForUpdate()->first();
if ($job === null
|| (string) $job->kind !== 'template_import_replica'
|| (int) $job->host_id !== (int) $replica->host_id
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| (string) $job->effect_resolution !== 'not_applied'
|| (bool) $job->effect_may_have_applied
|| !(bool) $job->safe_to_resubmit
|| (bool) $job->finalization_incident
|| CloudJobService::hasAgentRecoveryEnvelopeConflict($job)
|| !hash_equals(
(string) $job->operation_id,
(string) $replica->operation_id
)
|| !hash_equals(
(string) $template->generation_token,
(string) $replica->generation_token
)) {
throw new CloudError(
'宿主机副本失败结果尚未证明可安全重试',
409
);
}
}
private static function isExact(
CloudTemplate $template,
CloudTemplateReplica $replica
): bool {
return (string) $replica->state === 'active'
&& (int) $replica->revision === (int) $template->revision
&& (int) $replica->image_bytes === (int) $template->image_bytes
&& hash_equals(
(string) $template->generation_token,
(string) $replica->generation_token
)
&& hash_equals(
(string) $template->sha256,
(string) $replica->sha256
);
}
private static function assertIdentity(
string $templateId,
int $hostId,
string $reason
): void {
if (preg_match(
'/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D',
$templateId
) !== 1
|| $hostId < 1
|| preg_match('/^[a-z][a-z0-9_]{1,47}$/D', $reason) !== 1) {
throw new CloudError('云镜像宿主机同步参数无效', 422);
}
}
}

@ -0,0 +1,164 @@
<?php
namespace App\Services\Cloud;
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
/**
* The one executable state definition for template masters and host replicas.
*
* A template is available when at least one eligible host owns the exact
* immutable generation. Missing or failed replicas degrade availability; they
* do not rewrite the master artifact or make healthy replicas unusable.
*/
final class CloudTemplateStateMachine
{
/** @var array<string,list<string>> */
private const TEMPLATE_TRANSITIONS = [
'preparing' => ['importing', 'source_failed', 'source_unknown'],
'source_failed' => ['retired'],
'source_unknown' => ['retired'],
'importing' => ['active', 'degraded', 'error', 'retired'],
'active' => ['active', 'degraded', 'deleting'],
'degraded' => ['active', 'degraded', 'error', 'deleting', 'retired'],
'error' => ['active', 'degraded', 'error', 'deleting', 'retired'],
'deleting' => ['active', 'degraded', 'delete_failed', 'retired'],
'delete_failed' => ['active', 'degraded', 'deleting', 'retired'],
'retired' => [],
];
/** @var array<string,list<string>> */
private const REPLICA_TRANSITIONS = [
'pending' => ['importing'],
'importing' => ['active', 'failed'],
'failed' => ['importing', 'active', 'failed'],
'active' => ['active', 'syncing', 'deleting'],
'stale' => ['active', 'stale', 'syncing', 'deleting'],
'syncing' => ['active', 'stale'],
'deleting' => ['delete_succeeded', 'delete_failed'],
'delete_failed' => [
'active', 'stale', 'deleting', 'delete_failed', 'delete_succeeded',
],
'delete_succeeded' => [],
];
public static function moveTemplate(
CloudTemplate $template,
string $next
): void {
self::assertTransition(
self::TEMPLATE_TRANSITIONS,
(string) $template->state,
$next,
'云镜像主记录'
);
$template->state = $next;
}
public static function moveReplica(
CloudTemplateReplica $replica,
string $next
): void {
self::assertTransition(
self::REPLICA_TRANSITIONS,
(string) $replica->state,
$next,
'云镜像宿主机副本'
);
$replica->state = $next;
}
public static function assertTemplateTransition(
string $current,
string $next
): void {
self::assertTransition(
self::TEMPLATE_TRANSITIONS,
$current,
$next,
'云镜像主记录'
);
}
public static function assertReplicaTransition(
string $current,
string $next
): void {
self::assertTransition(
self::REPLICA_TRANSITIONS,
$current,
$next,
'云镜像宿主机副本'
);
}
/**
* Project catalog availability from exact replicas on currently eligible
* hosts. Replicas on disabled hosts do not make a template healthy.
*
* @param iterable<CloudTemplateReplica> $replicas
* @param list<int> $eligibleHostIds
*/
public static function availability(
CloudTemplate $template,
iterable $replicas,
array $eligibleHostIds
): string {
$eligible = [];
foreach ($eligibleHostIds as $hostId) {
$hostId = (int) $hostId;
if ($hostId > 0) {
$eligible[$hostId] = true;
}
}
if ($eligible === []) {
return 'error';
}
$ready = [];
foreach ($replicas as $replica) {
$hostId = (int) $replica->host_id;
if (!isset($eligible[$hostId])
|| (string) $replica->state !== 'active'
|| (int) $replica->revision !== (int) $template->revision
|| !hash_equals(
(string) $template->generation_token,
(string) $replica->generation_token
)
|| (int) $replica->image_bytes !== (int) $template->image_bytes
|| !hash_equals(
(string) $template->sha256,
(string) $replica->sha256
)) {
continue;
}
$ready[$hostId] = true;
}
if ($ready === []) {
return 'error';
}
return count($ready) === count($eligible) ? 'active' : 'degraded';
}
/** @param array<string,list<string>> $definition */
private static function assertTransition(
array $definition,
string $current,
string $next,
string $label
): void {
if (array_key_exists($current, $definition) && $current === $next) {
return;
}
if (!array_key_exists($current, $definition)
|| !in_array($next, $definition[$current], true)) {
throw new CloudError(sprintf(
'%s状态不能从 %s 变更为 %s',
$label,
$current === '' ? '(empty)' : $current,
$next === '' ? '(empty)' : $next
), 409);
}
}
}

@ -384,7 +384,7 @@ namespace {
$generation = md5('projection-generation:' . $parentId);
$sha256 = hash('sha256', 'projection-sha:' . $parentId);
$state = match ((string) $parent->kind) {
'template_import' => 'error',
'template_import' => 'degraded',
'template_metadata_sync' => 'active',
'template_delete' => 'delete_failed',
};
@ -589,7 +589,7 @@ namespace {
&& $partial->lease_token === null
&& (int) $partial->available_at === 0
&& $partialTemplate !== null
&& (string) $partialTemplate->state === 'error'
&& (string) $partialTemplate->state === 'degraded'
&& $partialTemplate->active_job_id === null
&& $partialReplicaStates === ['active', 'failed']
&& is_array($partialTemplateDto)
@ -2995,7 +2995,7 @@ namespace {
->where('id', $legacyPartialResourceId)->first();
$expectedState = str_ends_with($parentKind, '_import')
? (str_starts_with($parentKind, 'template_')
? 'error' : 'import_failed')
? 'degraded' : 'import_failed')
: 'delete_failed';
$assert((string) $legacyPartial->effect_resolution === 'applied'
&& str_ends_with(

@ -0,0 +1,57 @@
<?php
$root = dirname(__DIR__);
$coordinator = file_get_contents(
$root . '/app/Services/Cloud/CloudTemplateReplicaCoordinator.php'
);
$catalog = file_get_contents(
$root . '/app/Services/Cloud/CloudTemplateCatalogService.php'
);
$identity = file_get_contents(
$root . '/app/Services/Cloud/CloudOperationRecoveryIdentity.php'
);
$automatic = file_get_contents(
$root . '/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php'
);
$agent = file_get_contents(
dirname($root) . '/agent/hypervisor/libvirt_catalog_features.go'
);
if (!is_string($coordinator) || !is_string($catalog)
|| !is_string($identity) || !is_string($automatic)
|| !is_string($agent)) {
throw new RuntimeException('Cannot read template replica sources.');
}
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$assert(str_contains($coordinator,
'CloudTemplateArtifactManager::downloadSource($snapshot)')
&& str_contains($coordinator,
"'template_import_replica',")
&& str_contains($coordinator,
"'replica_sync' => true")
&& str_contains($coordinator,
"\$job->effect_resolution !== 'not_applied'")
&& str_contains($coordinator,
'CloudTemplateStateMachine::availability(')
&& str_contains($catalog,
'CloudTemplateReplicaCoordinator::settleStandaloneLocked('),
'Standalone replica synchronization is not bound to the immutable artifact and safe retry evidence.');
$assert(str_contains($identity,
'$standaloneImport = (string) $job->kind')
&& str_contains($identity,
"&& (\$payload['replica_sync'] ?? null) === true")
&& str_contains($identity,
'? (int) $job->id : (int) $parent->id)'),
'Recovery identity cannot distinguish a standalone replica sync from a damaged aggregate child.');
$assert(str_contains($automatic,
"&& (string) \$job->execution_mode === 'aggregate'")
&& !str_contains($agent, '"template_name_exists"'),
'Standalone repair is misclassified as aggregate damage or template versions still conflict by display name.');
echo "Template standalone replica coordinator contract passed.\n";

@ -0,0 +1,77 @@
<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudTemplateStateMachine;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$template = new CloudTemplate();
$template->state = 'preparing';
$template->revision = 3;
$template->generation_token = str_repeat('a', 32);
$template->image_bytes = 4096;
$template->sha256 = str_repeat('b', 64);
CloudTemplateStateMachine::moveTemplate($template, 'importing');
CloudTemplateStateMachine::moveTemplate($template, 'degraded');
$assert((string) $template->state === 'degraded',
'Template state machine did not permit a partial initial import.');
$invalidRejected = false;
try {
CloudTemplateStateMachine::moveTemplate($template, 'source_failed');
} catch (CloudError $error) {
$invalidRejected = $error->httpStatus === 409;
}
$assert($invalidRejected,
'Template state machine accepted an impossible source transition.');
$ready = new CloudTemplateReplica();
$ready->host_id = 11;
$ready->state = 'pending';
$ready->revision = 3;
$ready->generation_token = str_repeat('a', 32);
$ready->image_bytes = 4096;
$ready->sha256 = str_repeat('b', 64);
CloudTemplateStateMachine::moveReplica($ready, 'importing');
CloudTemplateStateMachine::moveReplica($ready, 'active');
$failed = new CloudTemplateReplica();
$failed->host_id = 12;
$failed->state = 'pending';
$failed->revision = 3;
$failed->generation_token = str_repeat('a', 32);
$failed->image_bytes = 0;
$failed->sha256 = str_repeat('b', 64);
CloudTemplateStateMachine::moveReplica($failed, 'importing');
CloudTemplateStateMachine::moveReplica($failed, 'failed');
$assert(CloudTemplateStateMachine::availability(
$template, [$ready, $failed], [11, 12]
) === 'degraded',
'One healthy and one failed eligible replica was not degraded.');
$assert(CloudTemplateStateMachine::availability(
$template, [$ready, $failed], [11]
) === 'active',
'A complete eligible-host projection was not active.');
$assert(CloudTemplateStateMachine::availability(
$template, [$failed], [12]
) === 'error',
'A catalog without any exact healthy replica was exposed as available.');
CloudTemplateStateMachine::moveReplica($failed, 'importing');
CloudTemplateStateMachine::moveReplica($failed, 'active');
$failed->image_bytes = 4096;
$assert(CloudTemplateStateMachine::availability(
$template, [$ready, $failed], [11, 12]
) === 'active',
'A safely retried exact replica did not restore full availability.');
echo "Template replica state-machine behavior passed.\n";

@ -0,0 +1,199 @@
<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudJob;
use App\Models\CloudTemplate;
use App\Models\CloudTemplateReplica;
use App\Services\Cloud\CloudTemplateCatalogService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('cloud_hosts', static function (Blueprint $table): void {
$table->integer('id')->primary();
$table->boolean('enabled');
$table->string('default_network');
});
$schema->create('cloud_host_agents', static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->boolean('enabled');
$table->string('last_test_status');
$table->integer('tested_revision');
$table->integer('revision');
});
$schema->create('cloud_host_storage_configurations',
static function (Blueprint $table): void {
$table->integer('host_id')->primary();
$table->string('state');
});
$schema->create('cloud_templates', static function (Blueprint $table): void {
$table->string('id')->primary();
$table->string('catalog_key');
$table->string('version_label');
$table->boolean('is_default');
$table->string('name');
$table->string('os_family');
$table->string('os_version');
$table->string('source_type');
$table->text('sealed_source_url');
$table->integer('image_bytes');
$table->string('sha256');
$table->text('startup_script');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('active_job_id')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_template_replicas',
static function (Blueprint $table): void {
$table->integer('host_id');
$table->string('template_id');
$table->integer('revision');
$table->string('generation_token');
$table->string('state');
$table->integer('image_bytes');
$table->string('sha256');
$table->string('operation_id');
$table->integer('job_id')->nullable();
$table->string('error_code');
$table->integer('created_at');
$table->integer('updated_at');
$table->primary(['host_id', 'template_id']);
});
foreach ([31, 32] as $hostId) {
Capsule::table('cloud_hosts')->insert([
'id' => $hostId, 'enabled' => 1,
'default_network' => 'bridge:br0',
]);
Capsule::table('cloud_host_agents')->insert([
'host_id' => $hostId, 'enabled' => 1,
'last_test_status' => 'healthy', 'tested_revision' => 4,
'revision' => 4,
]);
Capsule::table('cloud_host_storage_configurations')->insert([
'host_id' => $hostId, 'state' => 'applied',
]);
}
$now = time();
$generation = str_repeat('a', 32);
$sha = str_repeat('b', 64);
Capsule::table('cloud_templates')->insert([
'id' => 'standalone-replica-template',
'catalog_key' => 'debian-13',
'version_label' => 'v2',
'is_default' => 1,
'name' => 'Debian 13',
'os_family' => 'Debian',
'os_version' => '13',
'source_type' => 'object',
'sealed_source_url' => 'test-only',
'image_bytes' => 4096,
'sha256' => $sha,
'startup_script' => '',
'revision' => 2,
'generation_token' => $generation,
'state' => 'degraded',
'active_job_id' => 501,
'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_template_replicas')->insert([
[
'host_id' => 31, 'template_id' => 'standalone-replica-template',
'revision' => 2, 'generation_token' => $generation,
'state' => 'active', 'image_bytes' => 4096, 'sha256' => $sha,
'operation_id' => str_repeat('1', 32), 'job_id' => 500,
'error_code' => '', 'created_at' => $now, 'updated_at' => $now,
],
[
'host_id' => 32, 'template_id' => 'standalone-replica-template',
'revision' => 2, 'generation_token' => $generation,
'state' => 'importing', 'image_bytes' => 0, 'sha256' => $sha,
'operation_id' => str_repeat('2', 32), 'job_id' => 501,
'error_code' => '', 'created_at' => $now, 'updated_at' => $now,
],
]);
$job = new CloudJob();
$job->id = 501;
$job->host_id = 32;
$job->kind = 'template_import_replica';
$job->parent_job_id = null;
$job->operation_id = str_repeat('2', 32);
$payload = ['template_id' => 'standalone-replica-template'];
$result = [
'id' => 'standalone-replica-template',
'revision' => 2,
'generation_token' => $generation,
'size_bytes' => 4096,
'sha256' => $sha,
];
CloudTemplateCatalogService::finishJobLocked(
$job,
'succeeded',
$result,
$payload,
['effect_may_have_applied' => false, 'safe_to_resubmit' => false]
);
$template = CloudTemplate::findOrFail('standalone-replica-template');
$assert((string) $template->state === 'active'
&& $template->active_job_id === null
&& (string) CloudTemplateReplica::query()->where('host_id', 32)
->where('template_id', (string) $template->id)->value('state')
=== 'active',
'An exact standalone replica receipt did not restore full availability.');
Capsule::table('cloud_templates')->where('id', (string) $template->id)
->update(['state' => 'degraded', 'active_job_id' => 502]);
Capsule::table('cloud_template_replicas')->where('host_id', 32)
->where('template_id', (string) $template->id)->update([
'state' => 'importing', 'image_bytes' => 0,
'job_id' => 502, 'operation_id' => str_repeat('3', 32),
]);
$job->id = 502;
$job->operation_id = str_repeat('3', 32);
CloudTemplateCatalogService::finishJobLocked(
$job,
'failed',
null,
$payload,
[
'effect_may_have_applied' => true,
'safe_to_resubmit' => false,
'error_code' => 'agent_timeout',
]
);
$template->refresh();
$assert((int) $template->active_job_id === 502
&& (string) $template->state === 'degraded',
'An unknown standalone replica effect released the template fence.');
CloudTemplateCatalogService::finishJobLocked(
$job,
'succeeded',
$result,
$payload,
['effect_may_have_applied' => false, 'safe_to_resubmit' => false],
true
);
$template->refresh();
$assert($template->active_job_id === null
&& (string) $template->state === 'active',
'Recovered standalone replica evidence did not release the original fence.');
echo "Template standalone replica finalization behavior passed.\n";

@ -83,6 +83,8 @@ $adminBillingStyles = $public . '/admin-billing.css';
$adminSharedStyles = file_get_contents($root . '/public/theme/venix/css/admin-home.css');
$templateCatalogService = file_get_contents(
$root . '/app/Services/Cloud/CloudTemplateCatalogService.php');
$templateStateMachine = file_get_contents(
$root . '/app/Services/Cloud/CloudTemplateStateMachine.php');
$assert(str_contains($cloudShell, "file='user/panel_sidebar.tpl'")
&& str_contains($userLayout, "file='user/panel_sidebar.tpl'"), 'Cloud and account pages must share one sidebar partial');
$assert(str_contains($cloudShell, "file='user/panel_topbar.tpl'")
@ -711,7 +713,10 @@ $assert(str_contains($cloudSchema, 'CREATE TABLE cloud_templates')
&& !str_contains($cloudSchema, 'cloud_template_retirements')
&& !str_contains($cloudSchema, 'cloud_template_catalog_locks')
&& str_contains($templateCatalogService, 'lockForUpdate()')
&& str_contains($templateCatalogService, '$template->state = \'retired\'')
&& str_contains($templateCatalogService,
"CloudTemplateStateMachine::moveTemplate(\$template, 'retired')")
&& is_string($templateStateMachine)
&& str_contains($templateStateMachine, 'TEMPLATE_TRANSITIONS')
&& str_contains($deploymentPageService, 'CloudTemplateCatalogService::publicCatalog()'),
'Template catalog must use the fresh PHP schema and CAS retirement boundary.');
$assert(str_contains($clientScript,

Loading…
Cancel
Save