feat(template): coordinate replica lifecycle
parent
59c3586e3a
commit
d166b56aa4
@ -0,0 +1,286 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudJob;
|
||||
use App\Models\CloudTemplate;
|
||||
use App\Models\CloudTemplateReplica;
|
||||
|
||||
/**
|
||||
* Coordinates one disposable host replica from the immutable master artifact.
|
||||
*
|
||||
* The coordinator never republishes a master object and never replays an
|
||||
* ambiguous Agent operation. A failed replica is retryable only after its
|
||||
* original job proves that no host effect occurred.
|
||||
*/
|
||||
final class CloudTemplateReplicaCoordinator
|
||||
{
|
||||
/**
|
||||
* Ensure one eligible host receives the exact template generation.
|
||||
*
|
||||
* @return array{scheduled:bool,template_id:string,host_id:int,task_id?:string,status_url?:string}
|
||||
*/
|
||||
public static function syncHost(
|
||||
string $templateId,
|
||||
int $hostId,
|
||||
array $actor,
|
||||
string $reason = 'manual'
|
||||
): array {
|
||||
CloudAccess::manage($actor);
|
||||
self::assertIdentity($templateId, $hostId, $reason);
|
||||
$eligibleHostIds = array_values(array_unique(array_map(
|
||||
'intval',
|
||||
CloudAgentSettingsService::enabledHostIds()
|
||||
)));
|
||||
sort($eligibleHostIds, SORT_NUMERIC);
|
||||
if (!in_array($hostId, $eligibleHostIds, true)) {
|
||||
throw new CloudError('目标宿主机当前不能接收云镜像副本', 409);
|
||||
}
|
||||
|
||||
$snapshot = CloudTemplate::find($templateId);
|
||||
if ($snapshot === null || !in_array((string) $snapshot->state, [
|
||||
'active', 'degraded', 'error',
|
||||
], true)) {
|
||||
throw new CloudError('云镜像主记录当前不能补齐宿主机副本', 409);
|
||||
}
|
||||
$source = CloudTemplateArtifactManager::downloadSource($snapshot);
|
||||
$generation = (string) $snapshot->generation_token;
|
||||
$revision = (int) $snapshot->revision;
|
||||
|
||||
$job = $snapshot->getConnection()->transaction(static function () use (
|
||||
$templateId,
|
||||
$hostId,
|
||||
$actor,
|
||||
$reason,
|
||||
$eligibleHostIds,
|
||||
$source,
|
||||
$generation,
|
||||
$revision
|
||||
): ?CloudJob {
|
||||
$template = CloudTemplateCatalogService::lockForReplicaSync(
|
||||
$templateId
|
||||
);
|
||||
if ($template === null || !in_array((string) $template->state, [
|
||||
'active', 'degraded', 'error',
|
||||
], true)) {
|
||||
throw new CloudError(
|
||||
'云镜像主记录当前不能补齐宿主机副本',
|
||||
409
|
||||
);
|
||||
}
|
||||
if ((int) $template->revision !== $revision
|
||||
|| !hash_equals(
|
||||
(string) $template->generation_token,
|
||||
$generation
|
||||
)) {
|
||||
throw new CloudError('云镜像版本已变化,请重新同步', 409);
|
||||
}
|
||||
|
||||
$replica = CloudTemplateReplica::query()
|
||||
->where('host_id', $hostId)
|
||||
->where('template_id', $templateId)
|
||||
->lockForUpdate()->first();
|
||||
if ($replica !== null && self::isExact($template, $replica)) {
|
||||
return null;
|
||||
}
|
||||
if ($replica !== null) {
|
||||
if ((string) $replica->state !== 'failed') {
|
||||
throw new CloudError(
|
||||
'宿主机云镜像副本当前不能安全重新同步',
|
||||
409
|
||||
);
|
||||
}
|
||||
self::assertRetryableFailure($template, $replica);
|
||||
}
|
||||
|
||||
$operationId = bin2hex(random_bytes(16));
|
||||
$body = [
|
||||
'id' => (string) $template->id,
|
||||
'name' => (string) $template->name,
|
||||
'source_url' => $source['source_url'],
|
||||
'os_family' => (string) $template->os_family,
|
||||
'version' => (string) $template->os_version,
|
||||
'architecture' => 'x86_64',
|
||||
'sha256' => (string) $source['sha256'],
|
||||
'revision' => (int) $template->revision,
|
||||
'generation_token' => (string) $template->generation_token,
|
||||
'source_size_bytes' => (int) $source['source_size_bytes'],
|
||||
];
|
||||
$job = CloudJobService::reserveAgentMutation(
|
||||
$actor,
|
||||
$hostId,
|
||||
'template_import_replica',
|
||||
'POST',
|
||||
'/v1/templates',
|
||||
$body,
|
||||
[
|
||||
'can_manage' => true,
|
||||
'template_id' => (string) $template->id,
|
||||
'replica_sync' => true,
|
||||
'replica_sync_reason' => $reason,
|
||||
],
|
||||
null,
|
||||
$operationId
|
||||
);
|
||||
$now = time();
|
||||
if ($replica === null) {
|
||||
$replica = new CloudTemplateReplica();
|
||||
$replica->host_id = $hostId;
|
||||
$replica->template_id = (string) $template->id;
|
||||
$replica->created_at = $now;
|
||||
$replica->state = 'pending';
|
||||
}
|
||||
CloudTemplateStateMachine::moveReplica($replica, 'importing');
|
||||
$replica->revision = (int) $template->revision;
|
||||
$replica->generation_token =
|
||||
(string) $template->generation_token;
|
||||
$replica->image_bytes = 0;
|
||||
$replica->sha256 = (string) $template->sha256;
|
||||
$replica->operation_id = $operationId;
|
||||
$replica->job_id = (int) $job->id;
|
||||
$replica->error_code = '';
|
||||
$replica->updated_at = $now;
|
||||
$replica->save();
|
||||
|
||||
$replicas = CloudTemplateReplica::query()
|
||||
->where('template_id', (string) $template->id)
|
||||
->orderBy('host_id')->lockForUpdate()->get();
|
||||
$availability = CloudTemplateStateMachine::availability(
|
||||
$template,
|
||||
$replicas,
|
||||
$eligibleHostIds
|
||||
);
|
||||
CloudTemplateStateMachine::moveTemplate(
|
||||
$template,
|
||||
$availability
|
||||
);
|
||||
$template->active_job_id = (int) $job->id;
|
||||
$template->updated_at = $now;
|
||||
$template->save();
|
||||
return $job;
|
||||
}, 3);
|
||||
|
||||
if ($job === null) {
|
||||
return [
|
||||
'scheduled' => false,
|
||||
'template_id' => $templateId,
|
||||
'host_id' => $hostId,
|
||||
];
|
||||
}
|
||||
return ['scheduled' => true, 'template_id' => $templateId,
|
||||
'host_id' => $hostId] + CloudJobService::acknowledgement($job);
|
||||
}
|
||||
|
||||
/** Publish one standalone replica result while its job and rows are locked. */
|
||||
public static function settleStandaloneLocked(
|
||||
CloudTemplate $template,
|
||||
CloudTemplateReplica $replica,
|
||||
CloudJob $job,
|
||||
string $status,
|
||||
array $operation
|
||||
): void {
|
||||
if ($job->parent_job_id !== null
|
||||
|| (string) $job->kind !== 'template_import_replica'
|
||||
|| (int) $template->active_job_id !== (int) $job->id
|
||||
|| (int) $replica->job_id !== (int) $job->id) {
|
||||
throw new CloudError('云镜像单宿主机同步账本不匹配', 409);
|
||||
}
|
||||
$settled = $status === 'succeeded'
|
||||
|| (($operation['effect_may_have_applied'] ?? null) === false
|
||||
&& ($operation['safe_to_resubmit'] ?? null) === true);
|
||||
if (!$settled) {
|
||||
return;
|
||||
}
|
||||
$eligibleHostIds = array_values(array_unique(array_map(
|
||||
'intval',
|
||||
CloudAgentSettingsService::enabledHostIds()
|
||||
)));
|
||||
sort($eligibleHostIds, SORT_NUMERIC);
|
||||
$replicas = CloudTemplateReplica::query()
|
||||
->where('template_id', (string) $template->id)
|
||||
->orderBy('host_id')->lockForUpdate()->get();
|
||||
$availability = CloudTemplateStateMachine::availability(
|
||||
$template,
|
||||
$replicas,
|
||||
$eligibleHostIds
|
||||
);
|
||||
CloudTemplateStateMachine::moveTemplate($template, $availability);
|
||||
if ($availability !== 'error') {
|
||||
CloudTemplate::query()
|
||||
->where('catalog_key', (string) $template->catalog_key)
|
||||
->where('id', '<>', (string) $template->id)
|
||||
->where('is_default', 1)
|
||||
->update(['is_default' => 0, 'updated_at' => time()]);
|
||||
$template->is_default = 1;
|
||||
}
|
||||
$template->active_job_id = null;
|
||||
$template->updated_at = time();
|
||||
$template->save();
|
||||
}
|
||||
|
||||
private static function assertRetryableFailure(
|
||||
CloudTemplate $template,
|
||||
CloudTemplateReplica $replica
|
||||
): void {
|
||||
$job = $replica->job_id === null ? null
|
||||
: CloudJob::query()->where('id', (int) $replica->job_id)
|
||||
->lockForUpdate()->first();
|
||||
if ($job === null
|
||||
|| (string) $job->kind !== 'template_import_replica'
|
||||
|| (int) $job->host_id !== (int) $replica->host_id
|
||||
|| (string) $job->execution_mode !== 'agent'
|
||||
|| (string) $job->phase !== 'terminal'
|
||||
|| (string) $job->outcome !== 'failed'
|
||||
|| (string) $job->status !== 'failed'
|
||||
|| (string) $job->effect_resolution !== 'not_applied'
|
||||
|| (bool) $job->effect_may_have_applied
|
||||
|| !(bool) $job->safe_to_resubmit
|
||||
|| (bool) $job->finalization_incident
|
||||
|| CloudJobService::hasAgentRecoveryEnvelopeConflict($job)
|
||||
|| !hash_equals(
|
||||
(string) $job->operation_id,
|
||||
(string) $replica->operation_id
|
||||
)
|
||||
|| !hash_equals(
|
||||
(string) $template->generation_token,
|
||||
(string) $replica->generation_token
|
||||
)) {
|
||||
throw new CloudError(
|
||||
'宿主机副本失败结果尚未证明可安全重试',
|
||||
409
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private static function isExact(
|
||||
CloudTemplate $template,
|
||||
CloudTemplateReplica $replica
|
||||
): bool {
|
||||
return (string) $replica->state === 'active'
|
||||
&& (int) $replica->revision === (int) $template->revision
|
||||
&& (int) $replica->image_bytes === (int) $template->image_bytes
|
||||
&& hash_equals(
|
||||
(string) $template->generation_token,
|
||||
(string) $replica->generation_token
|
||||
)
|
||||
&& hash_equals(
|
||||
(string) $template->sha256,
|
||||
(string) $replica->sha256
|
||||
);
|
||||
}
|
||||
|
||||
private static function assertIdentity(
|
||||
string $templateId,
|
||||
int $hostId,
|
||||
string $reason
|
||||
): void {
|
||||
if (preg_match(
|
||||
'/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D',
|
||||
$templateId
|
||||
) !== 1
|
||||
|| $hostId < 1
|
||||
|| preg_match('/^[a-z][a-z0-9_]{1,47}$/D', $reason) !== 1) {
|
||||
throw new CloudError('云镜像宿主机同步参数无效', 422);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,164 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudTemplate;
|
||||
use App\Models\CloudTemplateReplica;
|
||||
|
||||
/**
|
||||
* The one executable state definition for template masters and host replicas.
|
||||
*
|
||||
* A template is available when at least one eligible host owns the exact
|
||||
* immutable generation. Missing or failed replicas degrade availability; they
|
||||
* do not rewrite the master artifact or make healthy replicas unusable.
|
||||
*/
|
||||
final class CloudTemplateStateMachine
|
||||
{
|
||||
/** @var array<string,list<string>> */
|
||||
private const TEMPLATE_TRANSITIONS = [
|
||||
'preparing' => ['importing', 'source_failed', 'source_unknown'],
|
||||
'source_failed' => ['retired'],
|
||||
'source_unknown' => ['retired'],
|
||||
'importing' => ['active', 'degraded', 'error', 'retired'],
|
||||
'active' => ['active', 'degraded', 'deleting'],
|
||||
'degraded' => ['active', 'degraded', 'error', 'deleting', 'retired'],
|
||||
'error' => ['active', 'degraded', 'error', 'deleting', 'retired'],
|
||||
'deleting' => ['active', 'degraded', 'delete_failed', 'retired'],
|
||||
'delete_failed' => ['active', 'degraded', 'deleting', 'retired'],
|
||||
'retired' => [],
|
||||
];
|
||||
|
||||
/** @var array<string,list<string>> */
|
||||
private const REPLICA_TRANSITIONS = [
|
||||
'pending' => ['importing'],
|
||||
'importing' => ['active', 'failed'],
|
||||
'failed' => ['importing', 'active', 'failed'],
|
||||
'active' => ['active', 'syncing', 'deleting'],
|
||||
'stale' => ['active', 'stale', 'syncing', 'deleting'],
|
||||
'syncing' => ['active', 'stale'],
|
||||
'deleting' => ['delete_succeeded', 'delete_failed'],
|
||||
'delete_failed' => [
|
||||
'active', 'stale', 'deleting', 'delete_failed', 'delete_succeeded',
|
||||
],
|
||||
'delete_succeeded' => [],
|
||||
];
|
||||
|
||||
public static function moveTemplate(
|
||||
CloudTemplate $template,
|
||||
string $next
|
||||
): void {
|
||||
self::assertTransition(
|
||||
self::TEMPLATE_TRANSITIONS,
|
||||
(string) $template->state,
|
||||
$next,
|
||||
'云镜像主记录'
|
||||
);
|
||||
$template->state = $next;
|
||||
}
|
||||
|
||||
public static function moveReplica(
|
||||
CloudTemplateReplica $replica,
|
||||
string $next
|
||||
): void {
|
||||
self::assertTransition(
|
||||
self::REPLICA_TRANSITIONS,
|
||||
(string) $replica->state,
|
||||
$next,
|
||||
'云镜像宿主机副本'
|
||||
);
|
||||
$replica->state = $next;
|
||||
}
|
||||
|
||||
public static function assertTemplateTransition(
|
||||
string $current,
|
||||
string $next
|
||||
): void {
|
||||
self::assertTransition(
|
||||
self::TEMPLATE_TRANSITIONS,
|
||||
$current,
|
||||
$next,
|
||||
'云镜像主记录'
|
||||
);
|
||||
}
|
||||
|
||||
public static function assertReplicaTransition(
|
||||
string $current,
|
||||
string $next
|
||||
): void {
|
||||
self::assertTransition(
|
||||
self::REPLICA_TRANSITIONS,
|
||||
$current,
|
||||
$next,
|
||||
'云镜像宿主机副本'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Project catalog availability from exact replicas on currently eligible
|
||||
* hosts. Replicas on disabled hosts do not make a template healthy.
|
||||
*
|
||||
* @param iterable<CloudTemplateReplica> $replicas
|
||||
* @param list<int> $eligibleHostIds
|
||||
*/
|
||||
public static function availability(
|
||||
CloudTemplate $template,
|
||||
iterable $replicas,
|
||||
array $eligibleHostIds
|
||||
): string {
|
||||
$eligible = [];
|
||||
foreach ($eligibleHostIds as $hostId) {
|
||||
$hostId = (int) $hostId;
|
||||
if ($hostId > 0) {
|
||||
$eligible[$hostId] = true;
|
||||
}
|
||||
}
|
||||
if ($eligible === []) {
|
||||
return 'error';
|
||||
}
|
||||
|
||||
$ready = [];
|
||||
foreach ($replicas as $replica) {
|
||||
$hostId = (int) $replica->host_id;
|
||||
if (!isset($eligible[$hostId])
|
||||
|| (string) $replica->state !== 'active'
|
||||
|| (int) $replica->revision !== (int) $template->revision
|
||||
|| !hash_equals(
|
||||
(string) $template->generation_token,
|
||||
(string) $replica->generation_token
|
||||
)
|
||||
|| (int) $replica->image_bytes !== (int) $template->image_bytes
|
||||
|| !hash_equals(
|
||||
(string) $template->sha256,
|
||||
(string) $replica->sha256
|
||||
)) {
|
||||
continue;
|
||||
}
|
||||
$ready[$hostId] = true;
|
||||
}
|
||||
if ($ready === []) {
|
||||
return 'error';
|
||||
}
|
||||
return count($ready) === count($eligible) ? 'active' : 'degraded';
|
||||
}
|
||||
|
||||
/** @param array<string,list<string>> $definition */
|
||||
private static function assertTransition(
|
||||
array $definition,
|
||||
string $current,
|
||||
string $next,
|
||||
string $label
|
||||
): void {
|
||||
if (array_key_exists($current, $definition) && $current === $next) {
|
||||
return;
|
||||
}
|
||||
if (!array_key_exists($current, $definition)
|
||||
|| !in_array($next, $definition[$current], true)) {
|
||||
throw new CloudError(sprintf(
|
||||
'%s状态不能从 %s 变更为 %s',
|
||||
$label,
|
||||
$current === '' ? '(empty)' : $current,
|
||||
$next === '' ? '(empty)' : $next
|
||||
), 409);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
<?php
|
||||
|
||||
$root = dirname(__DIR__);
|
||||
$coordinator = file_get_contents(
|
||||
$root . '/app/Services/Cloud/CloudTemplateReplicaCoordinator.php'
|
||||
);
|
||||
$catalog = file_get_contents(
|
||||
$root . '/app/Services/Cloud/CloudTemplateCatalogService.php'
|
||||
);
|
||||
$identity = file_get_contents(
|
||||
$root . '/app/Services/Cloud/CloudOperationRecoveryIdentity.php'
|
||||
);
|
||||
$automatic = file_get_contents(
|
||||
$root . '/app/Services/Cloud/CloudAutomaticOperationRecoveryService.php'
|
||||
);
|
||||
$agent = file_get_contents(
|
||||
dirname($root) . '/agent/hypervisor/libvirt_catalog_features.go'
|
||||
);
|
||||
if (!is_string($coordinator) || !is_string($catalog)
|
||||
|| !is_string($identity) || !is_string($automatic)
|
||||
|| !is_string($agent)) {
|
||||
throw new RuntimeException('Cannot read template replica sources.');
|
||||
}
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$assert(str_contains($coordinator,
|
||||
'CloudTemplateArtifactManager::downloadSource($snapshot)')
|
||||
&& str_contains($coordinator,
|
||||
"'template_import_replica',")
|
||||
&& str_contains($coordinator,
|
||||
"'replica_sync' => true")
|
||||
&& str_contains($coordinator,
|
||||
"\$job->effect_resolution !== 'not_applied'")
|
||||
&& str_contains($coordinator,
|
||||
'CloudTemplateStateMachine::availability(')
|
||||
&& str_contains($catalog,
|
||||
'CloudTemplateReplicaCoordinator::settleStandaloneLocked('),
|
||||
'Standalone replica synchronization is not bound to the immutable artifact and safe retry evidence.');
|
||||
|
||||
$assert(str_contains($identity,
|
||||
'$standaloneImport = (string) $job->kind')
|
||||
&& str_contains($identity,
|
||||
"&& (\$payload['replica_sync'] ?? null) === true")
|
||||
&& str_contains($identity,
|
||||
'? (int) $job->id : (int) $parent->id)'),
|
||||
'Recovery identity cannot distinguish a standalone replica sync from a damaged aggregate child.');
|
||||
|
||||
$assert(str_contains($automatic,
|
||||
"&& (string) \$job->execution_mode === 'aggregate'")
|
||||
&& !str_contains($agent, '"template_name_exists"'),
|
||||
'Standalone repair is misclassified as aggregate damage or template versions still conflict by display name.');
|
||||
|
||||
echo "Template standalone replica coordinator contract passed.\n";
|
||||
@ -0,0 +1,77 @@
|
||||
<?php
|
||||
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudTemplate;
|
||||
use App\Models\CloudTemplateReplica;
|
||||
use App\Services\Cloud\CloudError;
|
||||
use App\Services\Cloud\CloudTemplateStateMachine;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
|
||||
$template = new CloudTemplate();
|
||||
$template->state = 'preparing';
|
||||
$template->revision = 3;
|
||||
$template->generation_token = str_repeat('a', 32);
|
||||
$template->image_bytes = 4096;
|
||||
$template->sha256 = str_repeat('b', 64);
|
||||
CloudTemplateStateMachine::moveTemplate($template, 'importing');
|
||||
CloudTemplateStateMachine::moveTemplate($template, 'degraded');
|
||||
$assert((string) $template->state === 'degraded',
|
||||
'Template state machine did not permit a partial initial import.');
|
||||
|
||||
$invalidRejected = false;
|
||||
try {
|
||||
CloudTemplateStateMachine::moveTemplate($template, 'source_failed');
|
||||
} catch (CloudError $error) {
|
||||
$invalidRejected = $error->httpStatus === 409;
|
||||
}
|
||||
$assert($invalidRejected,
|
||||
'Template state machine accepted an impossible source transition.');
|
||||
|
||||
$ready = new CloudTemplateReplica();
|
||||
$ready->host_id = 11;
|
||||
$ready->state = 'pending';
|
||||
$ready->revision = 3;
|
||||
$ready->generation_token = str_repeat('a', 32);
|
||||
$ready->image_bytes = 4096;
|
||||
$ready->sha256 = str_repeat('b', 64);
|
||||
CloudTemplateStateMachine::moveReplica($ready, 'importing');
|
||||
CloudTemplateStateMachine::moveReplica($ready, 'active');
|
||||
|
||||
$failed = new CloudTemplateReplica();
|
||||
$failed->host_id = 12;
|
||||
$failed->state = 'pending';
|
||||
$failed->revision = 3;
|
||||
$failed->generation_token = str_repeat('a', 32);
|
||||
$failed->image_bytes = 0;
|
||||
$failed->sha256 = str_repeat('b', 64);
|
||||
CloudTemplateStateMachine::moveReplica($failed, 'importing');
|
||||
CloudTemplateStateMachine::moveReplica($failed, 'failed');
|
||||
|
||||
$assert(CloudTemplateStateMachine::availability(
|
||||
$template, [$ready, $failed], [11, 12]
|
||||
) === 'degraded',
|
||||
'One healthy and one failed eligible replica was not degraded.');
|
||||
$assert(CloudTemplateStateMachine::availability(
|
||||
$template, [$ready, $failed], [11]
|
||||
) === 'active',
|
||||
'A complete eligible-host projection was not active.');
|
||||
$assert(CloudTemplateStateMachine::availability(
|
||||
$template, [$failed], [12]
|
||||
) === 'error',
|
||||
'A catalog without any exact healthy replica was exposed as available.');
|
||||
|
||||
CloudTemplateStateMachine::moveReplica($failed, 'importing');
|
||||
CloudTemplateStateMachine::moveReplica($failed, 'active');
|
||||
$failed->image_bytes = 4096;
|
||||
$assert(CloudTemplateStateMachine::availability(
|
||||
$template, [$ready, $failed], [11, 12]
|
||||
) === 'active',
|
||||
'A safely retried exact replica did not restore full availability.');
|
||||
|
||||
echo "Template replica state-machine behavior passed.\n";
|
||||
@ -0,0 +1,199 @@
|
||||
<?php
|
||||
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudJob;
|
||||
use App\Models\CloudTemplate;
|
||||
use App\Models\CloudTemplateReplica;
|
||||
use App\Services\Cloud\CloudTemplateCatalogService;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
$capsule = new Capsule();
|
||||
$capsule->addConnection([
|
||||
'driver' => 'sqlite', 'database' => ':memory:',
|
||||
], 'default');
|
||||
$capsule->setAsGlobal();
|
||||
$capsule->bootEloquent();
|
||||
$schema = $capsule->schema('default');
|
||||
$schema->create('cloud_hosts', static function (Blueprint $table): void {
|
||||
$table->integer('id')->primary();
|
||||
$table->boolean('enabled');
|
||||
$table->string('default_network');
|
||||
});
|
||||
$schema->create('cloud_host_agents', static function (Blueprint $table): void {
|
||||
$table->integer('host_id')->primary();
|
||||
$table->boolean('enabled');
|
||||
$table->string('last_test_status');
|
||||
$table->integer('tested_revision');
|
||||
$table->integer('revision');
|
||||
});
|
||||
$schema->create('cloud_host_storage_configurations',
|
||||
static function (Blueprint $table): void {
|
||||
$table->integer('host_id')->primary();
|
||||
$table->string('state');
|
||||
});
|
||||
$schema->create('cloud_templates', static function (Blueprint $table): void {
|
||||
$table->string('id')->primary();
|
||||
$table->string('catalog_key');
|
||||
$table->string('version_label');
|
||||
$table->boolean('is_default');
|
||||
$table->string('name');
|
||||
$table->string('os_family');
|
||||
$table->string('os_version');
|
||||
$table->string('source_type');
|
||||
$table->text('sealed_source_url');
|
||||
$table->integer('image_bytes');
|
||||
$table->string('sha256');
|
||||
$table->text('startup_script');
|
||||
$table->integer('revision');
|
||||
$table->string('generation_token');
|
||||
$table->string('state');
|
||||
$table->integer('active_job_id')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_template_replicas',
|
||||
static function (Blueprint $table): void {
|
||||
$table->integer('host_id');
|
||||
$table->string('template_id');
|
||||
$table->integer('revision');
|
||||
$table->string('generation_token');
|
||||
$table->string('state');
|
||||
$table->integer('image_bytes');
|
||||
$table->string('sha256');
|
||||
$table->string('operation_id');
|
||||
$table->integer('job_id')->nullable();
|
||||
$table->string('error_code');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
$table->primary(['host_id', 'template_id']);
|
||||
});
|
||||
|
||||
foreach ([31, 32] as $hostId) {
|
||||
Capsule::table('cloud_hosts')->insert([
|
||||
'id' => $hostId, 'enabled' => 1,
|
||||
'default_network' => 'bridge:br0',
|
||||
]);
|
||||
Capsule::table('cloud_host_agents')->insert([
|
||||
'host_id' => $hostId, 'enabled' => 1,
|
||||
'last_test_status' => 'healthy', 'tested_revision' => 4,
|
||||
'revision' => 4,
|
||||
]);
|
||||
Capsule::table('cloud_host_storage_configurations')->insert([
|
||||
'host_id' => $hostId, 'state' => 'applied',
|
||||
]);
|
||||
}
|
||||
$now = time();
|
||||
$generation = str_repeat('a', 32);
|
||||
$sha = str_repeat('b', 64);
|
||||
Capsule::table('cloud_templates')->insert([
|
||||
'id' => 'standalone-replica-template',
|
||||
'catalog_key' => 'debian-13',
|
||||
'version_label' => 'v2',
|
||||
'is_default' => 1,
|
||||
'name' => 'Debian 13',
|
||||
'os_family' => 'Debian',
|
||||
'os_version' => '13',
|
||||
'source_type' => 'object',
|
||||
'sealed_source_url' => 'test-only',
|
||||
'image_bytes' => 4096,
|
||||
'sha256' => $sha,
|
||||
'startup_script' => '',
|
||||
'revision' => 2,
|
||||
'generation_token' => $generation,
|
||||
'state' => 'degraded',
|
||||
'active_job_id' => 501,
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_template_replicas')->insert([
|
||||
[
|
||||
'host_id' => 31, 'template_id' => 'standalone-replica-template',
|
||||
'revision' => 2, 'generation_token' => $generation,
|
||||
'state' => 'active', 'image_bytes' => 4096, 'sha256' => $sha,
|
||||
'operation_id' => str_repeat('1', 32), 'job_id' => 500,
|
||||
'error_code' => '', 'created_at' => $now, 'updated_at' => $now,
|
||||
],
|
||||
[
|
||||
'host_id' => 32, 'template_id' => 'standalone-replica-template',
|
||||
'revision' => 2, 'generation_token' => $generation,
|
||||
'state' => 'importing', 'image_bytes' => 0, 'sha256' => $sha,
|
||||
'operation_id' => str_repeat('2', 32), 'job_id' => 501,
|
||||
'error_code' => '', 'created_at' => $now, 'updated_at' => $now,
|
||||
],
|
||||
]);
|
||||
|
||||
$job = new CloudJob();
|
||||
$job->id = 501;
|
||||
$job->host_id = 32;
|
||||
$job->kind = 'template_import_replica';
|
||||
$job->parent_job_id = null;
|
||||
$job->operation_id = str_repeat('2', 32);
|
||||
$payload = ['template_id' => 'standalone-replica-template'];
|
||||
$result = [
|
||||
'id' => 'standalone-replica-template',
|
||||
'revision' => 2,
|
||||
'generation_token' => $generation,
|
||||
'size_bytes' => 4096,
|
||||
'sha256' => $sha,
|
||||
];
|
||||
CloudTemplateCatalogService::finishJobLocked(
|
||||
$job,
|
||||
'succeeded',
|
||||
$result,
|
||||
$payload,
|
||||
['effect_may_have_applied' => false, 'safe_to_resubmit' => false]
|
||||
);
|
||||
$template = CloudTemplate::findOrFail('standalone-replica-template');
|
||||
$assert((string) $template->state === 'active'
|
||||
&& $template->active_job_id === null
|
||||
&& (string) CloudTemplateReplica::query()->where('host_id', 32)
|
||||
->where('template_id', (string) $template->id)->value('state')
|
||||
=== 'active',
|
||||
'An exact standalone replica receipt did not restore full availability.');
|
||||
|
||||
Capsule::table('cloud_templates')->where('id', (string) $template->id)
|
||||
->update(['state' => 'degraded', 'active_job_id' => 502]);
|
||||
Capsule::table('cloud_template_replicas')->where('host_id', 32)
|
||||
->where('template_id', (string) $template->id)->update([
|
||||
'state' => 'importing', 'image_bytes' => 0,
|
||||
'job_id' => 502, 'operation_id' => str_repeat('3', 32),
|
||||
]);
|
||||
$job->id = 502;
|
||||
$job->operation_id = str_repeat('3', 32);
|
||||
CloudTemplateCatalogService::finishJobLocked(
|
||||
$job,
|
||||
'failed',
|
||||
null,
|
||||
$payload,
|
||||
[
|
||||
'effect_may_have_applied' => true,
|
||||
'safe_to_resubmit' => false,
|
||||
'error_code' => 'agent_timeout',
|
||||
]
|
||||
);
|
||||
$template->refresh();
|
||||
$assert((int) $template->active_job_id === 502
|
||||
&& (string) $template->state === 'degraded',
|
||||
'An unknown standalone replica effect released the template fence.');
|
||||
|
||||
CloudTemplateCatalogService::finishJobLocked(
|
||||
$job,
|
||||
'succeeded',
|
||||
$result,
|
||||
$payload,
|
||||
['effect_may_have_applied' => false, 'safe_to_resubmit' => false],
|
||||
true
|
||||
);
|
||||
$template->refresh();
|
||||
$assert($template->active_job_id === null
|
||||
&& (string) $template->state === 'active',
|
||||
'Recovered standalone replica evidence did not release the original fence.');
|
||||
|
||||
echo "Template standalone replica finalization behavior passed.\n";
|
||||
Loading…
Reference in New Issue