feat(template): make deletion two phase
parent
b3c7879afb
commit
c13468f054
@ -0,0 +1,742 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\Cloud;
|
||||
|
||||
use App\Models\CloudJob;
|
||||
use App\Models\CloudTemplate;
|
||||
use App\Models\CloudTemplateArtifact;
|
||||
use App\Models\CloudTemplateReplica;
|
||||
use App\Models\CloudUserIsoReplica;
|
||||
|
||||
/**
|
||||
* Owns the destructive half of the template lifecycle.
|
||||
*
|
||||
* Host replicas are removed by immutable Agent operations. Only after every
|
||||
* child is proven applied does this module remove the exact object-storage
|
||||
* version. An ambiguous object DELETE is reconciled with two spaced HEAD
|
||||
* observations and a fresh generation; it is never blindly replayed.
|
||||
*/
|
||||
final class CloudTemplateDeletionService
|
||||
{
|
||||
private const STABLE_SECONDS = 300;
|
||||
private const MAX_DELETE_GENERATIONS = 3;
|
||||
private const DUE_STATES = [
|
||||
'delete_pending', 'deleting', 'delete_uncertain',
|
||||
];
|
||||
|
||||
public static function finishReplicaLocked(
|
||||
CloudTemplate $template,
|
||||
CloudTemplateReplica $replica,
|
||||
string $status,
|
||||
array $operation,
|
||||
bool $recovery
|
||||
): void {
|
||||
if ((string) $replica->state !== 'deleting'
|
||||
&& !($recovery
|
||||
&& (string) $replica->state === 'delete_failed')) {
|
||||
throw new CloudError(
|
||||
'云镜像副本状态与删除任务恢复证据不一致',
|
||||
409
|
||||
);
|
||||
}
|
||||
if ($status === 'succeeded') {
|
||||
if (!CloudAgentCatalogDeletion::confirms(
|
||||
$operation,
|
||||
'template',
|
||||
(string) $template->id
|
||||
)) {
|
||||
throw new CloudError('Agent 云镜像删除回执缺少清理证据', 502);
|
||||
}
|
||||
CloudTemplateStateMachine::moveReplica(
|
||||
$replica,
|
||||
'delete_succeeded'
|
||||
);
|
||||
$replica->error_code = '';
|
||||
} else {
|
||||
CloudTemplateStateMachine::moveReplica(
|
||||
$replica,
|
||||
'delete_failed'
|
||||
);
|
||||
$replica->error_code = self::errorCode(
|
||||
$operation,
|
||||
'template_delete_failed'
|
||||
);
|
||||
}
|
||||
CloudTemplateReplica::query()
|
||||
->where('host_id', (int) $replica->host_id)
|
||||
->where('template_id', (string) $replica->template_id)
|
||||
->where('job_id', (int) $replica->job_id)->update([
|
||||
'state' => (string) $replica->state,
|
||||
'error_code' => (string) $replica->error_code,
|
||||
'updated_at' => time(),
|
||||
]);
|
||||
}
|
||||
|
||||
/** Arm local master cleanup while the aggregate parent is held. */
|
||||
public static function armLocked(
|
||||
CloudTemplate $template,
|
||||
CloudJob $parent
|
||||
): void {
|
||||
self::assertChildProjectionLocked($template, $parent, 'applied');
|
||||
$artifact = CloudTemplateArtifact::query()
|
||||
->where('template_id', (string) $template->id)
|
||||
->lockForUpdate()->first();
|
||||
if ($artifact === null) {
|
||||
throw new CloudRecoveryInstanceStateConflict(
|
||||
'云镜像主副本删除账本缺失',
|
||||
409
|
||||
);
|
||||
}
|
||||
if ((string) $artifact->state === 'delete_attention') {
|
||||
throw new CloudRecoveryInstanceStateConflict(
|
||||
'云镜像主副本清理需要管理员重新核对',
|
||||
409
|
||||
);
|
||||
}
|
||||
if (in_array((string) $artifact->state, self::DUE_STATES, true)) {
|
||||
return;
|
||||
}
|
||||
if ((string) $artifact->state !== 'ready'
|
||||
|| !self::artifactMatchesTemplate($artifact, $template)) {
|
||||
throw new CloudRecoveryInstanceStateConflict(
|
||||
'云镜像主副本身份或状态不允许删除',
|
||||
409
|
||||
);
|
||||
}
|
||||
$now = time();
|
||||
$artifact->state = 'delete_pending';
|
||||
$artifact->error_code = '';
|
||||
$artifact->delete_operation_id = '';
|
||||
$artifact->delete_generation = 0;
|
||||
$artifact->delete_cycle_attempts = 0;
|
||||
$artifact->delete_presence_confirmations = 0;
|
||||
$artifact->delete_last_checked_at = 0;
|
||||
$artifact->delete_next_check_at = $now;
|
||||
$artifact->delete_attention_at = null;
|
||||
$artifact->updated_at = $now;
|
||||
$artifact->save();
|
||||
}
|
||||
|
||||
/** Retry only local master cleanup; Agent delete operations stay immutable. */
|
||||
public static function retryAttentionLocked(
|
||||
CloudTemplate $template,
|
||||
CloudJob $parent
|
||||
): void {
|
||||
self::assertChildProjectionLocked($template, $parent, 'applied');
|
||||
$artifact = CloudTemplateArtifact::query()
|
||||
->where('template_id', (string) $template->id)
|
||||
->lockForUpdate()->first();
|
||||
if ($artifact === null
|
||||
|| (string) $artifact->state !== 'delete_attention'
|
||||
|| !self::artifactMatchesTemplate($artifact, $template)) {
|
||||
throw new CloudError('云镜像主副本不在可重试清理状态', 409);
|
||||
}
|
||||
CloudTemplateStateMachine::moveTemplate($template, 'deleting');
|
||||
$now = time();
|
||||
$template->updated_at = $now;
|
||||
$template->save();
|
||||
$artifact->state = 'delete_pending';
|
||||
$artifact->error_code = '';
|
||||
$artifact->delete_operation_id = '';
|
||||
$artifact->delete_cycle_attempts = 0;
|
||||
$artifact->delete_presence_confirmations = 0;
|
||||
$artifact->delete_last_checked_at = 0;
|
||||
$artifact->delete_next_check_at = $now;
|
||||
$artifact->delete_attention_at = null;
|
||||
$artifact->updated_at = $now;
|
||||
$artifact->save();
|
||||
}
|
||||
|
||||
/** Restore sellable projection only when every host proves no deletion. */
|
||||
public static function restoreNotAppliedLocked(
|
||||
CloudTemplate $template,
|
||||
CloudJob $parent
|
||||
): void {
|
||||
self::assertChildProjectionLocked($template, $parent, 'not_applied');
|
||||
$replicas = self::lockedReplicas((string) $template->id);
|
||||
foreach ($replicas as $replica) {
|
||||
CloudTemplateStateMachine::moveReplica(
|
||||
$replica,
|
||||
(int) $replica->revision === (int) $template->revision
|
||||
? 'active' : 'stale'
|
||||
);
|
||||
$replica->error_code = '';
|
||||
$replica->updated_at = time();
|
||||
$replica->save();
|
||||
}
|
||||
CloudTemplateStateMachine::moveTemplate($template, 'active');
|
||||
$template->updated_at = time();
|
||||
$template->save();
|
||||
CloudJobService::completeAggregateLocked(
|
||||
$parent,
|
||||
'failed',
|
||||
false,
|
||||
true,
|
||||
'template_delete_not_applied'
|
||||
);
|
||||
}
|
||||
|
||||
/** Publish a fully known partial host deletion without touching the master. */
|
||||
public static function settleMixedLocked(
|
||||
CloudTemplate $template,
|
||||
CloudJob $parent
|
||||
): void {
|
||||
self::assertChildProjectionLocked($template, $parent, 'mixed');
|
||||
CloudTemplateStateMachine::moveTemplate($template, 'delete_failed');
|
||||
$template->updated_at = time();
|
||||
$template->save();
|
||||
CloudJobService::completeAggregateLocked(
|
||||
$parent,
|
||||
'failed',
|
||||
true,
|
||||
false,
|
||||
'template_delete_failed'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a bounded number of local object cleanup ledgers.
|
||||
*
|
||||
* @return array{processed:int,completed:int,deferred:int,attention:int,errors:int}
|
||||
*/
|
||||
public static function processDue(
|
||||
int $limit = 3,
|
||||
?CloudObjectStorageS3Client $client = null
|
||||
): array {
|
||||
$limit = max(1, min(16, $limit));
|
||||
$now = time();
|
||||
$ids = CloudTemplateArtifact::query()
|
||||
->whereIn('state', self::DUE_STATES)
|
||||
->where('delete_next_check_at', '<=', $now)
|
||||
->orderBy('delete_next_check_at')
|
||||
->orderBy('template_id')
|
||||
->limit($limit)
|
||||
->pluck('template_id')
|
||||
->map(static fn ($id): string => (string) $id)->all();
|
||||
$report = [
|
||||
'processed' => 0,
|
||||
'completed' => 0,
|
||||
'deferred' => 0,
|
||||
'attention' => 0,
|
||||
'errors' => 0,
|
||||
];
|
||||
$client ??= new CloudObjectStorageS3Client();
|
||||
foreach ($ids as $templateId) {
|
||||
try {
|
||||
$result = self::processOne($templateId, $client);
|
||||
$report['processed']++;
|
||||
$report[$result]++;
|
||||
} catch (\Throwable $error) {
|
||||
$report['errors']++;
|
||||
error_log(sprintf(
|
||||
'cloud template artifact cleanup template=%s failed: %s',
|
||||
$templateId,
|
||||
get_class($error)
|
||||
));
|
||||
}
|
||||
}
|
||||
return $report;
|
||||
}
|
||||
|
||||
/** Strict pre-settlement child/replica identity projection. */
|
||||
public static function assertChildProjectionLocked(
|
||||
CloudTemplate $template,
|
||||
CloudJob $parent,
|
||||
string $resolution
|
||||
): void {
|
||||
if ((string) $parent->kind !== 'template_delete'
|
||||
|| (string) $parent->execution_mode !== 'aggregate'
|
||||
|| (int) ($template->active_job_id ?? 0) !== (int) $parent->id
|
||||
|| !in_array($resolution, ['applied', 'not_applied', 'mixed'], true)) {
|
||||
throw new CloudRecoveryAggregateStateConflict(
|
||||
'云镜像删除汇总任务绑定无效',
|
||||
409
|
||||
);
|
||||
}
|
||||
$children = CloudJob::where('parent_job_id', (int) $parent->id)
|
||||
->orderBy('id')->lockForUpdate()->get();
|
||||
$replicas = self::lockedReplicas((string) $template->id);
|
||||
if ($children->isEmpty() || $children->count() !== $replicas->count()) {
|
||||
throw new CloudRecoveryAggregateStateConflict(
|
||||
'云镜像删除子任务与副本数量不一致',
|
||||
409
|
||||
);
|
||||
}
|
||||
if (CloudUserIsoReplica::query()->whereIn(
|
||||
'job_id',
|
||||
$children->map(
|
||||
static fn (CloudJob $child): int => (int) $child->id
|
||||
)->all()
|
||||
)->exists()) {
|
||||
throw new CloudRecoveryOperationIdentityConflict(
|
||||
'云镜像删除子任务被 ISO 副本账本重复引用',
|
||||
409
|
||||
);
|
||||
}
|
||||
foreach ($children as $child) {
|
||||
$childResolution = $resolution === 'mixed'
|
||||
? (string) $child->effect_resolution : $resolution;
|
||||
if (!in_array($childResolution, ['applied', 'not_applied'], true)) {
|
||||
throw new CloudRecoveryAggregateStateConflict(
|
||||
'云镜像删除子任务效果状态无效',
|
||||
409
|
||||
);
|
||||
}
|
||||
$matches = $replicas->filter(
|
||||
static fn (CloudTemplateReplica $replica): bool =>
|
||||
(int) ($replica->job_id ?? 0) === (int) $child->id
|
||||
);
|
||||
$replica = $matches->count() === 1 ? $matches->first() : null;
|
||||
$expectedState = $childResolution === 'applied'
|
||||
? 'delete_succeeded' : 'delete_failed';
|
||||
if ($replica === null
|
||||
|| (int) ($child->parent_job_id ?? 0) !== (int) $parent->id
|
||||
|| (string) $child->kind !== 'template_delete_replica'
|
||||
|| (string) $child->execution_mode !== 'agent'
|
||||
|| (int) $child->user_id !== (int) $parent->user_id
|
||||
|| (int) $child->host_id !== (int) $replica->host_id
|
||||
|| !hash_equals(
|
||||
(string) $child->operation_id,
|
||||
(string) $replica->operation_id
|
||||
)) {
|
||||
throw new CloudRecoveryOperationIdentityConflict(
|
||||
'云镜像删除子任务与副本身份不一致',
|
||||
409
|
||||
);
|
||||
}
|
||||
if (!hash_equals(
|
||||
(string) $template->generation_token,
|
||||
(string) $replica->generation_token
|
||||
)
|
||||
|| (string) $replica->state !== $expectedState
|
||||
|| (int) $replica->revision > (int) $template->revision
|
||||
|| (int) $replica->image_bytes !== (int) $template->image_bytes
|
||||
|| !hash_equals(
|
||||
(string) $template->sha256,
|
||||
(string) $replica->sha256
|
||||
)) {
|
||||
throw new CloudRecoveryInstanceStateConflict(
|
||||
'云镜像删除副本状态或内容不一致',
|
||||
409
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Strict post-settlement projection used by immutable parent rechecks. */
|
||||
public static function assertSettledProjectionLocked(
|
||||
CloudTemplate $template,
|
||||
CloudJob $parent,
|
||||
string $resolution
|
||||
): void {
|
||||
if ($resolution === 'applied') {
|
||||
if ((string) $template->state !== 'retired'
|
||||
|| !self::lockedReplicas((string) $template->id)->isEmpty()
|
||||
|| CloudTemplateArtifact::query()->where(
|
||||
'template_id',
|
||||
(string) $template->id
|
||||
)->lockForUpdate()->exists()) {
|
||||
throw new CloudRecoveryInstanceStateConflict(
|
||||
'云镜像删除已结算投影仍保留受管工件',
|
||||
409
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
self::assertChildProjectionLocked($template, $parent, $resolution);
|
||||
if ($resolution === 'not_applied'
|
||||
&& (string) $template->state !== 'active') {
|
||||
throw new CloudRecoveryInstanceStateConflict(
|
||||
'云镜像删除未应用投影没有恢复为可用状态',
|
||||
409
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public static function artifactState(string $templateId): ?array
|
||||
{
|
||||
$artifact = CloudTemplateArtifact::find($templateId);
|
||||
return $artifact === null ? null : [
|
||||
'state' => (string) $artifact->state,
|
||||
'error_code' => (string) $artifact->error_code,
|
||||
'delete_generation' => (int) $artifact->delete_generation,
|
||||
'next_check_at' => (int) $artifact->delete_next_check_at,
|
||||
'attention_at' => $artifact->delete_attention_at === null
|
||||
? null : (int) $artifact->delete_attention_at,
|
||||
];
|
||||
}
|
||||
|
||||
private static function processOne(
|
||||
string $templateId,
|
||||
CloudObjectStorageS3Client $client
|
||||
): string {
|
||||
$prepared = self::prepare($templateId, time());
|
||||
if ($prepared === null) {
|
||||
return 'deferred';
|
||||
}
|
||||
/** @var CloudTemplateArtifact $artifact */
|
||||
$artifact = $prepared['artifact'];
|
||||
try {
|
||||
$identity = CloudTemplateArtifactManager::inspectDeletionTarget(
|
||||
$artifact,
|
||||
$client
|
||||
);
|
||||
} catch (CloudObjectStorageTransferError $error) {
|
||||
self::deferInspection(
|
||||
$templateId,
|
||||
$error->publicCode,
|
||||
$prepared['mode'] === 'inspect'
|
||||
);
|
||||
return 'deferred';
|
||||
}
|
||||
if ($identity === null) {
|
||||
self::finishSuccess($templateId);
|
||||
return 'completed';
|
||||
}
|
||||
if (!CloudTemplateArtifactManager::deletionIdentityMatches(
|
||||
$artifact,
|
||||
$identity
|
||||
)) {
|
||||
self::markAttention($templateId, 'template_object_identity_changed');
|
||||
return 'attention';
|
||||
}
|
||||
if ($prepared['mode'] === 'inspect') {
|
||||
$artifact = self::recordStablePresence($templateId, time());
|
||||
if ($artifact === null) {
|
||||
$current = CloudTemplateArtifact::find($templateId);
|
||||
if ($current !== null
|
||||
&& (int) $current->delete_cycle_attempts
|
||||
>= self::MAX_DELETE_GENERATIONS
|
||||
&& (int) $current->delete_presence_confirmations >= 2) {
|
||||
self::markAttention(
|
||||
$templateId,
|
||||
'template_object_delete_limit_reached'
|
||||
);
|
||||
return 'attention';
|
||||
}
|
||||
return 'deferred';
|
||||
}
|
||||
}
|
||||
try {
|
||||
CloudTemplateArtifactManager::deleteExact($artifact, $client);
|
||||
} catch (CloudObjectStorageTransferError $error) {
|
||||
if ($error->effectMayHaveApplied) {
|
||||
self::markUncertain($templateId, $error->publicCode);
|
||||
return 'deferred';
|
||||
}
|
||||
self::markAttention($templateId, $error->publicCode);
|
||||
return 'attention';
|
||||
}
|
||||
self::finishSuccess($templateId);
|
||||
return 'completed';
|
||||
}
|
||||
|
||||
/** @return array{mode:string,artifact:CloudTemplateArtifact}|null */
|
||||
private static function prepare(string $templateId, int $now): ?array
|
||||
{
|
||||
$snapshot = CloudTemplate::find($templateId);
|
||||
$parentId = (int) ($snapshot?->active_job_id ?? 0);
|
||||
if ($parentId < 1) {
|
||||
return null;
|
||||
}
|
||||
return (new CloudTemplate())->getConnection()->transaction(
|
||||
static function () use ($templateId, $parentId, $now): ?array {
|
||||
$parent = CloudJob::query()->where('id', $parentId)
|
||||
->lockForUpdate()->first();
|
||||
$template = CloudTemplate::query()->where('id', $templateId)
|
||||
->lockForUpdate()->first();
|
||||
if ($parent === null || $template === null
|
||||
|| (int) ($template->active_job_id ?? 0) !== $parentId
|
||||
|| !in_array((string) $template->state, [
|
||||
'deleting', 'delete_failed',
|
||||
], true)) {
|
||||
return null;
|
||||
}
|
||||
self::assertChildProjectionLocked(
|
||||
$template,
|
||||
$parent,
|
||||
'applied'
|
||||
);
|
||||
$artifact = CloudTemplateArtifact::query()
|
||||
->where('template_id', $templateId)
|
||||
->lockForUpdate()->first();
|
||||
if ($artifact === null
|
||||
|| !in_array(
|
||||
(string) $artifact->state,
|
||||
self::DUE_STATES,
|
||||
true
|
||||
)
|
||||
|| (int) $artifact->delete_next_check_at > $now
|
||||
|| !self::artifactMatchesTemplate($artifact, $template)) {
|
||||
return null;
|
||||
}
|
||||
if ((string) $artifact->state === 'delete_pending') {
|
||||
self::claimDeleteGeneration($artifact, $now);
|
||||
return ['mode' => 'delete', 'artifact' => $artifact];
|
||||
}
|
||||
return ['mode' => 'inspect', 'artifact' => $artifact];
|
||||
},
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
private static function claimDeleteGeneration(
|
||||
CloudTemplateArtifact $artifact,
|
||||
int $now
|
||||
): void {
|
||||
$artifact->state = 'deleting';
|
||||
$artifact->delete_operation_id = bin2hex(random_bytes(16));
|
||||
$artifact->delete_generation = max(
|
||||
0,
|
||||
(int) $artifact->delete_generation
|
||||
) + 1;
|
||||
$artifact->delete_cycle_attempts = max(
|
||||
0,
|
||||
(int) $artifact->delete_cycle_attempts
|
||||
) + 1;
|
||||
$artifact->delete_presence_confirmations = 0;
|
||||
$artifact->delete_last_checked_at = 0;
|
||||
$artifact->delete_next_check_at = $now + self::STABLE_SECONDS;
|
||||
$artifact->error_code = '';
|
||||
$artifact->updated_at = $now;
|
||||
$artifact->save();
|
||||
}
|
||||
|
||||
private static function recordStablePresence(
|
||||
string $templateId,
|
||||
int $now
|
||||
): ?CloudTemplateArtifact {
|
||||
return (new CloudTemplateArtifact())->getConnection()->transaction(
|
||||
static function () use ($templateId, $now): ?CloudTemplateArtifact {
|
||||
$artifact = CloudTemplateArtifact::query()
|
||||
->where('template_id', $templateId)
|
||||
->lockForUpdate()->first();
|
||||
if ($artifact === null || !in_array(
|
||||
(string) $artifact->state,
|
||||
['deleting', 'delete_uncertain'],
|
||||
true
|
||||
)) {
|
||||
return null;
|
||||
}
|
||||
$last = (int) $artifact->delete_last_checked_at;
|
||||
if ($last > 0 && $now - $last < self::STABLE_SECONDS) {
|
||||
$artifact->delete_next_check_at = $last
|
||||
+ self::STABLE_SECONDS;
|
||||
$artifact->updated_at = $now;
|
||||
$artifact->save();
|
||||
return null;
|
||||
}
|
||||
$confirmations = max(
|
||||
0,
|
||||
(int) $artifact->delete_presence_confirmations
|
||||
) + 1;
|
||||
$artifact->state = 'delete_uncertain';
|
||||
$artifact->delete_presence_confirmations = $confirmations;
|
||||
$artifact->delete_last_checked_at = $now;
|
||||
$artifact->delete_next_check_at = $now
|
||||
+ self::STABLE_SECONDS;
|
||||
$artifact->error_code = 'template_object_still_present';
|
||||
$artifact->updated_at = $now;
|
||||
if ($confirmations < 2) {
|
||||
$artifact->save();
|
||||
return null;
|
||||
}
|
||||
if ((int) $artifact->delete_cycle_attempts
|
||||
>= self::MAX_DELETE_GENERATIONS) {
|
||||
$artifact->save();
|
||||
return null;
|
||||
}
|
||||
self::claimDeleteGeneration($artifact, $now);
|
||||
return $artifact;
|
||||
},
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
private static function deferInspection(
|
||||
string $templateId,
|
||||
string $code,
|
||||
bool $priorDeleteMayHaveApplied
|
||||
): void {
|
||||
CloudTemplateArtifact::query()->where('template_id', $templateId)
|
||||
->whereIn('state', self::DUE_STATES)->update([
|
||||
'state' => $priorDeleteMayHaveApplied
|
||||
? 'delete_uncertain' : 'delete_pending',
|
||||
'error_code' => substr($code, 0, 64),
|
||||
'delete_next_check_at' => time() + self::STABLE_SECONDS,
|
||||
'updated_at' => time(),
|
||||
]);
|
||||
}
|
||||
|
||||
private static function markUncertain(
|
||||
string $templateId,
|
||||
string $code
|
||||
): void {
|
||||
CloudTemplateArtifact::query()->where('template_id', $templateId)
|
||||
->where('state', 'deleting')->update([
|
||||
'state' => 'delete_uncertain',
|
||||
'error_code' => substr($code, 0, 64),
|
||||
'delete_presence_confirmations' => 0,
|
||||
'delete_last_checked_at' => 0,
|
||||
'delete_next_check_at' => time() + self::STABLE_SECONDS,
|
||||
'updated_at' => time(),
|
||||
]);
|
||||
}
|
||||
|
||||
private static function markAttention(
|
||||
string $templateId,
|
||||
string $code
|
||||
): void {
|
||||
$snapshot = CloudTemplate::find($templateId);
|
||||
$parentId = (int) ($snapshot?->active_job_id ?? 0);
|
||||
if ($parentId < 1) {
|
||||
return;
|
||||
}
|
||||
(new CloudTemplate())->getConnection()->transaction(
|
||||
static function () use ($templateId, $parentId, $code): void {
|
||||
$parent = CloudJob::query()->where('id', $parentId)
|
||||
->lockForUpdate()->first();
|
||||
$template = CloudTemplate::query()->where('id', $templateId)
|
||||
->lockForUpdate()->first();
|
||||
if ($parent === null || $template === null
|
||||
|| (int) ($template->active_job_id ?? 0) !== $parentId) {
|
||||
return;
|
||||
}
|
||||
self::assertChildProjectionLocked(
|
||||
$template,
|
||||
$parent,
|
||||
'applied'
|
||||
);
|
||||
$artifact = CloudTemplateArtifact::query()
|
||||
->where('template_id', $templateId)
|
||||
->lockForUpdate()->first();
|
||||
if ($artifact === null) {
|
||||
return;
|
||||
}
|
||||
$now = time();
|
||||
$artifact->state = 'delete_attention';
|
||||
$artifact->error_code = substr($code, 0, 64);
|
||||
$artifact->delete_next_check_at = 0;
|
||||
$artifact->delete_attention_at = $now;
|
||||
$artifact->updated_at = $now;
|
||||
$artifact->save();
|
||||
if ((string) $template->state === 'deleting') {
|
||||
CloudTemplateStateMachine::moveTemplate(
|
||||
$template,
|
||||
'delete_failed'
|
||||
);
|
||||
$template->updated_at = $now;
|
||||
$template->save();
|
||||
}
|
||||
CloudJobService::completeAggregateLocked(
|
||||
$parent,
|
||||
'failed',
|
||||
true,
|
||||
false,
|
||||
substr($code, 0, 64)
|
||||
);
|
||||
},
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
private static function finishSuccess(string $templateId): void
|
||||
{
|
||||
$snapshot = CloudTemplate::find($templateId);
|
||||
$parentId = (int) ($snapshot?->active_job_id ?? 0);
|
||||
if ($parentId < 1) {
|
||||
return;
|
||||
}
|
||||
(new CloudTemplate())->getConnection()->transaction(
|
||||
static function () use ($templateId, $parentId): void {
|
||||
$parent = CloudJob::query()->where('id', $parentId)
|
||||
->lockForUpdate()->first();
|
||||
$template = CloudTemplate::query()->where('id', $templateId)
|
||||
->lockForUpdate()->first();
|
||||
if ($parent === null || $template === null
|
||||
|| (int) ($template->active_job_id ?? 0) !== $parentId) {
|
||||
return;
|
||||
}
|
||||
self::assertChildProjectionLocked(
|
||||
$template,
|
||||
$parent,
|
||||
'applied'
|
||||
);
|
||||
$artifact = CloudTemplateArtifact::query()
|
||||
->where('template_id', $templateId)
|
||||
->lockForUpdate()->first();
|
||||
if ($artifact === null || !in_array(
|
||||
(string) $artifact->state,
|
||||
self::DUE_STATES,
|
||||
true
|
||||
)) {
|
||||
throw new CloudRecoveryInstanceStateConflict(
|
||||
'云镜像主副本清理终结状态不一致',
|
||||
409
|
||||
);
|
||||
}
|
||||
CloudTemplateArtifact::query()
|
||||
->where('template_id', $templateId)->delete();
|
||||
CloudTemplateReplica::query()
|
||||
->where('template_id', $templateId)->delete();
|
||||
CloudTemplateStateMachine::moveTemplate($template, 'retired');
|
||||
$template->revision = (int) $template->revision + 1;
|
||||
$template->updated_at = time();
|
||||
$template->save();
|
||||
if ((string) $parent->phase !== 'terminal') {
|
||||
CloudJobService::completeAggregateLocked(
|
||||
$parent,
|
||||
'succeeded',
|
||||
true,
|
||||
false,
|
||||
'',
|
||||
['template_id' => $templateId, 'deleted' => true]
|
||||
);
|
||||
return;
|
||||
}
|
||||
if ((string) $parent->outcome === 'failed'
|
||||
&& (string) $parent->effect_resolution === 'unknown') {
|
||||
$parent->effect_resolution = 'applied';
|
||||
$parent->effect_resolved_at = time();
|
||||
$parent->effect_resolved_by = null;
|
||||
$parent->effect_resolution_reason =
|
||||
'template_master_cleanup_confirmed';
|
||||
$parent->finalization_incident = 0;
|
||||
$parent->finalization_error_code = null;
|
||||
$parent->available_at = 0;
|
||||
$parent->updated_at = time();
|
||||
$parent->save();
|
||||
}
|
||||
},
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
private static function artifactMatchesTemplate(
|
||||
CloudTemplateArtifact $artifact,
|
||||
CloudTemplate $template
|
||||
): bool {
|
||||
return (int) $artifact->size_bytes === (int) $template->image_bytes
|
||||
&& hash_equals(
|
||||
(string) $artifact->sha256,
|
||||
(string) $template->sha256
|
||||
)
|
||||
&& (string) $artifact->object_key !== ''
|
||||
&& (string) $artifact->object_etag !== '';
|
||||
}
|
||||
|
||||
private static function lockedReplicas(string $templateId)
|
||||
{
|
||||
return CloudTemplateReplica::query()
|
||||
->where('template_id', $templateId)
|
||||
->orderBy('host_id')->lockForUpdate()->get();
|
||||
}
|
||||
|
||||
private static function errorCode(array $operation, string $fallback): string
|
||||
{
|
||||
$code = $operation['error_code'] ?? null;
|
||||
return is_string($code)
|
||||
&& preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $code) === 1
|
||||
? $code : $fallback;
|
||||
}
|
||||
}
|
||||
@ -1,2 +1,2 @@
|
||||
{* Shared Venix CSS/JS cache key. Increment whenever a shared browser asset changes. *}
|
||||
{assign var=venix_asset_version value='2026103012' scope='global'}
|
||||
{assign var=venix_asset_version value='2026103013' scope='global'}
|
||||
|
||||
@ -0,0 +1,354 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services {
|
||||
final class Config
|
||||
{
|
||||
public static string $key = '';
|
||||
|
||||
public static function get($name, $default = null)
|
||||
{
|
||||
return $name === 'cloud_data_key' ? self::$key : $default;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
use App\Models\CloudJob;
|
||||
use App\Models\CloudTemplate;
|
||||
use App\Models\CloudTemplateArtifact;
|
||||
use App\Services\Cloud\CloudObjectStorageS3Client;
|
||||
use App\Services\Cloud\CloudSecrets;
|
||||
use App\Services\Cloud\CloudTemplateDeletionService;
|
||||
use App\Services\Config;
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\Exception\ConnectException;
|
||||
use GuzzleHttp\Handler\MockHandler;
|
||||
use GuzzleHttp\HandlerStack;
|
||||
use GuzzleHttp\Middleware;
|
||||
use GuzzleHttp\Psr7\Request;
|
||||
use GuzzleHttp\Psr7\Response;
|
||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
|
||||
$assert = static function (bool $condition, string $message): void {
|
||||
if (!$condition) {
|
||||
throw new RuntimeException($message);
|
||||
}
|
||||
};
|
||||
Config::$key = base64_encode(str_repeat('d', 32));
|
||||
$capsule = new Capsule();
|
||||
$capsule->addConnection([
|
||||
'driver' => 'sqlite', 'database' => ':memory:',
|
||||
], 'default');
|
||||
$capsule->setAsGlobal();
|
||||
$capsule->bootEloquent();
|
||||
$schema = $capsule->schema('default');
|
||||
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
||||
$table->increments('id');
|
||||
$table->string('operation_id');
|
||||
$table->string('request_idempotency_key');
|
||||
$table->string('request_fingerprint');
|
||||
$table->integer('parent_job_id')->nullable();
|
||||
$table->integer('user_id');
|
||||
$table->integer('host_id')->nullable();
|
||||
$table->string('kind');
|
||||
$table->string('execution_mode');
|
||||
$table->string('phase');
|
||||
$table->string('outcome')->nullable();
|
||||
$table->string('status');
|
||||
$table->text('sealed_payload');
|
||||
$table->integer('attempts')->default(0);
|
||||
$table->integer('dispatch_attempts')->default(0);
|
||||
$table->integer('dispatch_deadline_at')->default(1);
|
||||
$table->integer('dispatch_started_at')->nullable();
|
||||
$table->integer('agent_accepted_at')->nullable();
|
||||
$table->integer('dispatched_at')->nullable();
|
||||
$table->integer('receipt_deadline_at')->nullable();
|
||||
$table->integer('terminal_at')->nullable();
|
||||
$table->boolean('effect_may_have_applied')->nullable();
|
||||
$table->boolean('safe_to_resubmit')->nullable();
|
||||
$table->string('effect_resolution')->default('not_applicable');
|
||||
$table->integer('effect_resolved_at')->nullable();
|
||||
$table->integer('effect_resolved_by')->nullable();
|
||||
$table->string('effect_resolution_reason')->default('');
|
||||
$table->string('completion_event_id')->nullable();
|
||||
$table->integer('read_at')->nullable();
|
||||
$table->string('lease_token')->nullable();
|
||||
$table->integer('lease_until')->default(0);
|
||||
$table->integer('available_at')->default(0);
|
||||
$table->string('error_code')->nullable();
|
||||
$table->boolean('finalization_incident')->default(0);
|
||||
$table->string('finalization_error_code')->nullable();
|
||||
$table->text('result_json')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_templates', static function (Blueprint $table): void {
|
||||
$table->string('id')->primary();
|
||||
$table->string('catalog_key');
|
||||
$table->string('version_label');
|
||||
$table->boolean('is_default')->default(false);
|
||||
$table->string('name');
|
||||
$table->string('os_family');
|
||||
$table->string('os_version');
|
||||
$table->string('source_type');
|
||||
$table->text('sealed_source_url');
|
||||
$table->integer('image_bytes');
|
||||
$table->string('sha256');
|
||||
$table->text('startup_script');
|
||||
$table->integer('revision');
|
||||
$table->string('generation_token');
|
||||
$table->string('state');
|
||||
$table->integer('active_job_id')->nullable();
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
$schema->create('cloud_template_replicas', static function (Blueprint $table): void {
|
||||
$table->integer('host_id');
|
||||
$table->string('template_id');
|
||||
$table->integer('revision');
|
||||
$table->string('generation_token');
|
||||
$table->string('state');
|
||||
$table->integer('image_bytes');
|
||||
$table->string('sha256');
|
||||
$table->string('operation_id');
|
||||
$table->integer('job_id')->nullable();
|
||||
$table->string('error_code')->default('');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
$table->primary(['host_id', 'template_id']);
|
||||
});
|
||||
$schema->create('cloud_user_iso_replicas', static function (Blueprint $table): void {
|
||||
$table->integer('job_id')->nullable();
|
||||
});
|
||||
$schema->create('cloud_template_artifacts', static function (Blueprint $table): void {
|
||||
$table->string('template_id')->primary();
|
||||
$table->integer('storage_revision');
|
||||
$table->text('sealed_storage_runtime');
|
||||
$table->string('object_key');
|
||||
$table->string('object_version_id')->default('');
|
||||
$table->string('object_etag');
|
||||
$table->integer('size_bytes');
|
||||
$table->string('sha256');
|
||||
$table->string('state');
|
||||
$table->string('error_code')->default('');
|
||||
$table->string('delete_operation_id')->default('');
|
||||
$table->integer('delete_generation')->default(0);
|
||||
$table->integer('delete_cycle_attempts')->default(0);
|
||||
$table->integer('delete_presence_confirmations')->default(0);
|
||||
$table->integer('delete_last_checked_at')->default(0);
|
||||
$table->integer('delete_next_check_at')->default(0);
|
||||
$table->integer('delete_attention_at')->nullable();
|
||||
$table->integer('verified_at');
|
||||
$table->integer('created_at');
|
||||
$table->integer('updated_at');
|
||||
});
|
||||
|
||||
$runtime = [
|
||||
'config' => [
|
||||
'endpoint' => 'https://objects.example.com', 'region' => 'ewr',
|
||||
'bucket' => 'templates', 'prefix' => 'virt', 'path_style' => true,
|
||||
],
|
||||
'credentials' => [
|
||||
'access_key_id' => 'DELETEACCESSKEY',
|
||||
'secret_access_key' => 'delete-secret',
|
||||
],
|
||||
'revision' => 1,
|
||||
];
|
||||
$now = time();
|
||||
$fixture = static function (string $suffix) use ($runtime, $now): array {
|
||||
$parentId = (int) Capsule::table('cloud_jobs')->insertGetId([
|
||||
'operation_id' => md5('parent-' . $suffix),
|
||||
'request_idempotency_key' => md5('parent-' . $suffix),
|
||||
'request_fingerprint' => hash('sha256', 'parent-' . $suffix),
|
||||
'parent_job_id' => null, 'user_id' => 1, 'host_id' => null,
|
||||
'kind' => 'template_delete', 'execution_mode' => 'aggregate',
|
||||
'phase' => 'awaiting_receipt', 'outcome' => null,
|
||||
'status' => 'running',
|
||||
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
|
||||
'attempts' => 0, 'dispatch_attempts' => 0,
|
||||
'dispatch_deadline_at' => $now + 300,
|
||||
'receipt_deadline_at' => $now + 3600,
|
||||
'effect_resolution' => 'not_applicable',
|
||||
'effect_resolution_reason' => '', 'lease_until' => 0,
|
||||
'available_at' => $now, 'finalization_incident' => 0,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
$childOperation = md5('child-' . $suffix);
|
||||
$childId = (int) Capsule::table('cloud_jobs')->insertGetId([
|
||||
'operation_id' => $childOperation,
|
||||
'request_idempotency_key' => $childOperation,
|
||||
'request_fingerprint' => hash('sha256', 'child-' . $suffix),
|
||||
'parent_job_id' => $parentId, 'user_id' => 1, 'host_id' => 7,
|
||||
'kind' => 'template_delete_replica', 'execution_mode' => 'agent',
|
||||
'phase' => 'terminal', 'outcome' => 'succeeded',
|
||||
'status' => 'succeeded',
|
||||
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
|
||||
'attempts' => 1, 'dispatch_attempts' => 1,
|
||||
'dispatch_deadline_at' => $now - 20,
|
||||
'dispatch_started_at' => $now - 19,
|
||||
'agent_accepted_at' => $now - 18,
|
||||
'dispatched_at' => $now - 17,
|
||||
'receipt_deadline_at' => $now + 900,
|
||||
'terminal_at' => $now - 1,
|
||||
'effect_may_have_applied' => 0, 'safe_to_resubmit' => 0,
|
||||
'effect_resolution' => 'applied',
|
||||
'effect_resolution_reason' => '', 'lease_until' => 0,
|
||||
'available_at' => 0, 'finalization_incident' => 0,
|
||||
'created_at' => $now - 20, 'updated_at' => $now - 1,
|
||||
]);
|
||||
$templateId = 'delete-template-' . $suffix;
|
||||
$generation = md5('generation-' . $suffix);
|
||||
$sha = hash('sha256', 'image-' . $suffix);
|
||||
Capsule::table('cloud_templates')->insert([
|
||||
'id' => $templateId, 'catalog_key' => 'debian-13',
|
||||
'version_label' => $suffix, 'is_default' => 0,
|
||||
'name' => 'Delete ' . $suffix, 'os_family' => 'Debian',
|
||||
'os_version' => '13', 'source_type' => 'object',
|
||||
'sealed_source_url' => '', 'image_bytes' => 4096,
|
||||
'sha256' => $sha, 'startup_script' => '', 'revision' => 1,
|
||||
'generation_token' => $generation, 'state' => 'deleting',
|
||||
'active_job_id' => $parentId, 'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_template_replicas')->insert([
|
||||
'host_id' => 7, 'template_id' => $templateId, 'revision' => 1,
|
||||
'generation_token' => $generation, 'state' => 'delete_succeeded',
|
||||
'image_bytes' => 4096, 'sha256' => $sha,
|
||||
'operation_id' => $childOperation, 'job_id' => $childId,
|
||||
'error_code' => '', 'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
Capsule::table('cloud_template_artifacts')->insert([
|
||||
'template_id' => $templateId, 'storage_revision' => 1,
|
||||
'sealed_storage_runtime' => CloudSecrets::encrypt(
|
||||
$runtime, 'template-artifact-runtime'
|
||||
),
|
||||
'object_key' => 'virt/cloud-templates/' . $templateId
|
||||
. '/' . $generation . '/system.qcow2',
|
||||
'object_version_id' => 'version-' . $suffix,
|
||||
'object_etag' => '"etag-' . $suffix . '"',
|
||||
'size_bytes' => 4096, 'sha256' => $sha, 'state' => 'ready',
|
||||
'error_code' => '', 'delete_operation_id' => '',
|
||||
'delete_generation' => 0, 'delete_cycle_attempts' => 0,
|
||||
'delete_presence_confirmations' => 0,
|
||||
'delete_last_checked_at' => 0, 'delete_next_check_at' => 0,
|
||||
'delete_attention_at' => null, 'verified_at' => $now,
|
||||
'created_at' => $now, 'updated_at' => $now,
|
||||
]);
|
||||
$template = CloudTemplate::findOrFail($templateId);
|
||||
$parent = CloudJob::findOrFail($parentId);
|
||||
$template->getConnection()->transaction(
|
||||
static function () use ($templateId, $parentId): void {
|
||||
$parent = CloudJob::where('id', $parentId)
|
||||
->lockForUpdate()->firstOrFail();
|
||||
$template = CloudTemplate::where('id', $templateId)
|
||||
->lockForUpdate()->firstOrFail();
|
||||
CloudTemplateDeletionService::armLocked($template, $parent);
|
||||
},
|
||||
3
|
||||
);
|
||||
return [$templateId, $parentId];
|
||||
};
|
||||
|
||||
[$normalId, $normalParentId] = $fixture('normal');
|
||||
$normalHistory = [];
|
||||
$normalStack = HandlerStack::create(new MockHandler([
|
||||
new Response(200, [
|
||||
'Content-Length' => '4096', 'ETag' => '"etag-normal"',
|
||||
'x-amz-version-id' => 'version-normal',
|
||||
]),
|
||||
new Response(204),
|
||||
]));
|
||||
$normalStack->push(Middleware::history($normalHistory));
|
||||
$normalClient = new CloudObjectStorageS3Client(new Client([
|
||||
'handler' => $normalStack,
|
||||
]));
|
||||
$normal = CloudTemplateDeletionService::processDue(1, $normalClient);
|
||||
$assert($normal['completed'] === 1
|
||||
&& CloudTemplateArtifact::find($normalId) === null
|
||||
&& (string) CloudTemplate::findOrFail($normalId)->state === 'retired'
|
||||
&& (int) CloudTemplate::findOrFail($normalId)->revision === 2
|
||||
&& (string) CloudJob::findOrFail($normalParentId)->outcome
|
||||
=== 'succeeded'
|
||||
&& array_map(
|
||||
static fn (array $item): string => $item['request']->getMethod(),
|
||||
$normalHistory
|
||||
) === ['HEAD', 'DELETE'],
|
||||
'Confirmed master deletion did not retire the exact catalog version.');
|
||||
|
||||
[$unknownId, $unknownParentId] = $fixture('unknown');
|
||||
$unknownHistory = [];
|
||||
$unknownStack = HandlerStack::create(new MockHandler([
|
||||
new Response(200, [
|
||||
'Content-Length' => '4096', 'ETag' => '"etag-unknown"',
|
||||
'x-amz-version-id' => 'version-unknown',
|
||||
]),
|
||||
new ConnectException('lost after delete', new Request(
|
||||
'DELETE', 'https://objects.example.com'
|
||||
)),
|
||||
new Response(200, [
|
||||
'Content-Length' => '4096', 'ETag' => '"etag-unknown"',
|
||||
'x-amz-version-id' => 'version-unknown',
|
||||
]),
|
||||
new Response(200, [
|
||||
'Content-Length' => '4096', 'ETag' => '"etag-unknown"',
|
||||
'x-amz-version-id' => 'version-unknown',
|
||||
]),
|
||||
new Response(204),
|
||||
]));
|
||||
$unknownStack->push(Middleware::history($unknownHistory));
|
||||
$unknownClient = new CloudObjectStorageS3Client(new Client([
|
||||
'handler' => $unknownStack,
|
||||
]));
|
||||
CloudTemplateDeletionService::processDue(1, $unknownClient);
|
||||
$unknownArtifact = CloudTemplateArtifact::findOrFail($unknownId);
|
||||
$assert((string) $unknownArtifact->state === 'delete_uncertain'
|
||||
&& (int) $unknownArtifact->delete_generation === 1,
|
||||
'Ambiguous DELETE was not fenced on its original generation.');
|
||||
Capsule::table('cloud_template_artifacts')->where('template_id', $unknownId)
|
||||
->update(['delete_next_check_at' => 0]);
|
||||
CloudTemplateDeletionService::processDue(1, $unknownClient);
|
||||
$unknownArtifact = CloudTemplateArtifact::findOrFail($unknownId);
|
||||
$assert((int) $unknownArtifact->delete_presence_confirmations === 1
|
||||
&& (int) $unknownArtifact->delete_generation === 1,
|
||||
'One presence probe incorrectly replayed an ambiguous DELETE.');
|
||||
Capsule::table('cloud_template_artifacts')->where('template_id', $unknownId)
|
||||
->update([
|
||||
'delete_last_checked_at' => time() - 301,
|
||||
'delete_next_check_at' => 0,
|
||||
]);
|
||||
$converged = CloudTemplateDeletionService::processDue(1, $unknownClient);
|
||||
$assert($converged['completed'] === 1
|
||||
&& CloudTemplateArtifact::find($unknownId) === null
|
||||
&& (string) CloudTemplate::findOrFail($unknownId)->state === 'retired'
|
||||
&& (string) CloudJob::findOrFail($unknownParentId)->outcome
|
||||
=== 'succeeded'
|
||||
&& array_map(
|
||||
static fn (array $item): string => $item['request']->getMethod(),
|
||||
$unknownHistory
|
||||
) === ['HEAD', 'DELETE', 'HEAD', 'HEAD', 'DELETE'],
|
||||
'Stable two-probe evidence did not create exactly one fresh delete generation.');
|
||||
|
||||
[$attentionId, $attentionParentId] = $fixture('attention');
|
||||
$attentionClient = new CloudObjectStorageS3Client(new Client([
|
||||
'handler' => HandlerStack::create(new MockHandler([
|
||||
new Response(200, [
|
||||
'Content-Length' => '8192', 'ETag' => '"different"',
|
||||
'x-amz-version-id' => 'version-attention',
|
||||
]),
|
||||
])),
|
||||
]));
|
||||
$attention = CloudTemplateDeletionService::processDue(1, $attentionClient);
|
||||
$assert($attention['attention'] === 1
|
||||
&& (string) CloudTemplateArtifact::findOrFail($attentionId)->state
|
||||
=== 'delete_attention'
|
||||
&& (string) CloudTemplate::findOrFail($attentionId)->state
|
||||
=== 'delete_failed'
|
||||
&& (string) CloudJob::findOrFail($attentionParentId)->outcome
|
||||
=== 'failed',
|
||||
'Changed object identity was not fenced for explicit administrator review.');
|
||||
|
||||
echo "Cloud-template two-phase deletion and recovery behavior passed.\n";
|
||||
}
|
||||
Loading…
Reference in New Issue