You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudWorkerService.php

4065 lines
181 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudInstance;
use App\Models\CloudBackupSubscription;
use App\Models\CloudInvoice;
use App\Models\CloudJob;
use App\Models\CloudOffer;
use App\Models\CloudOrder;
use App\Models\CloudRescueTemplateReplica;
use App\Models\CloudServicePeriod;
use App\Models\User;
use App\Services\Payment\PaymentOrderCreationException;
final class CloudWorkerService
{
private const AGENT_TIMEOUT_SECONDS = 900.0;
private const POLICY_AGENT_TIMEOUT_SECONDS = 120.0;
private const LEASE_SECONDS = 930;
private const POLICY_KINDS = ['service_suspend',
'network_suspend', 'network_resume'];
private const VPC_BATCH_KINDS = [
'vpc_host_apply', 'vpc_host_remove',
'vpc_port_attach', 'vpc_port_detach',
];
private const REINSTALL_REVIEW_CODES = [
'reinstall_cleanup_pending',
];
private bool $policyOnly;
private bool $vpcBatchOnly;
private array $observedErrors = [];
private CloudServiceCompensationExecutionService $compensations;
public function observationErrors(): array
{
return array_values(array_unique($this->observedErrors));
}
public function __construct(private ?CloudAgentClient $agent = null,
bool $policyOnly = false, bool $vpcBatchOnly = false,
?CloudServiceCompensationExecutionService $compensations = null)
{
if ($policyOnly && $vpcBatchOnly) {
throw new \InvalidArgumentException(
'Policy-only and VPC-batch-only modes are mutually exclusive'
);
}
$this->policyOnly = $policyOnly;
$this->vpcBatchOnly = $vpcBatchOnly;
$this->compensations = $compensations
?? new CloudServiceCompensationExecutionService();
// A cloud-image clone can legitimately take minutes on remote storage.
// The layered deadlines are Agent API 840s, Agent WSS control 855s,
// Gateway 870s, PHP 900s and worker lease 930s. Each outer layer has
// time to persist the inner layer's structured result before expiry.
}
private function agentForJob(CloudJob $job): CloudAgentClient
{
if ($this->agent !== null) {
return $this->agent;
}
if ((string) $job->kind === 'host_storage_configure') {
return CloudAgentSettingsService::clientForConfiguration(
(int) $job->host_id,
self::POLICY_AGENT_TIMEOUT_SECONDS
);
}
return CloudAgentSettingsService::clientForHost(
(int) $job->host_id,
$this->policyOnly
? self::POLICY_AGENT_TIMEOUT_SECONDS
: self::AGENT_TIMEOUT_SECONDS
);
}
public static function disposition(array $operation, bool $creating,
?string $kind = null): string
{
$status = is_string($operation['status'] ?? null)
? strtolower(trim($operation['status'])) : 'failed';
return $status === 'succeeded' ? 'succeeded'
: ($status === 'failed' ? 'failed' : 'running');
}
/** A create may refund only after both the VM and all managed artifacts are absent. */
private static function createFailureConfirmedAbsent(array $operation): bool
{
$result = $operation['result'] ?? null;
return is_array($result)
&& ($result['creation_absent'] ?? false) === true
&& ($result['managed_artifacts_absent'] ?? false) === true;
}
/** Exact Agent evidence required before PHP releases a deleted VM's ledger. */
private static function deleteSuccessConfirmed(array $operation,
string $kind): bool
{
$result = $operation['result'] ?? null;
return is_array($result) && !array_is_list($result)
&& ($result['deleted'] ?? null) === true
&& ($result['managed_artifacts_absent'] ?? null) === true
&& ($kind === 'delete_force'
? ($result['forced'] ?? null) === true
: ($result['forced'] ?? null) === false);
}
/** Exact evidence for a create request proven not to have crossed dispatch. */
private static function createNotDispatchedFailure(): array
{
return [
'status' => 'failed',
'result' => [
'creation_absent' => true,
'managed_artifacts_absent' => true,
],
];
}
/** Treat unconfirmed reinstall cleanup as a failed receipt, never as active work. */
private static function reinstallNeedsReview(array $operation): bool
{
$code = is_string($operation['error_code'] ?? null)
? strtolower(trim($operation['error_code'])) : '';
if ($code === '' && is_array($operation['error'] ?? null)
&& is_string($operation['error']['code'] ?? null)) {
$code = strtolower(trim($operation['error']['code']));
}
if (in_array($code, self::REINSTALL_REVIEW_CODES, true)) {
return true;
}
$result = $operation['result'] ?? null;
return is_array($result)
&& ($result['cleanup_pending'] ?? false) === true;
}
public function tick(): bool
{
$this->observedErrors = [];
$quarantined = self::quarantineInvalidExecutionModeJob();
$expired = self::expireDueJobs();
// Deadline convergence is always first. Recovery inspection runs in
// the dedicated --recovery-only worker, so this worker never waits on
// an offline recovery target before dispatching ordinary work.
$jobWorked = !$quarantined && $this->tickJob();
$compensationWorked = false;
$backupPreparationWorked = false;
$templateCleanup = false;
$templateTemporaryCleanup = false;
if (!$this->policyOnly && !$this->vpcBatchOnly) {
try {
// Backup intent commits before any provider I/O. Resume at
// most one missing multipart-plan/job binding per tick, after
// already accepted Agent work had the first dispatch chance.
$backupPreparationWorked =
(new CloudBackupPublicationPreparation())->resumeOne();
} catch (\Throwable $error) {
$this->observedErrors[] =
'backup_publication_preparation_failed';
}
try {
// This is a PHP-only ledger mutation. The execution service
// owns its bounded lease and idempotency evidence; the worker
// advances at most one item and never replays an Agent call.
$compensationWorked = $this->compensations->tick();
} catch (\Throwable $error) {
// Ordinary Agent work already advanced above and remains
// independent from a temporarily unavailable local ledger.
$this->observedErrors[] = 'service_compensation_failed';
}
try {
$templateCleanup = (new CloudTemplateImportCancellation())
->cleanupOneExpired();
} catch (\Throwable $error) {
// The intent has already moved to source_unknown when an S3
// abort cannot be proven. Surface one worker observation and
// never loop a destructive provider request blindly.
$this->observedErrors[] =
'template_import_cleanup_unresolved';
}
try {
$templateTemporaryCleanup =
CloudTemplateRemoteFetcher::cleanupOneStaleFile();
} catch (\Throwable $error) {
$this->observedErrors[] =
'template_import_temporary_cleanup_failed';
}
}
return $quarantined || $jobWorked || $compensationWorked
|| $backupPreparationWorked
|| $templateCleanup
|| $templateTemporaryCleanup || $expired > 0;
}
/**
* Fail closed when a durable job's execution-domain discriminator is
* damaged. Reinterpreting a local row as an Agent mutation (or an
* aggregate parent as a leaf) could replay the original operation. Keep
* the immutable dispatch evidence and encrypted payload untouched, publish
* one sticky PHP-only incident, and let the recovery worker recheck it
* with bounded leases forever without contacting the Agent.
*/
private static function quarantineInvalidExecutionModeJob(): bool
{
return (new CloudJob())->getConnection()->transaction(
static function (): bool {
$job = CloudJob::where('phase', '<>', 'terminal')
->where(static function ($invalid): void {
$invalid->where(static function ($leaf): void {
$leaf->whereIn(
'kind',
CloudOperationRecoveryPolicy::ledgerFinalizerKinds()
)->where(static function ($mode): void {
$mode->whereNull('execution_mode')
->orWhere('execution_mode', '<>', 'agent');
});
})->orWhere(static function ($aggregate): void {
$aggregate->whereIn(
'kind',
CloudJobService::aggregateKinds()
)->where(static function ($mode): void {
$mode->whereNull('execution_mode')
->orWhere(
'execution_mode',
'<>',
'aggregate'
);
});
});
})
->orderBy('id')
->lockForUpdate()
->first();
if ($job === null) {
return false;
}
$now = time();
$aggregate = CloudJobService::isAggregateKind(
(string) $job->kind
);
$job->phase = 'terminal';
$job->outcome = 'failed';
$job->status = 'failed';
$job->terminal_at = $now;
$job->effect_may_have_applied = 1;
$job->safe_to_resubmit = 0;
$job->effect_resolution = 'unknown';
$job->error_code = $aggregate
? 'recovery_aggregate_state_conflict'
: 'recovery_instance_state_conflict';
$job->finalization_incident = 1;
$job->finalization_error_code = $job->error_code;
$job->read_at = null;
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = $now + 300;
$job->updated_at = $now;
$job->save();
return true;
},
3
);
}
/** Claim and advance at most one durable cloud job. */
private function tickJob(): bool
{
$job = (new CloudJob())->getConnection()->transaction(function () {
$now = time();
$query = self::dispatchableBeforeDeadline(
CloudJob::where('phase', 'accepted')
->where(static function ($mode): void {
$mode->where('execution_mode', 'agent')
->orWhere(static function ($local): void {
$local->where('execution_mode', 'local')
->where('kind', 'payment_create');
});
})
->where('dispatch_attempts', 0),
$now
)->where('available_at', '<=', $now)
->where('lease_until', '<=', $now);
CloudJobService::applyVpcHostDispatchFence($query);
if ($this->policyOnly) {
$query->whereIn('kind', self::POLICY_KINDS);
} elseif ($this->vpcBatchOnly) {
$query->whereNotNull('vpc_batch_id')
->whereIn('kind', self::VPC_BATCH_KINDS);
} else {
// The regular worker remains a fail-safe policy consumer when
// the dedicated workers are not installed, but always claims
// target-state policy ahead of slow image/provisioning jobs.
$quoted = "'" . implode("','", self::POLICY_KINDS) . "'";
$query->orderByRaw("CASE WHEN kind IN (" . $quoted
. ") THEN 0 WHEN kind = 'expiry_delete' THEN 1 ELSE 2 END");
}
$job = $query->orderBy('id')->lockForUpdate()->first();
if ($job === null) {
return null;
}
$job->lease_token = bin2hex(random_bytes(16));
$job->lease_until = $now + (in_array($job->kind, self::POLICY_KINDS, true)
? 150 : self::LEASE_SECONDS);
$job->save();
return $job;
}, 3);
if ($job === null) {
return false;
}
$payload = [];
$dispatched = false;
try {
$payload = CloudSecrets::decrypt($job->sealed_payload, 'job-payload');
if ((string) $job->kind === 'payment_create') {
try {
$this->runPaymentCreate($job, $payload, $dispatched);
} catch (\Throwable $error) {
$this->observedErrors[] = 'payment_task_failed';
$crossed = $dispatched;
if (!$crossed) {
$fresh = CloudJob::find((int) $job->id);
$crossed = $fresh !== null
&& CloudDispatchLedger::mayHaveDispatched($fresh);
}
self::forceTerminalFailure(
(int) $job->id,
(string) $job->lease_token,
$crossed,
!$crossed,
'payment_finalization_failed'
);
error_log('cloud payment task ' . (int) $job->id
. ' failed: ' . get_class($error));
}
return true;
}
if (!is_string($payload['method'] ?? null)
|| !in_array(strtoupper($payload['method']),
['POST', 'PUT', 'PATCH', 'DELETE'], true)
|| !is_string($payload['request_target'] ?? null)
|| !str_starts_with($payload['request_target'], '/v1/')
|| !is_array($payload['body'] ?? null)) {
throw new CloudError('Agent 任务请求格式无效', 409);
}
$policyKind = in_array($job->kind, self::POLICY_KINDS, true);
$systemPolicy = ($payload['system_policy'] ?? false) === true
&& $policyKind && self::validSystemPolicy($job, $payload);
$systemCleanup = $job->kind === 'expiry_delete'
&& ($payload['system_cleanup'] ?? false) === true
&& self::validSystemCleanup($job, $payload);
$vpcSystem = CloudVpcPortService::isSystemJob($job, $payload);
if (($policyKind && !$systemPolicy)
|| ($job->kind === 'expiry_delete' && !$systemCleanup)) {
throw new CloudError('系统策略任务与实例状态不匹配', 409);
}
$actor = ['user_id' => (int) $job->user_id, 'is_admin' => false, 'is_super_admin' => false,
'can_manage' => false, 'session_id' => '', 'operation_id' => $job->operation_id];
$user = User::find($job->user_id);
if (!$systemPolicy && !$systemCleanup && !$vpcSystem
&& ($user === null || (int) $user->enable !== 1)) {
$operation = CloudOperationRecoveryPolicy::isCreateLike(
(string) $job->kind
)
? self::createNotDispatchedFailure()
: ['status' => 'failed'];
$operation['effect_may_have_applied'] = false;
$operation['safe_to_resubmit'] = true;
$this->finish($job, $operation, $payload);
return true;
}
if ($job->kind === 'create') {
self::assertCreateTrafficPoolSnapshot($job);
$actor['can_provision'] = true;
$actor['can_manage'] = true;
} elseif ($vpcSystem) {
// Host-route projections are PHP-derived control-plane work.
// The Agent keeps its host-global VPC endpoint behind the
// super-administrator boundary, while the job remains owned
// by the tenant whose accepted port mutation it completes.
$actor['is_admin'] = true;
$actor['is_super_admin'] = true;
$actor['can_manage'] = true;
} elseif ($systemPolicy || $systemCleanup) {
$actor['can_manage'] = true;
if ($systemCleanup) {
$actor['system_cleanup'] = true;
}
} elseif (($payload['can_manage'] ?? false) === true && $user !== null
&& (bool) $user->is_admin && in_array((int) $user->id,
array_map('intval', explode(',', (string) \App\Services\Config::get('super_admin'))), true)) {
$actor['is_admin'] = $actor['is_super_admin'] = $actor['can_manage'] = true;
} elseif ((int) ($payload['owner_id'] ?? 0) === (int) $job->user_id) {
$actor['can_manage'] = true;
}
if ($job->kind !== 'create' && !$actor['can_manage']) {
$this->finish($job, ['status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => true], $payload);
return true;
}
if (!CloudFirewallService::assertJobRunnable($job, $payload)) {
// A confirmed instance deletion dominates an overlapping
// tenant firewall request. Close the obsolete job without
// touching the deleted policy or contacting the Agent.
$this->finish($job, [
'operation_id' => (string) $job->operation_id,
'status' => 'failed',
'error_code' => 'firewall_apply_superseded',
'error' => '实例已删除,防火墙请求不再适用',
], $payload);
return true;
}
CloudVpcPortService::assertJobRunnable($job, $payload);
if ($systemPolicy && $job->instance_id !== null) {
$instance = CloudInstance::find($job->instance_id);
$policyMarker = in_array((string) $job->kind,
['network_suspend', 'network_resume'], true)
? CloudInstanceNetworkPolicy::payloadMarker(
$payload,
(int) $job->instance_id,
(string) $job->operation_id
) : null;
$desired = $policyMarker === null
? self::policyDesired($instance, (string) $job->kind)
: CloudInstanceNetworkPolicy::jobStillDesired(
(int) $job->instance_id,
(string) $job->kind,
(string) $job->operation_id
);
if (!$desired) {
$this->finish($job, ['status' => 'succeeded', 'result' => ['skipped' => true]], $payload);
return true;
}
}
if (!$systemPolicy && !$systemCleanup && !$vpcSystem
&& $job->kind !== 'create'
&& $job->instance_id !== null) {
$instance = CloudInstance::find($job->instance_id);
$policyTransition = $instance !== null
&& (in_array((string) $instance->service_status,
['suspending_expired', 'resume_pending', 'resuming',
'suspend_failed', 'resume_failed', 'expired_grace',
'purge_stopping', 'purge_stop_failed',
'purge_force_required',
'purge_deleting', 'purge_failed'], true)
|| in_array((string) $instance->network_status,
['blocking', 'blocking_expired', 'blocked_expired',
'restore_pending', 'restoring', 'block_failed',
'restore_failed'], true));
$globalOverride = $actor['is_super_admin'] && $actor['can_manage'];
if ($instance === null || ($job->kind !== 'delete' && !$globalOverride
&& ($policyTransition || !$instance->serviceAvailable()))) {
$this->finish($job, ['status' => 'failed',
'error' => '实例服务已到期或正在执行策略,不能执行该操作'], $payload);
return true;
}
}
if ((string) $job->kind === 'network_change'
&& $job->public_network_transition_id === null) {
throw new CloudError('旧公网网络写任务已停用', 409);
} elseif ((string) $job->kind === 'backup_restore') {
self::assertPortableRestoreDispatchReady($job, $payload);
} elseif (str_starts_with(
(string) $job->kind,
'instance_migration_'
)) {
// Canonical PeerMigration rows carry an exact recoverable
// identity. Reject stale scalar-IP or torn ledger jobs before
// the dispatch CAS so they can never contact an Agent. This
// must validate the live request directly: expected() also
// accepts a post-effect recovery escrow, which is never a
// valid shape for an accepted, not-yet-dispatched job.
CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$payload,
false
);
} elseif (in_array(
(string) $job->kind,
['delete', 'delete_force', 'expiry_delete'],
true
) && $job->instance_id !== null) {
$deleteInstance = CloudInstance::find(
(int) $job->instance_id
);
if ($deleteInstance !== null) {
CloudInstanceResourceRelease::
assertCanDispatchDelete($deleteInstance);
}
}
$wirePayload = $payload;
$dispatchPayload = $payload;
$dispatchSealedPayload = null;
if ((string) $job->kind === 'backup_restore') {
[$dispatchPayload, $wirePayload] =
self::portableRestoreDispatchPayload($job, $payload);
$dispatchSealedPayload = CloudSecrets::encrypt(
$dispatchPayload,
'job-payload'
);
}
$now = time();
$dispatchClaim = self::dispatchableBeforeDeadline(
CloudJob::where('id', $job->id)
->where('lease_token', $job->lease_token)
->where('phase', 'accepted')
->where('dispatch_attempts', 0),
$now
);
$dispatchUpdate = [
'phase' => 'dispatching',
'status' => 'running',
'attempts' => 1,
'dispatch_attempts' => 1,
'dispatch_started_at' => $now,
'dispatched_at' => $now,
'receipt_deadline_at' => $now
+ CloudJobService::receiptDeadlineSecondsForJob($job),
'updated_at' => $now,
];
if ($dispatchSealedPayload !== null) {
// Persist only the fingerprint of the exact one-shot wire
// request. The presigned capability itself exists solely in
// $wirePayload and can never become durable job state.
$dispatchUpdate['sealed_payload'] = $dispatchSealedPayload;
}
$updated = $dispatchClaim->update($dispatchUpdate);
if ($updated !== 1) {
return true;
}
// Keep the claimed model aligned with the successful CAS. The
// synchronous response is finalized through this same object; a
// stale dispatch_attempts=0 value would incorrectly classify a
// real Agent response as a local pre-dispatch failure and omit its
// immutable identity from the terminal payload.
$job->phase = 'dispatching';
$job->status = 'running';
$job->attempts = 1;
$job->dispatch_attempts = 1;
$job->dispatch_started_at = $now;
$job->dispatched_at = $now;
$job->receipt_deadline_at = $now
+ CloudJobService::receiptDeadlineSecondsForJob($job);
if ($dispatchSealedPayload !== null) {
$job->sealed_payload = $dispatchSealedPayload;
$payload = $dispatchPayload;
}
CloudFirewallService::markJobRunning($job);
CloudVpcPortService::markJobRunning($job);
CloudPortableRestoreFinalizer::markDispatched($job);
if ((string) $job->kind === 'network_change'
&& $job->public_network_transition_id !== null) {
CloudPublicNetworkTransitionWorkflow::markDispatched($job);
}
$dispatched = true;
$operation = self::operationForJob(
$job,
$this->agentForJob($job)->request(
$payload['method'],
$payload['request_target'],
$actor,
self::agentRequestBody((string) $job->kind,
$wirePayload['body'])
)
);
$operation = self::normalizeAgentOperation($operation);
$agentStatus = strtolower((string) ($operation['status'] ?? ''));
if (in_array($agentStatus, ['succeeded', 'failed'], true)) {
$operation['effect_may_have_applied'] =
$operation['effect_may_have_applied'] ?? true;
$operation['safe_to_resubmit'] =
$operation['safe_to_resubmit'] ?? false;
$this->finish($job, $operation, $payload);
} elseif (in_array($agentStatus,
['accepted', 'reserved', 'dispatching', 'pending', 'running'],
true)) {
CloudJob::where('id', $job->id)
->where('lease_token', $job->lease_token)
->where('phase', 'dispatching')->update([
'phase' => 'awaiting_receipt',
'agent_accepted_at' => time(),
'lease_token' => null,
'lease_until' => 0,
'updated_at' => time(),
]);
$fresh = CloudJob::find((int) $job->id);
$event = $fresh === null ? null
: CloudOperationCompletionService::pendingForJob($fresh);
if ($event !== null) {
$this->settleCompletionReceipt((int) $job->id,
(string) $event->event_id);
}
} else {
throw new CloudError('Agent 返回了无效的任务状态', 502);
}
} catch (\Throwable $error) {
// Diagnostics deliberately contain no raw payloads, credentials or upstream tracebacks.
$this->observedErrors[] = 'worker_tick_failed';
$diagnostic = self::exceptionDiagnostic($error, $dispatched,
(string) $job->kind, $job->order_id !== null, self::payloadAction($payload));
$log = 'cloud worker job ' . $job->id . ' failed: ' . get_class($error);
if ($error instanceof CloudError) {
$log .= ' status=' . $error->httpStatus . ' message=' . $diagnostic;
}
error_log($log);
$safe = !$dispatched || ($error instanceof CloudAgentError
&& ($error->safeValidationFailure
|| $error->safePreDispatchFailure));
$operation = $safe && CloudOperationRecoveryPolicy::isCreateLike(
(string) $job->kind
)
? self::createNotDispatchedFailure() : ['status' => 'failed'];
$operation['operation_id'] = (string) $job->operation_id;
$operation['error_code'] = $error instanceof CloudAgentError
? $error->agentCode : 'dispatch_failed';
$operation['effect_may_have_applied'] = !$safe;
$operation['safe_to_resubmit'] = $safe;
try {
$this->finish($job, $operation, $payload);
} catch (\Throwable $finishError) {
self::forceTerminalFailure((int) $job->id,
(string) $job->lease_token, !$safe,
$safe, 'local_finalization_failed', $safe);
}
}
return true;
}
/**
* Keep both the queue claim and the accepted-to-dispatching CAS behind the
* same absolute deadline. The second check closes the boundary race where
* an old storage task was scanned just before created_at+60 but would
* otherwise cross dispatch using its former persisted 300-second value.
*/
private static function dispatchableBeforeDeadline($query, int $now)
{
$storageWindow = CloudJobService::dispatchDeadlineSeconds(
'host_storage_configure'
);
return $query->where(static function ($safe): void {
$safe->where(static function ($payment): void {
$payment->where('execution_mode', 'local')
->where('kind', 'payment_create');
})->orWhere(static function ($agent): void {
$agent->where('execution_mode', 'agent')
->whereNotExists(static function ($events): void {
$events->selectRaw('1')
->from(
'cloud_agent_completion_events as dispatch_events'
)->whereColumn(
'dispatch_events.host_id',
'cloud_jobs.host_id'
)->whereColumn(
'dispatch_events.operation_id',
'cloud_jobs.operation_id'
);
});
});
})->where('dispatch_deadline_at', '>', $now)
->where(static function ($eligible) use (
$now, $storageWindow
): void {
$eligible->where('kind', '!=', 'host_storage_configure')
->orWhere('created_at', '>', $now - $storageWindow);
});
}
private function runPaymentCreate(CloudJob $job, array $payload,
bool &$dispatched): void
{
$input = $payload['input'] ?? null;
if ($job->invoice_id === null || !is_array($input)
|| !is_string($input['idempotency_key'] ?? null)
|| !is_string($input['method'] ?? null)) {
$this->finishPaymentCreate($job, false, null, false, true,
'payment_request_invalid');
return;
}
$now = time();
$dispatchClaim = self::dispatchableBeforeDeadline(
CloudJob::where('id', (int) $job->id)
->where('lease_token', (string) $job->lease_token)
->where('phase', 'accepted')
->where('dispatch_attempts', 0),
$now
);
$claimed = $dispatchClaim->update([
'phase' => 'dispatching', 'status' => 'running',
'attempts' => 1, 'dispatch_attempts' => 1,
'dispatch_started_at' => $now, 'dispatched_at' => $now,
'receipt_deadline_at' => $now
+ CloudJobService::receiptDeadlineSeconds(
(string) $job->kind
),
'updated_at' => $now,
]);
if ($claimed !== 1) {
return;
}
$dispatched = true;
try {
$invoice = CloudInvoice::where('id', (int) $job->invoice_id)
->where('user_id', (int) $job->user_id)->first();
$user = User::find((int) $job->user_id);
if ($invoice === null || $user === null) {
throw new CloudError('账单或账户不存在', 404);
}
$payment = CloudInvoicePaymentService::create(
$invoice,
$user,
$input,
(string) ($payload['client_ip'] ?? ''),
(string) ($payload['user_agent'] ?? '')
);
$this->finishPaymentCreate($job, true, $payment, false, false, '');
} catch (\Throwable $error) {
$attempt = \App\Models\CloudInvoicePayment::where(
'user_id', (int) $job->user_id
)->where('idempotency_key', (string) $input['idempotency_key'])
->first();
if ($error instanceof PaymentOrderCreationException) {
$effect = $error->effectMayHaveApplied;
$safe = $error->safeToResubmit;
$code = $error->failureCode;
} elseif ((string) $input['method'] === 'usdt') {
// USDT order creation is one local database transaction; it
// does not submit a non-idempotent provider mutation here.
$effect = false;
$safe = true;
$code = 'payment_creation_failed';
} else {
$status = (string) ($attempt?->status ?? '');
$effect = $attempt !== null
&& $status !== 'not_created';
$safe = !$effect;
$code = $effect ? 'payment_creation_unknown'
: 'payment_creation_failed';
}
$this->finishPaymentCreate($job, false, null, $effect,
$safe, $code);
}
}
private function finishPaymentCreate(CloudJob $job, bool $success,
?array $result, bool $effect, bool $safe, string $code): void
{
CloudJob::where('id', (int) $job->id)
->where('lease_token', (string) $job->lease_token)
->where('phase', '<>', 'terminal')->update([
'phase' => 'terminal',
'outcome' => $success ? 'succeeded' : 'failed',
'status' => $success ? 'succeeded' : 'failed',
'terminal_at' => time(),
'effect_may_have_applied' => $effect ? 1 : 0,
'safe_to_resubmit' => $safe ? 1 : 0,
'effect_resolution' => $success ? 'applied'
: ($effect ? 'unknown' : 'not_applied'),
'error_code' => $success ? null : $code,
'result_json' => $result === null ? null
: json_encode(['payment_id' => (string) ($result['id'] ?? '')], JSON_THROW_ON_ERROR
| JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE),
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
'lease_token' => null, 'lease_until' => 0,
'available_at' => 0, 'updated_at' => time(),
]);
if (!$success) {
$this->observedErrors[] = 'task_failed';
}
}
/** Expire accepted or dispatched work into a local terminal failure. */
public static function expireDueJobs(int $limit = 100): int
{
return self::expireDueJobsMatching(null, null, $limit);
}
/**
* Close only one owner's public tasks when the worker daemon is absent.
* This path consumes durable receipts and performs local finalization; it
* never opens an Agent connection or dispatches a mutation.
*/
public static function expireDueTasksForOwner(int $userId,
?int $taskId = null, int $limit = 100): int
{
if ($userId < 1 || ($taskId !== null && $taskId < 1)) {
return 0;
}
return self::expireDueJobsMatching($userId, $taskId, $limit);
}
/**
* Close one already-due task from an administrator read path.
* This never contacts an Agent and never dispatches a mutation.
*/
public static function expireDueTaskForAdmin(int $taskId): int
{
if ($taskId < 1) {
return 0;
}
return self::expireDueJobsMatching(null, $taskId, 1);
}
private static function expireDueJobsMatching(?int $userId,
?int $taskId, int $limit): int
{
$now = time();
$storageDispatchWindow = CloudJobService::dispatchDeadlineSeconds(
'host_storage_configure'
);
$storageWindow = CloudJobService::receiptDeadlineSeconds(
'host_storage_configure'
);
$query = CloudJob::where(static function ($query) use (
$now, $storageDispatchWindow, $storageWindow
): void {
$query->where(static function ($accepted) use (
$now, $storageDispatchWindow
): void {
$accepted->where('phase', 'accepted')
->where(static function ($deadline) use (
$now, $storageDispatchWindow
): void {
$deadline->where('dispatch_deadline_at', '<=', $now)
->orWhereNull('dispatch_deadline_at')
->orWhere(static function ($storage) use (
$now, $storageDispatchWindow
): void {
$storage->where(
'kind', 'host_storage_configure'
)->where('created_at', '<=',
$now - $storageDispatchWindow);
});
});
})->orWhere(static function ($sent) use (
$now, $storageWindow
): void {
$sent->whereIn('phase', ['dispatching', 'awaiting_receipt'])
->where(static function ($deadline) use (
$now, $storageWindow
): void {
$deadline->where(static function ($due) use (
$now
): void {
$due->whereNotNull('receipt_deadline_at')
->where('receipt_deadline_at', '<=', $now);
})->orWhereNull('receipt_deadline_at')
->orWhere(static function ($storage) use (
$now, $storageWindow
): void {
$storage->where(
'kind', 'host_storage_configure'
)->whereNotNull('dispatched_at')
->where('dispatched_at', '<=',
$now - $storageWindow);
});
});
});
})->where(static function ($claimable): void {
// A VPC batch parent is a PHP-local coordination ledger. It never
// crosses an Agent dispatch deadline; only its durable child
// finalizers may complete it.
$claimable->whereNotIn('kind', [
'vpc_membership_batch',
'vpc_batch_instance_fence',
'public_ip_pool_replacement_batch',
])
->orWhere('execution_mode', '<>', 'local');
});
if ($userId !== null) {
// Child jobs are not publicly listed, but the owner-scoped
// daemon-loss fallback must still terminalize them before their
// aggregate parent can close.
$query->where('user_id', $userId);
}
if ($taskId !== null) {
$query->where('id', $taskId);
}
$rows = $query
->orderByRaw('CASE WHEN parent_job_id IS NULL THEN 1 ELSE 0 END')
->orderBy('id')
->limit(max(1, min(100, $limit)))->get();
$expired = 0;
foreach ($rows as $row) {
$dispatchConflict = CloudDispatchLedger::isInvalid($row);
$effect = CloudDispatchLedger::mayHaveDispatched($row);
$failureCode = $dispatchConflict
? CloudDispatchLedger::CONFLICT_CODE
: ($effect
? 'completion_deadline_exceeded'
: 'dispatch_deadline_exceeded');
$token = bin2hex(random_bytes(16));
$claim = CloudJob::where('id', (int) $row->id)
->where('phase', (string) $row->phase)
->where('dispatch_attempts', (int) $row->dispatch_attempts);
if ($row->lease_token === null) {
$claim->whereNull('lease_token');
} else {
$claim->where('lease_token', (string) $row->lease_token);
}
// An absolute deadline wins over a stale worker lease. The CAS on
// phase/attempt/token prevents two local expiry paths from owning
// the same finalization.
$updated = $claim->update([
'lease_token' => $token,
'lease_until' => $now + self::LEASE_SECONDS,
'updated_at' => $now,
]);
if ($updated !== 1) {
continue;
}
$expired++;
$claimed = CloudJob::find((int) $row->id);
try {
if ($claimed !== null
&& CloudDispatchLedger::isInvalid($claimed)) {
self::forceTerminalFailure(
(int) $row->id,
$token,
true,
false,
CloudDispatchLedger::CONFLICT_CODE
);
continue;
}
if ($claimed !== null && $claimed->host_id !== null
&& CloudDispatchLedger::isDispatched($claimed)) {
$event = CloudOperationCompletionService::pendingForJob(
$claimed
);
if ($event !== null) {
(new self())->settleCompletionReceipt(
(int) $claimed->id,
(string) $event->event_id
);
continue;
}
}
if ($claimed !== null
&& (string) $claimed->execution_mode === 'local'
&& (string) $claimed->kind === 'payment_create') {
$payload = CloudSecrets::decrypt(
(string) $claimed->sealed_payload, 'job-payload'
);
[$success, $result, $effect, $safe, $code]
= self::expiredPaymentResult($claimed, $payload);
(new self())->finishPaymentCreate(
$claimed, $success, $result, $effect, $safe, $code
);
continue;
}
if ($claimed !== null && $claimed->host_id === null
&& $claimed->parent_job_id === null
&& (string) $claimed->execution_mode === 'aggregate') {
self::expireAggregate($claimed, $token);
continue;
}
$payload = CloudSecrets::decrypt(
(string) $claimed->sealed_payload, 'job-payload'
);
$operation = !$effect && CloudOperationRecoveryPolicy::isCreateLike(
(string) $claimed->kind
)
? self::createNotDispatchedFailure()
: ['status' => 'failed'];
$operation['operation_id'] = (string) $claimed->operation_id;
$operation['error_code'] = $failureCode;
$operation['effect_may_have_applied'] = $effect;
$operation['safe_to_resubmit'] = !$effect;
(new self())->finish($claimed, $operation, $payload);
} catch (\Throwable $error) {
self::forceTerminalFailure((int) $row->id, $token,
$effect, !$effect, $failureCode);
}
}
return $expired;
}
/**
* Derive a local payment task's deadline result from its durable attempt.
* Claiming the worker is not proof that the provider POST was sent.
* This read never creates or retransmits a provider order.
*
* @return array{0:bool,1:?array,2:bool,3:bool,4:string}
*/
private static function expiredPaymentResult(CloudJob $job,
array $payload): array
{
$key = $payload['input']['idempotency_key'] ?? null;
if (!is_string($key)
|| preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $key) !== 1) {
return [false, null, true, false, 'payment_creation_unknown'];
}
$attempt = \App\Models\CloudInvoicePayment::where(
'user_id', (int) $job->user_id
)->where('invoice_id', (int) $job->invoice_id)
->where('idempotency_key', $key)->first();
if ($attempt === null) {
return [false, null, false, true,
'payment_order_not_dispatched'];
}
$status = (string) $attempt->status;
if (in_array($status,
[
'pending', 'failed', 'expired',
'paid_invoice', 'credited_balance',
], true)) {
return [true, ['id' => (string) $attempt->id], false, false, ''];
}
if ($status === 'not_created') {
return [false, null, false, true, 'payment_order_rejected'];
}
return [false, null, true, false, 'payment_creation_unknown'];
}
public function settleCompletionReceipt(int $jobId,
string $eventId): CloudJob
{
$event = \App\Models\CloudAgentCompletionEvent::where(
'event_id', $eventId
)->first();
if ($event === null) {
throw new CloudError('完成回执不存在', 404);
}
$source = CloudJob::find($jobId);
if ($source === null) {
throw new CloudError('任务不存在', 404);
}
if ((string) $source->execution_mode !== 'agent'
|| $source->host_id === null
|| (int) $event->host_id !== (int) $source->host_id
|| !hash_equals((string) $source->operation_id,
(string) $event->operation_id)) {
CloudOperationCompletionService::mark($event, 'rejected');
throw new CloudError('完成回执与任务不匹配', 409);
}
if (CloudDispatchLedger::isInvalid($source)) {
CloudOperationCompletionService::deferDispatchLedgerConflict(
$source
);
return CloudJob::find($jobId) ?? $source;
}
if ((string) $source->phase === 'terminal') {
if (CloudOperationCompletionService::settledTaskWithEvent(
$source,
$event
)) {
CloudOperationCompletionService::mark($event, 'applied');
} else {
CloudOperationRecoveryService::settleLateReceipt(
$source,
$event
);
}
return CloudJob::find($jobId) ?? $source;
}
if ((string) $event->disposition !== 'received') {
throw new CloudError('完成回执已经处理', 409);
}
$receiptDeadline = CloudJobService::effectiveReceiptDeadline($source);
if ($receiptDeadline < 1
|| (int) $event->received_at >= $receiptDeadline) {
// The durable receive timestamp, rather than PHP processing time,
// decides the boundary. A receipt persisted before the absolute
// deadline can still settle later; one received after it is audit
// evidence only and can never revive or succeed the generation.
$token = bin2hex(random_bytes(16));
$claim = CloudJob::where('id', $jobId)
->whereIn('phase', ['dispatching', 'awaiting_receipt'])
->where('dispatch_attempts', 1);
if ($source->lease_token === null) {
$claim->whereNull('lease_token');
} else {
$claim->where('lease_token', (string) $source->lease_token);
}
$claimed = $claim->update([
'lease_token' => $token,
'lease_until' => time() + self::LEASE_SECONDS,
'updated_at' => time(),
]);
if ($claimed === 1) {
self::forceTerminalFailure(
$jobId,
$token,
true,
false,
'completion_deadline_exceeded'
);
}
$current = CloudJob::find($jobId) ?? $source;
if ((string) $current->phase === 'terminal') {
CloudOperationRecoveryService::settleLateReceipt(
$current,
$event
);
}
return $current;
}
$operation = CloudOperationCompletionService::operation($event);
$payload = CloudSecrets::decrypt((string) $source->sealed_payload,
'job-payload');
$token = bin2hex(random_bytes(16));
$claim = CloudJob::where('id', $jobId)
->whereIn('phase', ['dispatching', 'awaiting_receipt'])
->where('dispatch_attempts', 1);
if ($source->lease_token === null) {
$claim->whereNull('lease_token');
} else {
$claim->where('lease_token', (string) $source->lease_token);
}
$claimed = $claim->update([
'lease_token' => $token,
'lease_until' => time() + self::LEASE_SECONDS,
'completion_event_id' => $eventId,
'updated_at' => time(),
]);
if ($claimed !== 1) {
$current = CloudJob::find($jobId) ?? $source;
if ((string) $current->phase === 'terminal') {
if (CloudOperationCompletionService::settledTaskWithEvent(
$current,
$event
)) {
CloudOperationCompletionService::mark($event, 'applied');
} else {
CloudOperationRecoveryService::settleLateReceipt(
$current,
$event
);
}
}
return $current;
}
$job = CloudJob::find($jobId);
$normalized = self::normalizeAgentOperation(
self::operationForJob($job, $operation)
);
try {
$this->finish($job, $normalized, $payload);
} catch (\Throwable $error) {
$createNotApplied = CloudOperationRecoveryPolicy::isCreateLike(
(string) $job->kind
)
&& strtolower((string) ($normalized['status'] ?? ''))
=== 'failed'
&& self::createFailureConfirmedAbsent($normalized);
$safeToResubmit = $createNotApplied
&& ($normalized['safe_to_resubmit'] ?? false) === true;
self::forceTerminalFailure(
$jobId,
$token,
!$createNotApplied,
$safeToResubmit,
'completion_finalization_failed'
);
}
$current = CloudJob::find($jobId) ?? $source;
if (CloudOperationCompletionService::settledTaskWithEvent(
$current,
$event
)) {
CloudOperationCompletionService::mark($event, 'applied');
} elseif ((string) $current->phase === 'terminal') {
CloudOperationRecoveryService::settleLateReceipt(
$current,
$event
);
}
return $current;
}
private static function forceTerminalFailure(int $jobId, string $token,
bool $effect, bool $safe, string $code,
bool $confirmedNotDispatched = false): void
{
$claimed = CloudJob::where('id', $jobId)
->where('lease_token', $token)->where('phase', '<>', 'terminal')
->first();
$recoverableFinalizer = $claimed !== null
&& (CloudOperationRecoveryPolicy::supports(
(string) $claimed->kind
) || CloudOperationRecoveryPolicy::isCreateLike(
(string) $claimed->kind
));
$preserveRecoveryEnvelope = $recoverableFinalizer
|| ($claimed !== null
&& CloudOperationRecoveryPolicy::isCreateLike(
(string) $claimed->kind
));
// Preserve the opaque escrow before attempting decryption. A temporary
// key/configuration failure is itself recoverable; replacing the only
// encrypted request with an empty string would make the PHP-only
// finalizer impossible to resume after the environment is repaired.
$recoveryEnvelope = $preserveRecoveryEnvelope
? (string) $claimed->sealed_payload : '';
$incidentCode = $code;
$dispatchConflict = $claimed !== null
&& CloudDispatchLedger::isInvalid($claimed);
if ($dispatchConflict) {
// A corrupt counter can never prove non-dispatch. Do not invoke a
// normal success/not-applied finalizer or consume Agent evidence;
// preserve the encrypted escrow so repair can resume this exact
// operation without replaying it.
$effect = true;
$safe = false;
$code = CloudDispatchLedger::CONFLICT_CODE;
$recoveryEnvelope = (string) $claimed->sealed_payload;
} elseif ($claimed !== null) {
try {
if ((string) $claimed->execution_mode === 'aggregate') {
if (!self::forceAggregateFailure($claimed, $token)) {
return;
}
} elseif ((string) $claimed->kind === 'payment_create') {
[$success, $result, $paymentEffect, $paymentSafe,
$paymentCode] = self::expiredPaymentResult(
$claimed,
['input' => ['idempotency_key' =>
(string) $claimed->request_idempotency_key]]
);
(new self())->finishPaymentCreate(
$claimed,
$success,
$result,
$paymentEffect,
$paymentSafe,
$paymentCode
);
} else {
try {
$payload = CloudSecrets::decrypt(
(string) $claimed->sealed_payload,
'job-payload'
);
} catch (\Throwable $error) {
throw new CloudRecoveryPayloadUnavailable(
'任务加密恢复数据暂时无法读取',
409
);
}
// Every explicitly registered recovery kind retains its
// encrypted request until the exact Agent operation is
// reconciled. The recovery worker never re-dispatches
// this payload; it is needed only by PHP's existing
// kind-specific ledger finalizers.
if (CloudOperationRecoveryPolicy::supports(
(string) $claimed->kind
) || CloudOperationRecoveryPolicy::isCreateLike(
(string) $claimed->kind
)) {
// An exact create success receipt can finish the paid
// order only while the encrypted wire request and
// credential escrow remain available. Keep it across
// a local finalizer crash just like policy recovery
// payloads; successful convergence scrubs it.
$recoveryEnvelope = (string) $claimed->sealed_payload;
} else {
$safePayload = CloudInput::safeResult($payload);
if (CloudDispatchLedger::isDispatched($claimed)
&& CloudOperationRecoveryIdentity::supports(
(string) $claimed->kind
)) {
$identity = CloudOperationRecoveryIdentity::
expectedFromPayload($claimed, $payload, false);
$safePayload[
CloudOperationRecoveryIdentity::payloadKey()
] = CloudOperationRecoveryIdentity::persisted(
$identity
);
}
$recoveryEnvelope = CloudSecrets::encrypt(
$safePayload,
'job-payload'
);
}
(new self())->finish($claimed, [
'operation_id' => (string) $claimed->operation_id,
'status' => 'failed',
'error_code' => $code,
'effect_may_have_applied' => $effect,
'safe_to_resubmit' => $safe,
], $payload);
}
$terminal = CloudJob::where('id', $jobId)
->where('phase', 'terminal')->exists();
if ($terminal) {
return;
}
} catch (\Throwable $finalizationError) {
$incidentCode = self::localFinalizationIncidentCode(
$finalizationError
);
error_log(sprintf(
'cloud task ledger finalization failed job_id=%d kind=%s error=%s',
$jobId,
(string) $claimed->kind,
get_class($finalizationError)
));
}
}
if ($claimed !== null
&& (string) $claimed->execution_mode === 'aggregate'
&& self::releaseAggregateParentWhenChildrenActive(
(int) $claimed->id,
$token
)) {
// A child can win the lease CAS between our snapshot and claim.
// Never hide still-active child work behind a terminal parent;
// a later bounded deadline pass will finish children first.
return;
}
// An order-backed create must never fall through to a job-only write:
// its wallet, invoice, stock and placement ledgers are one atomic
// finalization unit. Ambiguous effects retain a durable placement
// fence; proven no-effect failures refund and release exactly once.
if ($claimed !== null && (string) $claimed->kind === 'create'
&& $claimed->order_id !== null) {
try {
CloudCreateFailureFinalizer::forceTerminalFailure(
$jobId,
$token,
$effect,
$safe,
$code
);
if (CloudJob::where('id', $jobId)
->where('phase', 'terminal')->exists()) {
return;
}
} catch (\Throwable $finalizationError) {
error_log(sprintf(
'cloud create ledger finalization incident job_id=%d error=%s',
$jobId,
get_class($finalizationError)
));
}
}
if ($claimed !== null) {
// Best-effort resource latches make the incident visible in the
// affected catalog. Their own failure must never keep the public
// task active; the durable finalization_incident below remains the
// authoritative operator alert.
try {
CloudSnapshotService::markJobUncertain($claimed);
} catch (\Throwable $ignored) {
}
try {
CloudBackupService::markJobUncertain($claimed);
} catch (\Throwable $ignored) {
}
try {
CloudFirewallService::markJobUncertain($claimed);
} catch (\Throwable $ignored) {
}
try {
CloudVpcPortService::markJobUncertain($claimed);
} catch (\Throwable $ignored) {
}
if ((string) $claimed->kind === 'network_change'
&& $claimed->public_network_transition_id !== null) {
try {
CloudPublicNetworkTransitionWorkflow::markJobUncertain(
$claimed
);
} catch (\Throwable $ignored) {
}
}
}
// Last-resort terminalization is deliberately business-conservative:
// never refund, release or claim success after a ledger finalizer
// failed. Retain a validated encrypted request only when it can support
// later ledger recovery, retain every resource fence, and publish an
// explicit durable incident for operators.
// Registered policy kinds and an order-backed create have PHP-only
// retry paths that can safely publish a known not-applied effect for
// generation zero or for one exact transport-level
// dispatch_state=not_dispatched proof. The immutable counter remains
// one in the latter case. Aggregate/other special finalizers retain
// the conservative unknown effect if their atomic fallback fails.
$neverDispatched = $recoverableFinalizer
&& CloudDispatchLedger::isNeverDispatched($claimed);
$transportNotDispatched = $recoverableFinalizer
&& $confirmedNotDispatched
&& !$effect
&& $safe
&& CloudDispatchLedger::isDispatched($claimed)
&& ($claimed->agent_accepted_at ?? null) === null
&& ($claimed->completion_event_id ?? null) === null;
$notDispatched = $neverDispatched || $transportNotDispatched;
$terminalUpdate = [
'phase' => 'terminal', 'outcome' => 'failed',
'status' => 'failed', 'terminal_at' => time(),
'effect_may_have_applied' => $notDispatched ? 0 : 1,
'safe_to_resubmit' => 0,
'effect_resolution' => $notDispatched
? 'not_applied' : 'unknown',
'error_code' => $dispatchConflict
? CloudDispatchLedger::CONFLICT_CODE
: 'local_finalization_failed',
'finalization_incident' => 1,
'finalization_error_code' => substr(
$dispatchConflict
? CloudDispatchLedger::CONFLICT_CODE
: $incidentCode,
0,
128
),
'sealed_payload' => $recoveryEnvelope,
'lease_token' => null,
'lease_until' => 0,
'available_at' => ($dispatchConflict
|| $recoverableFinalizer) ? time() + 300 : 0,
'updated_at' => time(),
];
if ($transportNotDispatched) {
$terminalUpdate['effect_resolution_reason'] =
CloudDispatchLedger::TRANSPORT_NOT_DISPATCHED_REASON;
}
CloudJob::where('id', $jobId)->where('lease_token', $token)
->where('phase', '<>', 'terminal')->update($terminalUpdate);
}
private static function localFinalizationIncidentCode(
\Throwable $error
): string {
return match (true) {
$error instanceof CloudRecoveryDispatchLedgerConflict =>
CloudDispatchLedger::CONFLICT_CODE,
$error instanceof CloudRecoveryAggregateStateConflict =>
'recovery_aggregate_state_conflict',
$error instanceof CloudRecoveryPayloadUnavailable =>
'recovery_payload_unavailable',
$error instanceof CloudRecoveryOperationIdentityConflict =>
'recovery_operation_identity_conflict',
$error instanceof CloudRecoveryReceiptInvalid =>
'recovery_receipt_invalid',
$error instanceof CloudRecoveryInstanceStateConflict =>
'recovery_instance_state_conflict',
$error instanceof CloudError => 'recovery_ledger_conflict',
default => 'recovery_internal_error',
};
}
/** Close aggregate children first, then atomically converge its catalog. */
private static function forceAggregateFailure(CloudJob $parent,
string $token): bool
{
$children = CloudJob::where('parent_job_id', (int) $parent->id)
->where('phase', '<>', 'terminal')->orderBy('id')->get();
foreach ($children as $child) {
$childToken = bin2hex(random_bytes(16));
$claim = CloudJob::where('id', (int) $child->id)
->where('phase', (string) $child->phase)
->where('dispatch_attempts', (int) $child->dispatch_attempts);
if ($child->lease_token === null) {
$claim->whereNull('lease_token');
} else {
$claim->where('lease_token', (string) $child->lease_token);
}
if ($claim->update([
'lease_token' => $childToken,
'lease_until' => time() + self::LEASE_SECONDS,
'updated_at' => time(),
]) === 1) {
$crossed = CloudDispatchLedger::mayHaveDispatched($child);
$failureCode = CloudDispatchLedger::isInvalid($child)
? CloudDispatchLedger::CONFLICT_CODE
: 'aggregate_finalization_failed';
self::forceTerminalFailure(
(int) $child->id,
$childToken,
$crossed,
!$crossed,
$failureCode
);
}
}
if (self::releaseAggregateParentWhenChildrenActive(
(int) $parent->id,
$token
)) {
return false;
}
(new CloudJob())->getConnection()->transaction(
static function () use ($parent, $token): void {
$fresh = CloudJob::where('id', (int) $parent->id)
->where('lease_token', $token)
->lockForUpdate()->first();
if ($fresh === null
|| (string) $fresh->phase === 'terminal') {
return;
}
CloudUserIsoService::expireAggregateLocked($fresh);
CloudTemplateCatalogService::expireAggregateLocked($fresh);
$fresh = CloudJob::where('id', (int) $parent->id)
->where('lease_token', $token)->lockForUpdate()->first();
if ($fresh !== null
&& (string) $fresh->phase !== 'terminal') {
CloudJobService::completeAggregateLocked(
$fresh,
'failed',
true,
false,
'aggregate_finalization_failed'
);
}
},
3
);
return true;
}
/** Release the parent lease while any child can still execute. */
private static function releaseAggregateParentWhenChildrenActive(
int $parentId,
string $token
): bool {
if (!CloudJob::where('parent_job_id', $parentId)
->where('phase', '<>', 'terminal')->exists()) {
return false;
}
CloudJob::where('id', $parentId)
->where('lease_token', $token)
->where('phase', '<>', 'terminal')->update([
'lease_token' => null,
'lease_until' => 0,
'updated_at' => time(),
]);
return true;
}
/**
* A parent deadline is an absolute local boundary. Close every remaining
* child without contacting the Agent, let resource finalizers consume
* those failures, then close any aggregate that still has no result.
*/
private static function expireAggregate(CloudJob $parent,
string $token): void
{
$children = CloudJob::where('parent_job_id', (int) $parent->id)
->where('phase', '<>', 'terminal')->orderBy('id')->get();
foreach ($children as $child) {
if ($child->host_id !== null
&& CloudDispatchLedger::isDispatched($child)) {
$event = CloudOperationCompletionService::pendingForJob(
$child
);
if ($event !== null) {
try {
(new self())->settleCompletionReceipt(
(int) $child->id,
(string) $event->event_id
);
} catch (\Throwable $ignored) {
// A malformed/unapplicable receipt cannot keep the
// public parent active past its absolute deadline.
}
$child = CloudJob::find((int) $child->id) ?? $child;
if ((string) $child->phase === 'terminal') {
continue;
}
}
}
$childToken = bin2hex(random_bytes(16));
$claim = CloudJob::where('id', (int) $child->id)
->where('phase', (string) $child->phase)
->where('dispatch_attempts', (int) $child->dispatch_attempts);
if ($child->lease_token === null) {
$claim->whereNull('lease_token');
} else {
$claim->where('lease_token', (string) $child->lease_token);
}
$claimed = $claim->update([
'lease_token' => $childToken,
'lease_until' => time() + self::LEASE_SECONDS,
'updated_at' => time(),
]);
if ($claimed !== 1) {
continue;
}
$crossed = CloudDispatchLedger::mayHaveDispatched($child);
$failureCode = CloudDispatchLedger::isInvalid($child)
? CloudDispatchLedger::CONFLICT_CODE
: 'aggregate_deadline_exceeded';
self::forceTerminalFailure((int) $child->id, $childToken,
$crossed, !$crossed, $failureCode);
}
if (CloudJob::where('parent_job_id', (int) $parent->id)
->where('phase', '<>', 'terminal')->exists()) {
// A child changed concurrently after our snapshot. Keep the parent
// active and release this local lease; a later bounded fallback
// pass will close the child first. Never hide active child work
// behind a terminal parent.
CloudJob::where('id', (int) $parent->id)
->where('lease_token', $token)
->where('phase', '<>', 'terminal')->update([
'lease_token' => null,
'lease_until' => 0,
'updated_at' => time(),
]);
return;
}
(new CloudJob())->getConnection()->transaction(
static function () use ($parent, $token): void {
$fresh = CloudJob::where('id', (int) $parent->id)
->where('lease_token', $token)
->lockForUpdate()->first();
if ($fresh === null
|| (string) $fresh->phase === 'terminal') {
return;
}
CloudUserIsoService::expireAggregateLocked($fresh);
CloudTemplateCatalogService::expireAggregateLocked($fresh);
$fresh = CloudJob::where('id', (int) $parent->id)
->where('lease_token', $token)->lockForUpdate()->first();
if ($fresh !== null
&& (string) $fresh->phase !== 'terminal') {
CloudJobService::completeAggregateLocked(
$fresh, 'failed', true, false,
'aggregate_deadline_exceeded'
);
}
}, 3
);
}
private static function policyDesired(?CloudInstance $instance, string $kind): bool
{
if ($instance === null) {
return false;
}
$status = (string) $instance->status;
return match ($kind) {
'service_suspend' => in_array($status, ['active', 'deleting'], true)
&& (string) $instance->service_status === 'purge_stopping',
'network_suspend' => $status === 'active'
&& in_array((string) $instance->network_status,
['blocking', 'blocking_expired'], true),
'network_resume' => $status === 'active'
&& in_array((string) $instance->network_status,
['restore_pending', 'restoring'], true),
default => false,
};
}
private static function validPreparedRecoveryMarker(string $marker): bool
{
if (strlen($marker) > 500) {
return false;
}
return preg_match(
'/^prepared:[a-z][a-z0-9_]{2,63}:[A-Za-z0-9_.-]{1,160}:[a-f0-9]{64}$/D',
$marker
) === 1 || preg_match(
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:[1-9][0-9]{0,9}:prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64}$/D',
$marker
) === 1;
}
/**
* Return the immutable prepared-evidence binding without the mutable
* automatic retry counter. Older prepared envelopes stored the complete
* automatic marker, while new envelopes store only this stable suffix.
*/
private static function preparedRecoveryBinding(string $marker): ?string
{
if (preg_match(
'/^(?:automatic_recovery:v1:[1-9][0-9]{0,10}:'
. '[1-9][0-9]{0,9}:)?'
. '(prepared:[a-z][a-z0-9_]{2,63}:'
. '[A-Za-z0-9_.-]{1,160}:[a-f0-9]{64})$/D',
$marker,
$matches
) !== 1) {
return null;
}
return $matches[1];
}
/**
* Capability URLs are needed only for the one Agent dispatch. Once exact
* operation identity and recovery evidence have been validated, retaining
* hundreds of expired multipart URLs would make the prepared recovery
* envelope unnecessarily large. The persisted request fingerprint still
* binds the complete original wire request.
*/
private static function compactPreparedRecoveryPayload(
string $kind,
array $payload
): array {
$identity = $payload[
CloudOperationRecoveryIdentity::payloadKey()
] ?? null;
if ($kind === 'backup_create') {
return [
CloudOperationRecoveryIdentity::payloadKey() => $identity,
'owner_id' => $payload['owner_id'] ?? null,
'can_manage' => $payload['can_manage'] ?? null,
'backup_key' => $payload['backup_key'] ?? null,
'maximum_size_bytes' =>
$payload['maximum_size_bytes'] ?? null,
];
}
if ($kind === 'backup_restore') {
return CloudPortableRestoreJobEnvelope::compactRecoveryPayload(
$payload
);
}
if ($kind === 'instance_migration_prepare') {
$body = is_array($payload['body'] ?? null)
? $payload['body'] : [];
return [
CloudOperationRecoveryIdentity::payloadKey() => $identity,
'owner_id' => $payload['owner_id'] ?? null,
'can_manage' => $payload['can_manage'] ?? null,
'maintenance_migration_id' =>
$payload['maintenance_migration_id'] ?? null,
'body' => [
'ticket' => $body['ticket'] ?? null,
],
];
}
if ($kind === 'instance_migration_transfer') {
$body = is_array($payload['body'] ?? null)
? $payload['body'] : [];
return [
CloudOperationRecoveryIdentity::payloadKey() => $identity,
'owner_id' => $payload['owner_id'] ?? null,
'can_manage' => $payload['can_manage'] ?? null,
'reservation_id' => $payload['reservation_id'] ?? null,
'maintenance_migration_id' =>
$payload['maintenance_migration_id'] ?? null,
'body' => [
'vcpu' => $body['vcpu'] ?? null,
'memory_mib' => $body['memory_mib'] ?? null,
'disk_gib' => $body['disk_gib'] ?? null,
'bandwidth_mbps' => $body['bandwidth_mbps'] ?? null,
'public_network' => $body['public_network'] ?? null,
'vpc_port' => $body['vpc_port'] ?? null,
],
];
}
if ($kind === 'backup_recover') {
$body = is_array($payload['body'] ?? null)
? $payload['body'] : [];
return [
CloudOperationRecoveryIdentity::payloadKey() => $identity,
'owner_id' => $payload['owner_id'] ?? null,
'can_manage' => $payload['can_manage'] ?? null,
'backup_key' => $payload['backup_key'] ?? null,
'recovery_key' => $payload['recovery_key'] ?? null,
'body' => [
'backup_key' => $body['backup_key'] ?? null,
'source_size_bytes' =>
$body['source_size_bytes'] ?? null,
'source_sha256' => $body['source_sha256'] ?? null,
],
];
}
return $payload;
}
private static function lateReceiptRecoveryMarker(string $marker): bool
{
return preg_match(
'/^(?:automatic_recovery:v1:[1-9][0-9]{0,10}:[1-9][0-9]{0,9}:)?prepared:late_completion_receipt:[A-Za-z0-9_.-]{1,160}:[a-f0-9]{64}$/D',
$marker
) === 1;
}
private static function preparedRecoveryTransitionAllowed(
string $current,
string $prepared,
array $operation
): bool {
if ($current === '' || hash_equals($prepared, $current)) {
return true;
}
$legacyOperationHash = CloudOperationRecoveryService::
legacyCompletionOperationHash($operation);
$currentLate = [];
$preparedLate = [];
if ($legacyOperationHash !== null
&& preg_match(
'/^(?:automatic_recovery:v1:[1-9][0-9]{0,10}:'
. '[1-9][0-9]{0,9}:)?'
. 'prepared:late_completion_receipt:'
. '([A-Za-z0-9_.-]{1,160}):([a-f0-9]{64})$/D',
$current,
$currentLate
) === 1
&& preg_match(
'/^prepared:late_completion_receipt:'
. '([A-Za-z0-9_.-]{1,160}):([a-f0-9]{64})$/D',
$prepared,
$preparedLate
) === 1
&& hash_equals($currentLate[1], $preparedLate[1])
&& hash_equals($currentLate[2], $legacyOperationHash)
&& hash_equals(
$preparedLate[2],
CloudOperationRecoveryService::preparedOperationHash(
$operation
)
)) {
return true;
}
if (preg_match(
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:[1-9][0-9]{0,9}:pending$/D',
$current
) === 1) {
if (preg_match(
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:1:prepared:automatic_(applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}:[a-f0-9]{64}$/D',
$prepared,
$next
) === 1) {
return ($next[1] === 'applied'
&& ($operation['status'] ?? null) === 'succeeded')
|| ($next[1] === 'not_applied'
&& ($operation['status'] ?? null) === 'failed');
}
return preg_match(
'/^prepared:late_completion_receipt:[A-Za-z0-9_.-]{1,160}:[a-f0-9]{64}$/D',
$prepared
) === 1;
}
if (preg_match(
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:'
. '[1-9][0-9]{0,9}:prepared:automatic_'
. '(applied|not_applied)_(succeeded|failed)'
. '_(fence|nofence):[a-f0-9]{64}:([a-f0-9]{64})$/D',
$current,
$existing
) !== 1) {
return false;
}
if (preg_match(
'/^prepared:late_completion_receipt:[A-Za-z0-9_.-]{1,160}:([a-f0-9]{64})$/D',
$prepared,
$late
) !== 1) {
return false;
}
$operationHashMatches = hash_equals($existing[4], $late[1]);
if (!$operationHashMatches && $legacyOperationHash !== null) {
$operationHashMatches = hash_equals(
$existing[4],
$legacyOperationHash
);
}
return $existing[3] === 'nofence'
&& $operationHashMatches
&& (($existing[1] === 'applied'
&& $existing[2] === 'succeeded'
&& ($operation['status'] ?? null) === 'succeeded')
|| ($existing[1] === 'not_applied'
&& $existing[2] === 'failed'
&& ($operation['status'] ?? null) === 'failed'));
}
private static function operationErrorCode(array $operation): string
{
$code = is_string($operation['error_code'] ?? null)
? strtolower(trim($operation['error_code'])) : '';
if ($code === '' && is_array($operation['error'] ?? null)
&& is_string($operation['error']['code'] ?? null)) {
$code = strtolower(trim($operation['error']['code']));
}
return preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $code) === 1
? $code : '';
}
/**
* Validate the common immutable job envelope used by every automatic
* recovery strategy. Kind-specific business validation remains in
* finish(), which is the one authoritative ledger finalizer.
*
* @return array{payload:array,instance:?CloudInstance}
*/
public static function recoveryContext(CloudJob $job): array
{
CloudDispatchLedger::assertValid($job);
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) $job->kind
)
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| (int) $job->dispatch_attempts !== 1
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown') {
throw new CloudRecoveryInstanceStateConflict(
'任务不处于可自动核对状态',
409
);
}
if (!is_string($job->operation_id)
|| preg_match('/^[a-f0-9]{32}$/D', $job->operation_id) !== 1) {
throw new CloudRecoveryOperationIdentityConflict(
'任务缺少合法的 Agent 操作标识',
409
);
}
if ((int) $job->host_id < 1
|| $job->dispatched_at === null
|| (int) $job->dispatched_at < 1) {
throw new CloudRecoveryInstanceStateConflict(
'任务缺少宿主机或派发时间',
409
);
}
try {
$payload = CloudSecrets::decrypt(
(string) $job->sealed_payload,
'job-payload'
);
} catch (\Throwable $error) {
throw new CloudRecoveryPayloadUnavailable(
'任务恢复数据无法解密',
409
);
}
if (!is_array($payload) || array_is_list($payload)) {
throw new CloudRecoveryPayloadUnavailable(
'任务恢复数据结构不完整',
409
);
}
$instance = null;
if ($job->instance_id !== null) {
$instance = CloudInstance::find((int) $job->instance_id);
if ($instance === null
|| (int) $instance->host_id !== (int) $job->host_id
|| !is_string($instance->uuid)
|| preg_match(
'/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/D',
strtolower((string) $instance->uuid)
) !== 1) {
throw new CloudRecoveryInstanceStateConflict(
'任务与实例账本不匹配',
409
);
}
}
return compact('payload', 'instance');
}
/**
* Return the exact normalized operation durably stored with a prepared
* recovery. This is encrypted at rest and bound to both kind and marker.
*/
public static function preparedRecoveryOperation(CloudJob $job): ?array
{
$context = self::recoveryContext($job);
$envelope = $context['payload']['_operation_recovery'] ?? null;
if ($envelope === null) {
return null;
}
$operation = is_array($envelope)
? ($envelope['operation'] ?? null) : null;
$storedMarker = is_array($envelope)
&& is_string($envelope['prepared_marker'] ?? null)
? self::preparedRecoveryBinding($envelope['prepared_marker'])
: null;
$currentMarker = self::preparedRecoveryBinding(
(string) $job->effect_resolution_reason
);
if (!is_array($envelope) || array_is_list($envelope)
|| array_diff(array_keys($envelope), [
'version', 'kind', 'prepared_marker', 'operation',
]) !== []
|| ($envelope['version'] ?? null) !== 1
|| ($envelope['kind'] ?? null) !== (string) $job->kind
|| $storedMarker === null || $currentMarker === null
|| !hash_equals($currentMarker, $storedMarker)
|| !is_array($operation) || array_is_list($operation)
|| ($operation['operation_id'] ?? null)
!== (string) $job->operation_id
|| !in_array($operation['status'] ?? null,
['succeeded', 'failed'], true)) {
throw new CloudRecoveryPayloadUnavailable(
'任务预提交恢复数据不完整',
409
);
}
return $operation;
}
/**
* Validate PHP's encrypted settings intent before recovery evidence can
* affect ledgers. Both live inspection and immutable receipt recovery use
* this exact contract so neither path can silently finalize an empty or
* damaged escrow.
*
* @return array{instance:CloudInstance,payload:array,body:array,fields:array}
*/
public static function settingsRecoveryContext(CloudJob $job): array
{
if ((string) $job->kind !== 'instance_settings') {
throw new CloudError('任务不处于可自动核对状态', 409);
}
$context = self::recoveryContext($job);
$instance = $context['instance'];
if ($instance === null
|| (string) $instance->status === 'deleted'
) {
throw new CloudRecoveryInstanceStateConflict(
'任务与实例账本不匹配',
409
);
}
$payload = $context['payload'];
$body = $payload['body'] ?? null;
$allowedBody = [
'hostname', 'root_password', 'ssh_public_key',
'confirm_restart',
];
if (!is_array($body) || array_is_list($body)
|| array_diff(array_keys($body), $allowedBody) !== []
|| ($body['confirm_restart'] ?? null) !== true
|| !in_array(strtoupper((string) ($payload['method'] ?? '')),
['POST', 'PATCH'], true)
|| ($payload['request_target'] ?? null) !== '/v1/vms/'
. rawurlencode((string) $instance->uuid) . '/settings'
|| (int) ($payload['owner_id'] ?? 0)
!== (int) $instance->user_id
|| ((int) $job->user_id !== (int) $instance->user_id
&& ($payload['can_manage'] ?? null) !== true)) {
throw new CloudError('实例设置恢复请求记录无效', 409);
}
$fields = [];
foreach ([
'hostname' => 'hostname',
'root_password' => 'password',
'ssh_public_key' => 'ssh_key',
] as $bodyField => $code) {
if (array_key_exists($bodyField, $body)) {
$fields[] = $code;
}
}
if ($fields === []
|| (array_key_exists('settings_fields', $payload)
&& $payload['settings_fields'] !== $fields)) {
throw new CloudError('实例设置恢复字段记录无效', 409);
}
if (array_key_exists('hostname', $body)
&& CloudInput::hostname($body['hostname'])
!== $body['hostname']) {
throw new CloudError('实例设置恢复主机名记录无效', 409);
}
if (array_key_exists('root_password', $body)
&& (!is_string($body['root_password'])
|| CloudInput::password($body['root_password'])
!== $body['root_password'])) {
throw new CloudError('实例设置恢复密码记录无效', 409);
}
if (array_key_exists('ssh_public_key', $body)) {
if (!is_string($body['ssh_public_key'])) {
throw new CloudError('实例设置恢复 SSH 公钥记录无效', 409);
}
$parsedKey = CloudSSHKeys::parse($body['ssh_public_key']);
if (($parsedKey['public_key'] ?? null)
!== $body['ssh_public_key']) {
throw new CloudError('实例设置恢复 SSH 公钥记录无效', 409);
}
}
$credential = $payload['credential'] ?? null;
if ($credential !== null) {
if (!is_array($credential) || array_is_list($credential)
|| array_diff(
array_keys($credential),
['password', 'login_name', 'auth_mode']
) !== []
|| ($credential['login_name'] ?? null) !== 'root'
|| !in_array(
$credential['auth_mode'] ?? null,
[
'password', 'password_and_key', 'ssh_key',
'ssh_key_password',
],
true
)) {
throw new CloudError('实例设置恢复凭据记录无效', 409);
}
$savedPassword = $credential['password'] ?? null;
if ($savedPassword !== null
&& (!is_string($savedPassword)
|| CloudInput::password($savedPassword)
!== $savedPassword)) {
throw new CloudError('实例设置恢复凭据记录无效', 409);
}
$passwordMode = in_array(
$credential['auth_mode'],
['password', 'password_and_key', 'ssh_key_password'],
true
);
if (($passwordMode && $savedPassword === null)
|| (!$passwordMode && $savedPassword !== null)) {
throw new CloudError('实例设置恢复凭据记录无效', 409);
}
}
$changesCredential = array_key_exists('root_password', $body)
|| array_key_exists('ssh_public_key', $body);
if (!$changesCredential && $credential !== null) {
throw new CloudError('实例设置恢复凭据与请求字段不匹配', 409);
}
if (array_key_exists('ssh_public_key', $body)
&& is_array($credential)
&& !in_array(
$credential['auth_mode'],
['password_and_key', 'ssh_key', 'ssh_key_password'],
true
)) {
throw new CloudError('实例 SSH 公钥恢复凭据模式不匹配', 409);
}
if (array_key_exists('root_password', $body)) {
if (!is_array($credential)
|| ($credential['password'] ?? null)
!== $body['root_password']) {
throw new CloudError('实例密码恢复凭据不匹配', 409);
}
}
return compact('instance', 'payload', 'body', 'fields');
}
/**
* Apply an already-validated recovery result to PHP-owned ledgers only.
* The original Agent mutation is never dispatched from this path.
*/
public function prepareRecoveredEffect(
int $jobId,
array $operation,
string $preparedMarker
): CloudJob {
if (!self::validPreparedRecoveryMarker($preparedMarker)) {
throw new CloudError('任务效果恢复标记无效', 409);
}
$job = CloudJob::find($jobId);
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) $job->kind
)) {
throw new CloudError('任务类型不支持自动效果恢复', 409);
}
$context = (string) $job->kind === 'instance_settings'
? self::settingsRecoveryContext($job)
: self::recoveryContext($job);
$payload = $context['payload'];
$normalized = self::normalizeAgentOperation(
self::operationForJob($job, $operation)
);
try {
$this->finish($job, $normalized, $payload, $preparedMarker);
} catch (CloudRecoveryPayloadUnavailable
| CloudRecoveryEvidenceConflict
| CloudRecoveryReceiptInvalid
| CloudRecoveryOperationIdentityConflict
| CloudRecoveryAggregateStateConflict
| CloudRecoveryInstanceStateConflict $error) {
throw $error;
} catch (CloudError $error) {
if ($job->instance_id !== null) {
throw new CloudRecoveryInstanceStateConflict(
'实例或关联资源账本与原任务证据不一致',
409
);
}
throw $error;
}
return CloudJob::find($jobId) ?? $job;
}
/**
* Materialize one paid create from its exact immutable success receipt.
*
* This is deliberately not part of the generic recovery policy: it never
* inspects or replays an Agent mutation. The original failed task remains
* terminal for audit, while its create ledgers and receipt ACK commit as
* one local database transaction.
*/
public function settleRecoveredCreateCompletion(
int $jobId,
\App\Models\CloudAgentCompletionEvent $event
): CloudJob {
$job = CloudJob::find($jobId);
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
if ((string) $job->kind !== 'create') {
throw new CloudError('任务类型不支持开通回执恢复', 409);
}
try {
$payload = CloudSecrets::decrypt(
(string) $job->sealed_payload,
'job-payload'
);
} catch (\Throwable $error) {
throw new CloudRecoveryPayloadUnavailable(
'任务加密恢复数据暂时无法读取',
409
);
}
$operation = self::normalizeAgentOperation(
self::operationForJob(
$job,
CloudOperationCompletionService::operation($event)
)
);
$this->finish(
$job,
$operation,
$payload,
null,
(string) $event->event_id
);
return CloudJob::find($jobId) ?? $job;
}
/**
* Retry only the PHP ledger finalizer for an operation proven not sent.
* The terminal task receipt remains immutable and no Agent request occurs.
*/
public function retryUndispatchedFinalization(
int $jobId,
string $leaseToken
): CloudJob {
$job = CloudJob::find($jobId);
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
if ((string) $job->kind === 'create') {
CloudCreateFailureFinalizer::retryUndispatchedFailure(
$jobId,
$leaseToken
);
return CloudJob::find($jobId) ?? $job;
}
try {
$payload = CloudSecrets::decrypt(
(string) $job->sealed_payload,
'job-payload'
);
} catch (\Throwable $error) {
throw new CloudRecoveryPayloadUnavailable(
'任务恢复数据无法解密',
409
);
}
if (!is_array($payload) || array_is_list($payload)) {
throw new CloudRecoveryPayloadUnavailable(
'任务恢复数据结构不完整',
409
);
}
$operation = CloudOperationRecoveryPolicy::isCreateLike(
(string) $job->kind
) ? self::createNotDispatchedFailure() : ['status' => 'failed'];
$operation['operation_id'] = (string) $job->operation_id;
$operation['error_code'] = 'dispatch_not_sent';
$operation['effect_may_have_applied'] = false;
$operation['safe_to_resubmit'] = true;
$this->finish(
$job,
$operation,
$payload,
null,
null,
$leaseToken
);
return CloudJob::find($jobId) ?? $job;
}
private function finish(
CloudJob $claimed,
array $operation,
array $payload,
?string $preparedRecoveryMarker = null,
?string $recoveredCreateEventId = null,
?string $localFinalizerToken = null
): void
{
if ((string) $claimed->execution_mode === 'agent') {
CloudDispatchLedger::assertValid($claimed);
}
$operation['operation_id'] = $operation['operation_id']
?? (string) $claimed->operation_id;
$status = self::disposition(
$operation,
CloudOperationRecoveryPolicy::isCreateLike(
(string) $claimed->kind
),
(string) $claimed->kind);
if (!in_array($status, ['succeeded', 'failed'], true)) {
throw new CloudError('任务尚未取得终态回执', 409);
}
$operationIdentity = null;
// The Agent triple exists only after the immutable mutation crossed
// the dispatch boundary. A locally rejected, never-dispatched task is
// already proven not-applied and must still reach its kind finalizer;
// requiring an Agent identity there would turn damaged local input
// into an unnecessary unknown-effect fence.
if (CloudDispatchLedger::isDispatched($claimed)
&& CloudOperationRecoveryIdentity::supports(
(string) $claimed->kind
)) {
$operationIdentity = self::operationIdentityForFinalization(
$claimed,
$payload,
$status === 'succeeded',
$preparedRecoveryMarker !== null
);
$payload[CloudOperationRecoveryIdentity::payloadKey()] =
CloudOperationRecoveryIdentity::persisted(
$operationIdentity
);
}
if ($status === 'succeeded'
&& in_array((string) $claimed->kind,
['delete', 'expiry_delete', 'delete_force'], true)
&& !self::deleteSuccessConfirmed($operation,
(string) $claimed->kind)) {
throw new CloudError('Agent 删除回执缺少资源已清理证据', 502);
}
if ((string) $claimed->kind === 'reinstall'
&& self::reinstallNeedsReview($operation)) {
throw new CloudError('Agent 重装回执未确认清理完成', 502);
}
if ($status === 'failed') {
$this->observedErrors[] = 'task_failed';
}
$forceRequired = $status === 'failed'
&& in_array((string) $claimed->kind,
['delete', 'expiry_delete', 'service_suspend'], true)
&& self::operationErrorCode($operation) === 'force_required';
$effectMayHaveApplied = $status === 'failed'
&& ($operation['effect_may_have_applied'] ?? false) === true;
$safeToResubmit = $status === 'failed'
&& ($operation['safe_to_resubmit'] ?? false) === true;
if ($forceRequired) {
$effectMayHaveApplied = false;
$safeToResubmit = false;
}
if ($effectMayHaveApplied) {
$safeToResubmit = false;
}
if ($status === 'failed'
&& CloudOperationRecoveryPolicy::isCreateLike(
(string) $claimed->kind
)) {
if (self::createFailureConfirmedAbsent($operation)) {
// Exact absence evidence is the authority for financial and
// placement rollback. safe_to_resubmit is informational only;
// this generation is never dispatched again.
$effectMayHaveApplied = false;
} else {
// Boolean transport hints alone cannot release a paid order.
// Keep every resource fenced until a fresh read-only host
// inspection proves both the guest and artifacts are absent.
$effectMayHaveApplied = true;
$safeToResubmit = false;
}
}
// Downstream error classification must use the validated PHP safety
// decision, never the Agent's original booleans. In particular an
// ambiguous create must not be labelled as refunded.
$operation['effect_may_have_applied'] = $effectMayHaveApplied;
$operation['safe_to_resubmit'] = $safeToResubmit;
$resourceStatus = $status === 'failed' && $effectMayHaveApplied
? 'uncertain' : $status;
$result = $operation['result'] ?? null;
CloudPortableRestoreFinalizer::recordEvidence(
$claimed,
$resourceStatus,
$result,
$payload
);
if ((string) $claimed->kind === 'network_change'
&& $claimed->public_network_transition_id !== null) {
CloudPublicNetworkTransitionWorkflow::recordEvidence(
$claimed,
$resourceStatus,
$result,
$payload,
$operation,
$preparedRecoveryMarker !== null
);
}
$aggregateParentId = (int) ($claimed->parent_job_id ?? 0);
$wakeAutomaticBackup = false;
(new CloudJob())->getConnection()->transaction(function () use ($claimed, $payload, $status, $resourceStatus, $effectMayHaveApplied, $safeToResubmit, $forceRequired, $result, $operation, $preparedRecoveryMarker, $operationIdentity, $recoveredCreateEventId, $localFinalizerToken, $aggregateParentId, &$wakeAutomaticBackup) {
// All financial transactions acquire the wallet first, avoiding wallet/order deadlocks.
$user = User::where('id', $claimed->user_id)->lockForUpdate()->first();
$order = $claimed->order_id === null ? null : CloudOrder::where('id', $claimed->order_id)->lockForUpdate()->first();
// Aggregate recovery and the legacy sweep both use the same
// parent -> children lock order. Never lock a leaf first and then
// wait for its parent: concurrent sibling completions otherwise
// form a deadlock cycle with the parent-led sweep.
$aggregateParent = null;
if ($aggregateParentId > 0) {
$aggregateParent = CloudJob::where('id', $aggregateParentId)
->lockForUpdate()->first();
if ($aggregateParent === null) {
throw new CloudRecoveryAggregateStateConflict(
'任务的汇总父任务不存在',
409
);
}
CloudTemplateCatalogService::lockAggregateCatalogLocked(
$aggregateParent
);
CloudUserIsoService::lockAggregateCatalogLocked(
$aggregateParent
);
}
$job = CloudJob::where('id', $claimed->id)->lockForUpdate()->first();
if ($job === null) {
throw new CloudError('任务不存在', 404);
}
if ((string) $job->execution_mode === 'agent') {
CloudDispatchLedger::assertValid($job);
if (CloudDispatchLedger::attempts($job)
!== CloudDispatchLedger::attempts($claimed)) {
throw new CloudRecoveryDispatchLedgerConflict(
'任务单次派发代次在收敛期间发生变化',
409
);
}
}
if ((int) ($job->parent_job_id ?? 0) !== $aggregateParentId) {
throw new CloudRecoveryAggregateStateConflict(
'任务的汇总父级关系在收敛期间发生变化',
409
);
}
$localFinalizerRecovery = $localFinalizerToken !== null;
if ($localFinalizerRecovery
&& (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
(string) $job->kind
)
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| !CloudDispatchLedger::
isLocallyFinalizableNotDispatched($job)
|| !(bool) $job->finalization_incident
|| !in_array((string) $job->effect_resolution,
['unknown', 'not_applied'], true)
|| !is_string($job->lease_token)
|| !hash_equals(
(string) $job->lease_token,
$localFinalizerToken
))) {
throw new CloudError(
'任务不处于可恢复的本地收尾状态',
409
);
}
if ($localFinalizerRecovery) {
if (CloudDispatchLedger::hasRepairIncident($job)) {
throw new CloudRecoveryInstanceStateConflict(
'任务仍处于派发账本修复状态',
409
);
}
CloudOperationCompletionService::
assertLocalNotDispatchedRecoveryState($job);
}
$recoveredCreateEvent = null;
if ($recoveredCreateEventId !== null) {
$recoveredCreateEvent = \App\Models\CloudAgentCompletionEvent::where(
'event_id',
$recoveredCreateEventId
)->lockForUpdate()->first();
if ($preparedRecoveryMarker !== null
|| (string) $job->kind !== 'create'
|| (string) $job->execution_mode !== 'agent'
|| (string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| (int) $job->dispatch_attempts !== 1
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown'
|| $job->order_id === null
|| $order === null
|| $operationIdentity === null
|| $recoveredCreateEvent === null
|| (string) $recoveredCreateEvent->disposition !== 'received'
|| (int) $recoveredCreateEvent->host_id
!== (int) $job->host_id
|| !hash_equals(
(string) $job->operation_id,
(string) $recoveredCreateEvent->operation_id
)
|| (string) $recoveredCreateEvent->outcome !== 'succeeded'
|| (bool) $recoveredCreateEvent->effect_may_have_applied
|| (bool) $recoveredCreateEvent->safe_to_resubmit
|| !CloudOperationCompletionService::isExactStoredEvent(
$recoveredCreateEvent
)) {
throw new CloudError(
'开通完成回执不处于可安全收敛状态',
409
);
}
$lockedIdentity = CloudOperationRecoveryIdentity::
expectedFromPayload($job, $payload, true);
CloudOperationRecoveryIdentity::assertMatches(
$lockedIdentity,
CloudOperationCompletionService::identity(
$recoveredCreateEvent
)
);
} elseif ($preparedRecoveryMarker !== null) {
CloudOperationRecoveryService::
assertRecoveryIncidentAllowsPrepare($job);
if ((string) $job->phase !== 'terminal'
|| (string) $job->outcome !== 'failed'
|| (string) $job->status !== 'failed'
|| !(bool) $job->effect_may_have_applied
|| (bool) $job->safe_to_resubmit
|| (string) $job->effect_resolution !== 'unknown') {
throw new CloudError('任务已不再等待效果恢复', 409);
}
if (!self::preparedRecoveryTransitionAllowed(
(string) $job->effect_resolution_reason,
$preparedRecoveryMarker,
$operation
)) {
if (self::lateReceiptRecoveryMarker(
$preparedRecoveryMarker
)) {
throw new CloudRecoveryEvidenceConflict(
'任务已有冲突的效果恢复证据',
409
);
}
throw new CloudError('任务已有不同的效果恢复证据', 409);
}
if (hash_equals(
$preparedRecoveryMarker,
(string) $job->effect_resolution_reason
)) {
if (self::lateReceiptRecoveryMarker(
$preparedRecoveryMarker
) && in_array(
(string) ($job->finalization_error_code ?? ''),
[
'recovery_limit_reached',
'recovery_state_invalid',
],
true
)) {
$job->finalization_incident = 0;
$job->finalization_error_code = null;
$job->updated_at = time();
$job->save();
}
return;
}
} elseif (!$localFinalizerRecovery
&& ($job->lease_token !== $claimed->lease_token
|| (string) $job->phase === 'terminal')) {
return;
}
self::assertRescueResultMatchesRequest(
$job,
$status,
$result,
$payload
);
$safe = is_array($result)
? CloudOperationErrorCatalog::workerResult($result) : null;
$causeCode = CloudOperationErrorCatalog::agentCauseCode(
$operation['cause_code'] ?? null
);
if ($causeCode !== null) {
$durableResult = is_array($result) && !array_is_list($result)
? $result : [];
// Keep the stable cause beside the sanitized operation
// evidence. It is never copied into the public error_code or
// ordinary user DTOs.
$durableResult['cause_code'] = $causeCode;
$safe = CloudOperationErrorCatalog::workerResult($durableResult);
}
$catalogStatus = $resourceStatus;
// Catalog/configuration mutations use the public terminal status,
// never the legacy resource-level "uncertain" label. Their
// replica rows and aggregate parent are settled in this same
// transaction as the child job.
CloudTemplateCatalogService::finishJobLocked(
$job, $status, $result, $payload, $operation,
$preparedRecoveryMarker !== null || $localFinalizerRecovery
);
CloudUserIsoService::finishJobLocked(
$job, $status, $result, $payload, $operation,
$preparedRecoveryMarker !== null || $localFinalizerRecovery
);
CloudRescueTemplateReplicaCoordinator::finishJobLocked(
$job, $status, $result, $payload, $operation,
$preparedRecoveryMarker !== null || $localFinalizerRecovery
);
CloudRescueTemplateRetirementService::finishJobLocked(
$job, $status, $result, $payload, $operation,
$preparedRecoveryMarker !== null || $localFinalizerRecovery
);
CloudHostStorageConfigurationService::finishJobLocked(
$job, $status, $result, $operation
);
CloudSnapshotService::finishJobLocked(
$job, $catalogStatus, $result, $payload
);
CloudBackupService::finishJobLocked(
$job,
$catalogStatus,
$result,
$payload,
$preparedRecoveryMarker !== null,
$operation
);
CloudSnapshotService::finishInstanceDeleteLocked(
$job, $catalogStatus
);
CloudFirewallService::finishJobLocked(
$job, $catalogStatus, $result, $payload, $operation
);
CloudVpcPortService::finishJobLocked(
$job, $catalogStatus, $result, $payload, $operation
);
CloudInstanceMigrationService::finishJobLocked(
$job, $catalogStatus, $result, $payload
);
CloudFirewallService::finishInstanceDeleteLocked(
$job, $catalogStatus
);
if ($job->kind === 'instance_upgrade'
&& in_array($catalogStatus,
['succeeded', 'failed', 'uncertain'], true)) {
CloudPlanUpgradeService::finishJobLocked(
$job,
$catalogStatus,
$result,
$payload,
$user
);
} elseif ($job->kind === 'network_change'
&& $job->public_network_transition_id !== null
&& in_array($catalogStatus,
['succeeded', 'failed', 'uncertain'], true)) {
CloudPublicNetworkTransitionWorkflow::finishJobLocked(
$job,
$catalogStatus,
$payload,
$user,
$preparedRecoveryMarker !== null,
$aggregateParent
);
} elseif ($status === 'succeeded' && $job->kind === 'create') {
$vm = is_array($result) ? ($result['vm'] ?? $result) : [];
$uuid = $vm['uuid'] ?? '';
if ($order === null || !is_string($uuid)
|| $operationIdentity === null
|| !hash_equals(
(string) $operationIdentity['resource_id'],
strtolower($uuid)
)
|| preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD', $uuid) !== 1) {
throw new CloudError('Agent 成功结果缺少虚拟机标识', 502);
}
$orderSpec = json_decode((string) $order->spec_json, true);
try {
$frozenPublicNetwork = CloudPublicNetworkSnapshot::fromArray(
is_array($orderSpec)
? ($orderSpec['public_network'] ?? null) : null
);
$requestedPublicNetwork =
CloudPublicNetworkSnapshot::fromArray(
is_array($payload['body'] ?? null)
? ($payload['body']['public_network'] ?? null)
: null
);
$confirmedPublicNetwork =
CloudPublicNetworkSnapshot::fromArray(
$vm['public_network'] ?? null
);
} catch (\InvalidArgumentException) {
throw new CloudError(
'Agent 成功结果缺少有效的双栈公网快照,必须人工核查',
502
);
}
if (!hash_equals(
$frozenPublicNetwork->fingerprint(),
$requestedPublicNetwork->fingerprint()
) || !hash_equals(
$requestedPublicNetwork->fingerprint(),
$confirmedPublicNetwork->fingerprint()
)) {
// A completion result may describe a VM whose network was
// changed out of band after dispatch. Never commit either
// original address in that case; retain the whole bundle.
throw new CloudError(
'Agent 成功结果与预留双栈公网不一致,必须人工核查',
502
);
}
$provisionInvoice = CloudInvoice::where(
'order_id',
(int) $order->id
)->lockForUpdate()->first();
if (!in_array((string) $order->status,
['queued', 'failed', 'uncertain'], true)
|| $order->refunded_at !== null
|| !self::createStockLedgerIsConsistent($order)
|| $order->instance_id !== null
|| $provisionInvoice === null
|| (string) $provisionInvoice->purpose !== 'provision'
|| (string) $provisionInvoice->status !== 'paid'
|| $provisionInvoice->paid_at === null
|| (int) $provisionInvoice->order_id !== (int) $order->id
|| (int) $provisionInvoice->user_id !== (int) $order->user_id
|| (int) $provisionInvoice->plan_id !== (int) $order->plan_id
|| (int) $provisionInvoice->offer_id !== (int) $order->offer_id
|| strtoupper((string) $provisionInvoice->currency)
!== strtoupper((string) $order->currency)
|| CloudMoney::cents((string) $provisionInvoice->amount)
!== CloudMoney::cents((string) $order->amount)) {
throw new CloudRecoveryInstanceStateConflict(
'开通成功回执与资金、库存或资源预留账本不一致',
409
);
}
$instance = CloudInstance::where('order_id', $order->id)->first();
$newInstance = $instance === null;
$instance ??= new CloudInstance();
$instance->uuid = $uuid;
$instance->host_id = $order->host_id;
$instance->user_id = $order->user_id;
$instance->order_id = $order->id;
$instance->offer_id = $order->offer_id;
$instance->resource_pool_id = $order->resource_pool_id;
$instance->reservation_id = $order->reservation_id;
$spec = $orderSpec;
$templateMetadata = self::trustedTemplateMetadata($payload);
$spec = array_replace($spec, $templateMetadata);
$now = time();
$days = filter_var($spec['validity_days'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$traffic = filter_var($spec['traffic_quota_bytes'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 0]]);
if ($days === false || $traffic === false) {
throw new CloudError('订单缺少不可变的计费周期快照', 409);
}
$trafficPoolBinding =
CloudTrafficPoolPeriodBinding::orderSnapshot(
$spec,
$traffic
);
$instancePrice = CloudMoney::cents(
$spec['instance_price'] ?? null
);
$instance->plan_id = $order->plan_id;
// A duplicate completion result must not overwrite a display
// name the owner changed after the first success committed.
if ($newInstance) {
$instance->name = $payload['body']['name'];
}
$instance->status = 'active';
$instance->service_status = 'active';
$instance->network_status = 'active';
$instance->expires_at = $now + $days * 86400;
// Optional services are renewed from their own subscription
// ledgers. Keep this field as the base compute price only.
$instance->renewal_price = CloudMoney::decimal(
$instancePrice
);
$instance->renewal_days = $days;
$instance->renewal_traffic_bytes = $traffic;
$instance->traffic_quota_bytes = $traffic;
$instance->traffic_download_bytes = 0;
$instance->traffic_upload_bytes = 0;
$instance->lifetime_download_bytes = 0;
$instance->lifetime_upload_bytes = 0;
$instance->traffic_synced_at = 0;
// Persist only the Agent's slow-changing public metadata.
// SSR pages can then render IP, operating system, login name
// and actual capacity without issuing a host read
// on every request. Billing/order fields remain authoritative
// because the allowlist cannot overwrite them.
$instanceSpec = CloudInstanceMetadata::merge($spec, $vm);
if ($templateMetadata !== null) {
// Agent observations cannot replace PHP's catalog
// snapshot, even if an older Agent happens to return
// similarly named fields.
$instanceSpec = array_replace(
$instanceSpec,
$templateMetadata
);
unset($instanceSpec['install_iso']);
}
// A VM is panel-managed only after the Agent's exact create
// operation has returned the validated success above.
$instanceSpec['public_network'] =
$confirmedPublicNetwork->toArray();
$instanceSpec['managed'] = true;
$instance->spec_json = json_encode(
$instanceSpec,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
if (isset($payload['backup_metadata'])) {
// The restored system disk is authoritative. The user may
// have changed the guest password after provisioning, so
// copying the panel's historical envelope would expose a
// password that is no longer valid.
$instance->sealed_credential = null;
} else {
$instance->sealed_credential = isset($payload['credential'])
? CloudSecrets::encrypt(
$payload['credential'], 'instance-credential'
) : null;
}
// The paid service begins when provisioning actually succeeds,
// not while an opening invoice is waiting for payment. A
// duplicate success must retain the original value.
if (!$instance->exists) {
$instance->created_at = $now;
}
$instance->updated_at = time();
$instance->save();
CloudPlacementService::commit(
$order,
(int) $instance->id,
$confirmedPublicNetwork->toArray()
);
// Seed the independent runtime observation immediately after
// provisioning. If the Agent returned a trusted power
// state, promote it now; otherwise the background host batch
// will provide the first observation without showing "未知".
self::recordRuntime($instance, $vm, 'create', []);
$period = CloudServicePeriod::where('order_id', $order->id)->first()
?? new CloudServicePeriod();
$period->instance_id = $instance->id;
$period->user_id = $order->user_id;
$period->plan_id = $order->plan_id;
$period->order_id = $order->id;
$period->starts_at = $now;
$period->ends_at = $instance->expires_at;
$period->traffic_quota_bytes = $traffic;
CloudTrafficPoolPeriodBinding::bindPeriod(
$period,
$trafficPoolBinding['traffic_pool_group_id'],
$traffic
);
$period->download_base_bytes = 0;
$period->upload_base_bytes = 0;
$period->download_used_bytes = 0;
$period->upload_used_bytes = 0;
$period->status = 'active';
$period->activated_at = $now;
$period->created_at = $period->created_at ?: $now;
$period->updated_at = $now;
$period->save();
$instance->current_period_id = $period->id;
$instance->save();
$backupSubscription =
CloudAutomaticBackupService::activatePurchased($instance);
CloudAutomaticBackupBillingService::grantProvisionEntitlementLocked(
$instance,
$period,
$provisionInvoice,
$backupSubscription,
$now
);
$job->instance_id = $order->instance_id = $instance->id;
$provisionInvoice->instance_id = $instance->id;
$provisionInvoice->updated_at = time();
$provisionInvoice->save();
CloudInvoiceService::queueProvisionSucceeded(
$provisionInvoice, $now
);
$safe = $safe ?? [];
$safe['id'] = (string) $instance->id;
$safe['uuid'] = $uuid;
} elseif ($job->kind === 'reinstall'
&& $job->instance_id !== null
&& $status === 'succeeded') {
$instance = CloudInstance::where('id', $job->instance_id)
->lockForUpdate()->first();
if ($instance === null) {
throw new CloudRecoveryInstanceStateConflict(
'重装任务关联实例已缺失',
409
);
}
$instance->sealed_credential = CloudSecrets::encrypt(
$payload['credential'], 'instance-credential'
);
// Once the Agent proves the replacement guest committed,
// its credential and template snapshot supersede the old
// guest even if deletion of the old volume still requires
// operator review. The task still ends as failed.
$decoded = json_decode((string) $instance->spec_json, true);
if (!is_array($decoded) || array_is_list($decoded)) {
throw new CloudRecoveryInstanceStateConflict(
'重装任务关联实例配置账本损坏',
409
);
}
$templateMetadata = self::trustedTemplateMetadata($payload);
$templateId = is_array($payload['body'] ?? null)
? ($payload['body']['template_id'] ?? null) : null;
if (is_string($templateId)
&& preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D',
$templateId) === 1) {
$decoded['template_id'] = $templateId;
}
$decoded = array_replace($decoded, $templateMetadata);
unset($decoded['install_iso']);
$decoded['managed'] = true;
// Force one fresh capacity/IP observation while the trusted
// PHP presentation snapshot remains usable.
unset($decoded['agent_metadata_version']);
$instance->spec_json = json_encode($decoded,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE);
self::recordRuntime($instance,
is_array($result) ? $result : [],
(string) $job->kind, $payload);
$instance->updated_at = time();
$instance->save();
} elseif ($status === 'failed' && !$effectMayHaveApplied
&& $job->kind === 'create' && $order !== null) {
if ($user === null) {
throw new CloudError('退款账户不存在', 409);
}
CloudCreateFailureFinalizer::refundAndReleaseLocked(
$job,
$user,
$order,
'create_failed'
);
} elseif ($forceRequired && $job->instance_id !== null) {
$instance = CloudInstance::where('id', $job->instance_id)
->lockForUpdate()->first();
if ($instance === null) {
throw new CloudRecoveryInstanceStateConflict(
'删除任务关联实例已缺失', 409
);
}
if ((string) $job->kind === 'delete'
&& (string) $instance->status === 'active') {
// A tenant graceful-delete request deliberately leaves the
// live instance unchanged when the Agent requires a new,
// separately authorized force-delete generation.
} elseif ((string) $instance->status === 'deleting') {
$instance->service_status = 'purge_force_required';
$instance->updated_at = time();
$instance->save();
} elseif ((string) $instance->service_status
!== 'purge_force_required') {
throw new CloudRecoveryInstanceStateConflict(
'删除任务的实例状态与强制阶段矛盾',
409
);
}
} elseif ($status === 'failed' && $job->kind === 'expiry_delete'
&& $job->instance_id !== null) {
$instance = CloudInstance::where('id', $job->instance_id)
->lockForUpdate()->first();
if ($instance !== null && (string) $instance->status !== 'deleted') {
$instance->status = 'delete_failed';
$instance->service_status = 'purge_failed';
$instance->updated_at = time();
$instance->save();
}
} elseif ($status === 'failed' && $job->instance_id !== null
&& in_array($job->kind,
['service_suspend', 'network_suspend', 'network_resume'], true)) {
// Latch a visible policy failure. This generation is never
// queued again automatically.
$instance = CloudInstance::where('id', $job->instance_id)->lockForUpdate()->first();
if ($instance === null) {
throw new CloudRecoveryInstanceStateConflict(
'系统策略任务关联实例已缺失', 409
);
}
$networkPolicyMarker = in_array((string) $job->kind,
['network_suspend', 'network_resume'], true)
? CloudInstanceNetworkPolicy::payloadMarker(
$payload,
(int) $job->instance_id,
(string) $job->operation_id
) : null;
if ($networkPolicyMarker !== null) {
CloudInstanceNetworkPolicy::finishJobLocked(
$instance,
(string) $job->kind,
(string) $job->operation_id,
'failed',
false,
$effectMayHaveApplied,
time()
);
} else {
$expected = [
'service_suspend' =>
['suspending_expired', 'purge_stopping'],
'network_suspend' =>
['blocking', 'blocking_expired'],
'network_resume' =>
['restore_pending', 'restoring'],
][$job->kind];
$already = match ((string) $job->kind) {
'service_suspend' => in_array(
(string) $instance->service_status,
['suspend_failed', 'purge_stop_failed'], true
),
'network_suspend' => (string) $instance->network_status
=== 'block_failed',
'network_resume' => (string) $instance->network_status
=== 'restore_failed',
default => false,
};
$transitioned = false;
if ($job->kind === 'service_suspend'
&& in_array((string) $instance->service_status, $expected, true)) {
if ((string) $instance->status === 'deleting'
&& (string) $instance->service_status === 'purge_stopping') {
// Preserve the irreversible deletion latch, but do
// not enqueue this failed generation again.
$instance->service_status = 'purge_stop_failed';
} else {
$instance->service_status = 'suspend_failed';
}
$transitioned = true;
} elseif ($job->kind === 'network_suspend'
&& in_array((string) $instance->network_status, $expected, true)) {
$instance->network_status = 'block_failed';
$transitioned = true;
} elseif ($job->kind === 'network_resume'
&& in_array((string) $instance->network_status, $expected, true)) {
$instance->network_status = 'restore_failed';
$transitioned = true;
}
if (!$transitioned && !$already) {
throw new CloudRecoveryInstanceStateConflict(
'系统策略失败结论与实例当前状态矛盾',
409
);
}
}
$instance->updated_at = time();
$instance->save();
} elseif ($status === 'succeeded' && $job->instance_id !== null) {
$instance = CloudInstance::where('id', $job->instance_id)->lockForUpdate()->first();
if ($instance === null) {
throw new CloudRecoveryInstanceStateConflict(
'任务成功结论的关联实例已缺失', 409
);
}
$networkPolicyMarker = in_array((string) $job->kind,
['network_suspend', 'network_resume'], true)
? CloudInstanceNetworkPolicy::payloadMarker(
$payload,
(int) $job->instance_id,
(string) $job->operation_id
) : null;
if ($networkPolicyMarker !== null) {
CloudInstanceNetworkPolicy::finishJobLocked(
$instance,
(string) $job->kind,
(string) $job->operation_id,
'succeeded',
is_array($result)
&& ($result['skipped'] ?? false) === true,
false,
time()
);
} elseif (in_array((string) $job->kind,
['delete', 'expiry_delete', 'delete_force'], true)) {
CloudVpcPortService::releaseAfterInstanceDeletedLocked(
$instance
);
CloudInstanceNetworkPolicy::
releaseAfterInstanceDeletedLocked($instance);
$instance->status = 'deleted';
$instance->sealed_credential = null;
CloudBackupSubscription::where(
'instance_id', (int) $instance->id
)->update([
'enabled' => 0,
'renew_next_period' => 0,
'updated_at' => time(),
]);
CloudInstanceResourceRelease::
releaseAfterConfirmedDelete(
$instance,
'instance_deleted'
);
} elseif ($job->kind === 'instance_settings') {
// The Agent must acknowledge the controlled seed
// replacement before PHP changes any cached metadata.
// Passwords and public keys are intentionally never
// copied into spec_json or result_json.
if (!is_array($result) || ($result['settings_changed'] ?? false) !== true) {
self::throwInvalidSettingsResult(
$preparedRecoveryMarker,
'Agent 成功结果缺少已确认的实例设置'
);
}
$requested = is_array($payload['body'] ?? null)
? $payload['body'] : [];
if (array_key_exists('hostname', $requested)) {
$confirmed = $result['hostname'] ?? null;
if (!is_string($confirmed)
|| $confirmed !== $requested['hostname']
|| preg_match('/^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*$/D', $confirmed) !== 1
|| strlen($confirmed) > 253) {
self::throwInvalidSettingsResult(
$preparedRecoveryMarker,
'Agent 成功结果缺少已确认的主机名'
);
}
}
if (isset($payload['credential'])
&& is_array($payload['credential'])) {
$instance->sealed_credential = CloudSecrets::encrypt(
$payload['credential'], 'instance-credential'
);
}
$decoded = json_decode((string) $instance->spec_json, true);
if (!is_array($decoded) || array_is_list($decoded)) {
throw new CloudError('虚拟机配置记录损坏,无法保存实例设置', 409);
}
if (array_key_exists('hostname', $requested)) {
$decoded['guest_hostname'] = $result['hostname'];
}
$instance->spec_json = json_encode($decoded,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE);
} elseif (in_array((string) $job->kind, [
'service_suspend', 'network_suspend', 'network_resume',
], true) && is_array($result)
&& ($result['skipped'] ?? false) === true) {
// The desired state changed before dispatch. No remote
// effect occurred, so retain the current projection and
// let lifecycle derive the next generation if needed.
} elseif ($job->kind === 'service_suspend'
&& in_array((string) $instance->service_status,
array_merge(
['purge_stopping', 'purge_force_required'],
$preparedRecoveryMarker === null
? [] : ['purge_stop_failed', 'suspend_failed']
), true)) {
if ((string) $instance->status === 'deleting'
&& in_array((string) $instance->service_status,
[
'purge_stopping', 'purge_force_required',
'purge_stop_failed',
], true)) {
$instance->service_status = 'purge_deleting';
} elseif ((string) $instance->service_status
=== 'suspend_failed') {
$instance->service_status = 'suspended_expired';
$instance->suspended_at = time();
}
} elseif ($job->kind === 'service_suspend'
&& !in_array((string) $instance->service_status,
['suspended_expired', 'purge_deleting'], true)) {
throw new CloudRecoveryInstanceStateConflict(
'服务暂停成功结论与实例状态矛盾',
409
);
} elseif ($job->kind === 'network_suspend'
&& in_array((string) $instance->network_status,
array_merge(
['blocking', 'blocking_expired'],
$preparedRecoveryMarker === null
? [] : ['block_failed']
), true)) {
$instance->network_status = ($payload['suspend_reason'] ?? '') === 'expired'
? 'blocked_expired' : 'blocked_traffic';
$instance->network_suspended_at = time();
} elseif ($job->kind === 'network_suspend'
&& !in_array((string) $instance->network_status,
['blocked_expired', 'blocked_traffic'], true)) {
throw new CloudRecoveryInstanceStateConflict(
'网络暂停成功结论与实例状态矛盾',
409
);
} elseif ($job->kind === 'network_resume'
&& in_array((string) $instance->network_status,
array_merge(
[
'restoring', 'restore_pending',
'blocking', 'blocking_expired',
],
$preparedRecoveryMarker === null
? [] : ['restore_failed']
), true)) {
if (((string) $instance->service_status === 'expired_grace'
&& $instance->expires_at !== null
&& (int) $instance->expires_at <= time())
|| (string) $instance->network_status
=== 'blocking_expired') {
// Expiry may race with a previously valid resume.
// The receipt proves the links are now up, so queue
// a fresh suspend generation instead of publishing
// a misleading active projection.
$instance->network_status = 'blocking_expired';
} elseif ((string) $instance->network_status
=== 'blocking') {
$instance->network_status = 'blocking';
} else {
$instance->network_status = 'active';
}
$instance->network_suspended_at = null;
} elseif ($job->kind === 'network_resume'
&& (string) $instance->network_status !== 'active') {
throw new CloudRecoveryInstanceStateConflict(
'网络恢复成功结论与实例状态矛盾',
409
);
}
// Successful power/reinstall/policy operations often
// include the resulting VM state. Persist that bounded
// observation so the next page render is immediate;
// missing state is deliberately left for vm-states.
self::recordRuntime($instance, is_array($result) ? $result : [],
(string) $job->kind, $payload);
$instance->updated_at = time();
$instance->save();
}
if ($localFinalizerRecovery) {
// The kind-specific projection above is the only work being
// retried. Preserve the original terminal receipt audit
// (phase/outcome/status/terminal_at/error/result/event) and
// update only the independently recoverable effect ledger.
$now = time();
$job->effect_may_have_applied = 0;
$job->safe_to_resubmit = 1;
$job->effect_resolution = 'not_applied';
$job->effect_resolved_at = $now;
$job->effect_resolved_by = null;
$job->effect_resolution_reason =
'automatic_local_not_dispatched';
$job->sealed_payload = CloudSecrets::encrypt(
CloudInput::safeResult($payload),
'job-payload'
);
$job->finalization_incident = 0;
$job->finalization_error_code = null;
$job->read_at = null;
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
CloudJobService::reconcileAggregateParentEffectLocked(
$job,
null
);
return;
}
if ($recoveredCreateEvent !== null) {
$now = time();
$safePayload = CloudInput::safeResult($payload);
$safePayload[CloudOperationRecoveryIdentity::payloadKey()] =
CloudOperationRecoveryIdentity::persisted(
$operationIdentity
);
$job->completion_event_id =
(string) $recoveredCreateEvent->event_id;
$job->effect_resolution = 'applied';
$job->effect_resolved_at = $now;
$job->effect_resolved_by = null;
$job->effect_resolution_reason =
'late_completion_receipt:'
. (string) $recoveredCreateEvent->event_id;
$job->sealed_payload = CloudSecrets::encrypt(
$safePayload,
'job-payload'
);
$job->result_json = $safe === null ? null : json_encode(
$safe,
JSON_THROW_ON_ERROR
);
$job->finalization_incident = 0;
$job->finalization_error_code = null;
$job->read_at = null;
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
$order->status = 'succeeded';
$order->updated_at = $now;
$order->save();
$recoveredCreateEvent->disposition = 'applied';
$recoveredCreateEvent->applied_at = $now;
$recoveredCreateEvent->save();
return;
}
if ($preparedRecoveryMarker !== null) {
// The resource projection and this marker commit atomically.
// Keep the original encrypted request plus the exact validated
// operation until commit runs after the matching Agent fence
// has been released. This makes the prepare/fence crash window
// resumable for every registered kind, not only settings.
$preparedPayload = self::compactPreparedRecoveryPayload(
(string) $job->kind,
$payload
);
$preparedBinding = self::preparedRecoveryBinding(
$preparedRecoveryMarker
);
if ($preparedBinding === null) {
throw new CloudRecoveryPayloadUnavailable(
'任务预提交恢复标记无效',
409
);
}
$preparedPayload['_operation_recovery'] = [
'version' => 1,
'kind' => (string) $job->kind,
'prepared_marker' => $preparedBinding,
'operation' => $operation,
];
$encodedPrepared = json_encode(
$preparedPayload,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
if (strlen($encodedPrepared) > 196608) {
throw new CloudRecoveryPayloadUnavailable(
'任务预提交恢复数据过大',
409
);
}
$job->effect_resolution_reason = $preparedRecoveryMarker;
$job->sealed_payload = CloudSecrets::encrypt(
$preparedPayload,
'job-payload'
);
if ($forceRequired) {
// This exact no-effect Agent conclusion authorizes a new,
// separately audited force-delete generation. Keep the
// concrete code on the immutable graceful task instead of
// its earlier transport/deadline summary.
$job->error_code = 'force_required';
}
if (self::lateReceiptRecoveryMarker(
$preparedRecoveryMarker
) && in_array(
(string) ($job->finalization_error_code ?? ''),
[
'recovery_limit_reached',
'recovery_state_invalid',
],
true
)) {
$job->finalization_incident = 0;
$job->finalization_error_code = null;
}
$job->updated_at = time();
$job->save();
if ($order !== null) {
$order->status = $status;
$order->updated_at = time();
$order->save();
}
return;
}
$job->status = $status;
$job->phase = 'terminal';
$job->outcome = $status;
$job->terminal_at = time();
$job->effect_may_have_applied = $effectMayHaveApplied ? 1 : 0;
$job->safe_to_resubmit = $safeToResubmit ? 1 : 0;
$job->effect_resolution = $status === 'succeeded' ? 'applied'
: ($effectMayHaveApplied ? 'unknown' : 'not_applied');
$retainRecoveryEscrow = $status === 'failed'
&& $effectMayHaveApplied
&& (CloudOperationRecoveryPolicy::supports(
(string) $job->kind
) || CloudOperationRecoveryPolicy::isCreateLike(
(string) $job->kind
));
if (!$retainRecoveryEscrow) {
// A resolved terminal task never needs credentials again.
// Unknown operations in the explicit recovery policy retain
// encrypted escrow only until their effect is resolved.
$safePayload = CloudInput::safeResult($payload);
if ($operationIdentity !== null) {
$safePayload[
CloudOperationRecoveryIdentity::payloadKey()
] = CloudOperationRecoveryIdentity::persisted(
$operationIdentity
);
}
$job->sealed_payload = CloudSecrets::encrypt(
$safePayload, 'job-payload'
);
}
$job->result_json = $safe === null ? null : json_encode($safe, JSON_THROW_ON_ERROR);
$job->error_code = CloudOperationErrorCatalog::workerSummary(
(string) $job->kind, $status, $operation, $order !== null,
self::payloadAction($payload)
);
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = time();
$job->save();
// Aggregate catalog parents may be failed/unknown even though all
// leaf effects are already known (for example one replica applied
// and another failed before changing the host). Publish the parent
// effect now as well as after a later recovery commit, otherwise a
// fully-known partial aggregate would retain a permanent fence.
CloudJobService::reconcileAggregateParentEffectLocked(
$job,
null
);
if ($order !== null) {
if ($job->kind === 'create' && $effectMayHaveApplied) {
CloudPlacementService::markUncertain($order);
}
$order->status = $status;
$order->updated_at = time();
$order->save();
}
$wakeAutomaticBackup = $status === 'succeeded';
}, 3);
if ($wakeAutomaticBackup) {
try {
CloudAutomaticBackupService::wakeAfterSuccessfulMutation(
$claimed,
$payload,
time()
);
} catch (\Throwable $error) {
// The instance operation is already durably finalized. The
// scheduler's bounded retry remains the recovery path.
error_log(sprintf(
'automatic backup wake job=%d deferred: %s',
(int) $claimed->id,
get_class($error)
));
}
}
if ((string) $claimed->kind === 'host_storage_configure'
&& $status === 'succeeded') {
$this->scheduleTemplateBackfillAfterStorage(
(int) $claimed->host_id
);
$this->scheduleRescueBackfillAfterStorage(
(int) $claimed->host_id
);
}
if ((string) $claimed->kind
=== 'rescue_catalog_validate_replica'
&& $status === 'succeeded') {
$this->scheduleRescueDistributionAfterValidation(
$claimed
);
}
}
/** Run object-storage discovery only after the storage transaction commits. */
private function scheduleTemplateBackfillAfterStorage(int $hostId): void
{
try {
$actor = CloudDaemonActor::resolve('cloud-template-backfill');
if ($actor === null) {
throw new CloudError(
'云镜像补齐缺少可用的系统管理员执行者',
503
);
}
CloudTemplateHostBackfillService::scheduleReadyHost(
$hostId,
$actor,
'storage_ready'
);
} catch (\Throwable $error) {
error_log(sprintf(
'cloud template backfill host=%d trigger=storage_ready unavailable: %s',
$hostId,
get_class($error)
));
}
}
/** Start all-host distribution only after first-host boot validation. */
private function scheduleRescueDistributionAfterValidation(
CloudJob $job
): void {
try {
$replica = CloudRescueTemplateReplica::query()
->where('job_id', (int) $job->id)
->where('host_id', (int) $job->host_id)->first();
$templateId = $replica === null
? '' : (string) $replica->template_id;
if (preg_match('/^[a-f0-9]{32}$/D', $templateId) !== 1) {
throw new CloudError('救援 ISO 校验任务缺少目录标识', 409);
}
$actor = CloudDaemonActor::resolve(
'cloud-rescue-distribution'
);
if ($actor === null) {
throw new CloudError(
'救援 ISO 分发缺少可用的系统管理员执行者',
503
);
}
CloudRescueTemplateReplicaCoordinator::distribute(
$templateId,
$actor,
'validation_complete'
);
} catch (\Throwable $error) {
// The validation and immutable object promotion already committed.
// The periodic reconciler resumes this idempotently.
error_log(sprintf(
'cloud rescue distribution validation_job=%d deferred: %s',
(int) $job->id,
get_class($error)
));
}
}
/** Rescue replicas share the same host-readiness boundary as templates. */
private function scheduleRescueBackfillAfterStorage(int $hostId): void
{
try {
$actor = CloudDaemonActor::resolve('cloud-rescue-backfill');
if ($actor === null) {
throw new CloudError(
'救援 ISO 补齐缺少可用的系统管理员执行者',
503
);
}
CloudRescueTemplateHostBackfillService::scheduleReadyHost(
$hostId,
$actor,
'storage_ready'
);
} catch (\Throwable $error) {
error_log(sprintf(
'cloud rescue backfill host=%d trigger=storage_ready unavailable: %s',
$hostId,
get_class($error)
));
}
}
private static function throwInvalidSettingsResult(
?string $preparedRecoveryMarker,
string $message
): never {
if ($preparedRecoveryMarker !== null
&& self::lateReceiptRecoveryMarker(
$preparedRecoveryMarker
)) {
throw new CloudRecoveryReceiptInvalid($message, 502);
}
throw new CloudError($message, 502);
}
/** Rescue success is accepted only for the exact requested generation. */
private static function assertRescueResultMatchesRequest(
CloudJob $job,
string $status,
mixed $result,
array $payload
): void {
if ((string) $job->kind !== 'rescue' || $status !== 'succeeded') {
return;
}
$target = (string) ($payload['request_target'] ?? '');
$body = is_array($payload['body'] ?? null)
? $payload['body'] : [];
if (!is_array($result) || array_is_list($result)
|| !is_bool($result['active'] ?? null)
|| !is_bool($result['staged'] ?? null)
|| !is_bool($result['network_suspended'] ?? null)) {
throw new CloudError('Agent 救援操作成功回执无效', 502);
}
if (str_ends_with($target, '/rescue/enter')) {
if (!(($result['active'] ?? false)
|| ($result['staged'] ?? false))
|| ($result['template_id'] ?? null)
!== ($body['template_id'] ?? null)
|| ($result['revision'] ?? null)
!== ($body['revision'] ?? null)
|| ($result['generation_token'] ?? null)
!== ($body['generation_token'] ?? null)) {
throw new CloudError(
'Agent 救援操作回执与请求版本不匹配',
502
);
}
return;
}
if (str_ends_with($target, '/rescue/force-start')) {
if (($result['active'] ?? false) !== true
|| !is_string($result['template_id'] ?? null)
|| preg_match('/^[a-f0-9]{32}$/D',
$result['template_id']) !== 1
|| !is_int($result['revision'] ?? null)
|| (int) $result['revision'] < 1
|| !is_string($result['generation_token'] ?? null)
|| preg_match('/^[a-f0-9]{32}$/D',
$result['generation_token']) !== 1) {
throw new CloudError('Agent 救援强制启动回执无效', 502);
}
return;
}
if (!str_ends_with($target, '/rescue/exit')
|| ($result['active'] ?? true) !== false
|| ($result['staged'] ?? true) !== false
|| !in_array($result['template_id'] ?? '', ['', null], true)
|| !in_array($result['revision'] ?? 0, [0, null], true)
|| !in_array(
$result['generation_token'] ?? '',
['', null],
true
)) {
throw new CloudError('Agent 救援退出回执无效', 502);
}
}
private static function validSystemCleanup(CloudJob $job, array $payload): bool
{
if ($job->instance_id === null
|| ($payload['body'] ?? null) !== ['remove_managed_disks' => true]) {
return false;
}
$instance = CloudInstance::find($job->instance_id);
if ($instance === null || (string) $instance->status !== 'deleting'
|| (string) $instance->service_status !== 'purge_deleting'
|| (int) $job->user_id !== (int) $instance->user_id
|| (int) $job->host_id !== (int) $instance->host_id
|| (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id) {
return false;
}
return ($payload['method'] ?? null) === 'DELETE'
&& ($payload['request_target'] ?? null) === '/v1/vms/'
. rawurlencode((string) $instance->uuid);
}
private static function validSystemPolicy(CloudJob $job, array $payload): bool
{
if ($job->instance_id === null || !is_array($payload['body'] ?? null)) {
return false;
}
$instance = CloudInstance::find($job->instance_id);
if ($instance === null || (int) $job->user_id !== (int) $instance->user_id
|| (int) $job->host_id !== (int) $instance->host_id
|| (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id) {
return false;
}
$suffix = [
'service_suspend' => 'service/suspend',
'network_suspend' => 'network/suspend',
'network_resume' => 'network/resume',
][$job->kind] ?? null;
$expectedBody = match ((string) $job->kind) {
'service_suspend' => ['reason' => 'purge'],
'network_suspend' => ['reason' => (string) (
$payload['suspend_reason'] ?? ''
)],
'network_resume' => [],
default => null,
};
return $suffix !== null && $expectedBody !== null
&& ($payload['method'] ?? null) === 'POST'
&& ($payload['body'] ?? null) === $expectedBody
&& ($payload['request_target'] ?? null) === '/v1/vms/'
. rawurlencode((string) $instance->uuid) . '/' . $suffix;
}
private static function payloadAction(array $payload): ?string
{
$body = $payload['body'] ?? null;
$action = is_array($body) && is_string($body['action'] ?? null)
? $body['action'] : null;
return CloudOperationCatalog::isPowerAction($action ?? '') ? $action : null;
}
/**
* Validate the immutable purchase-time stock mode before materialising an
* exact Agent create success. Legacy finite orders retain their reservation
* marker. A markerless legacy order is accepted as unlimited only while the
* locked offer row still proves that it was unchanged before the order.
*/
private static function createStockLedgerIsConsistent(
CloudOrder $order
): bool {
$reserved = $order->stock_reserved_at !== null;
$released = $order->stock_released_at !== null;
$spec = json_decode((string) $order->spec_json, true);
if (!is_array($spec) || array_is_list($spec)) {
return false;
}
if (array_key_exists('stock_mode', $spec)) {
if ($spec['stock_mode'] === 'finite') {
return $reserved && !$released;
}
if ($spec['stock_mode'] === 'unlimited') {
return !$reserved && !$released;
}
return false;
}
if ($reserved) {
return !$released;
}
if ($released) {
return false;
}
foreach ([
'offer_id' => (int) $order->offer_id,
'plan_id' => (int) $order->plan_id,
'resource_pool_id' => (int) $order->resource_pool_id,
] as $key => $expected) {
if (($spec[$key] ?? null) !== $expected) {
return false;
}
}
$offer = CloudOffer::where('id', (int) $order->offer_id)
->lockForUpdate()->first();
return $offer !== null
&& (int) $offer->plan_id === (int) $order->plan_id
&& (int) $offer->resource_pool_id
=== (int) $order->resource_pool_id
&& (int) $offer->stock === -1
&& (int) $offer->created_at > 0
&& (int) $offer->created_at <= (int) $offer->updated_at
&& (int) $offer->updated_at < (int) $order->created_at
&& ($spec['price_version'] ?? null)
=== (int) $offer->price_version;
}
/**
* Validate the mandatory PHP-owned template snapshot without forwarding
* it to the Agent. This is a clean-break schema: create/reinstall work
* without the snapshot is invalid and finalizes as a local failure.
*/
private static function trustedTemplateMetadata(array $payload): array
{
if (array_key_exists('backup_metadata', $payload)) {
$metadata = $payload['backup_metadata'];
$body = $payload['body'] ?? null;
if (!is_array($metadata) || array_is_list($metadata)
|| !is_array($body) || array_is_list($body)
|| filter_var(
$metadata['backup_id'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
) === false
|| preg_match('/^[a-f0-9]{32}$/D',
(string) ($metadata['backup_key'] ?? '')) !== 1
|| !hash_equals(
(string) $metadata['backup_key'],
(string) ($body['backup_key'] ?? '')
)
|| ($metadata['os_source'] ?? null) !== 'cloud_template'
|| !is_string($metadata['os_name'] ?? null)
|| trim($metadata['os_name']) === ''
|| ($metadata['login_name'] ?? null) !== 'root'
|| !is_bool($metadata['debian_12_network_fix'] ?? null)
|| ($body['debian_12_network_fix'] ?? null)
!== $metadata['debian_12_network_fix']) {
throw new CloudError(
'任务中的云备份展示快照与 Agent 请求不匹配,必须人工核查',
409
);
}
return [
'os_source' => 'cloud_template',
'os_name' => $metadata['os_name'],
'login_name' => 'root',
'debian_12_network_fix' =>
$metadata['debian_12_network_fix'],
'restored_from_backup' => $metadata['backup_key'],
];
}
if (!array_key_exists('template_metadata', $payload)) {
throw new CloudError(
'任务缺少云镜像展示快照,必须终止并人工核查', 409
);
}
$metadata = $payload['template_metadata'];
$body = $payload['body'] ?? null;
if (!is_array($metadata) || array_is_list($metadata)
|| !is_array($body) || array_is_list($body)
|| !is_string($metadata['template_id'] ?? null)
|| !is_string($body['template_id'] ?? null)
|| preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D',
$metadata['template_id']) !== 1
|| !hash_equals($body['template_id'],
$metadata['template_id'])) {
throw new CloudError(
'任务中的云镜像展示快照与 Agent 请求不匹配,必须人工核查',
409
);
}
$public = CloudInstanceMetadata::publicSpec($metadata);
if (($public['os_source'] ?? null) !== 'cloud_template'
|| !is_string($public['os_name'] ?? null)
|| ($public['login_name'] ?? null) !== 'root'
|| !is_bool($metadata['debian_12_network_fix'] ?? null)) {
throw new CloudError('任务中的云镜像展示快照无效,必须人工核查',
409);
}
return [
'template_id' => $metadata['template_id'],
'os_source' => 'cloud_template',
'os_name' => $public['os_name'],
'login_name' => 'root',
'debian_12_network_fix' =>
$metadata['debian_12_network_fix'],
];
}
/** Reject retired restore jobs before the dispatch CAS can contact Agent. */
private static function assertPortableRestoreDispatchReady(
CloudJob $job,
array $payload
): void {
$attemptKey = $payload['attempt_key'] ?? null;
if (!CloudPortableRestoreFinalizer::owns($job)
|| !is_string($attemptKey)
|| preg_match('/^[a-f0-9]{32}$/D', $attemptKey) !== 1) {
throw new CloudError('旧备份恢复任务已停用', 409);
}
}
/**
* Create the one transient restore request and persist only its immutable
* Agent-journal identity. The URL is never returned in the stable payload,
* so every terminal, prepared and unknown-effect database path remains
* capability-free while still being able to reconcile the exact request.
*
* @return array{0:array,1:array} stable payload, wire payload
*/
private static function portableRestoreDispatchPayload(
CloudJob $job,
array $payload
): array {
$envelope = CloudPortableRestoreJobEnvelope::fromJobForDispatch(
$job,
$payload
);
$source = CloudBackupService::provisioningSource(
$envelope->backup()
);
$sourceUrl = $source['source_url'] ?? null;
if (!is_string($sourceUrl)) {
throw new CloudError('恢复任务临时下载能力缺失', 409);
}
$wire = $envelope->wirePayload($sourceUrl);
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$wire
);
$stable = $envelope->payload();
$stable[CloudOperationRecoveryIdentity::payloadKey()] =
CloudOperationRecoveryIdentity::persisted($identity);
return [$stable, $wire];
}
/**
* @return array{action:string,resource_id:string,request_fingerprint:string}
*/
private static function operationIdentityForFinalization(
CloudJob $job,
array $payload,
bool $forAppliedEffect,
bool $preparedRecovery
): array {
$portableAttempt = (string) $job->kind === 'backup_restore'
&& is_string($payload['attempt_key'] ?? null)
&& isset($payload[
CloudOperationRecoveryIdentity::payloadKey()
]);
if ($preparedRecovery || $portableAttempt) {
return CloudOperationRecoveryIdentity::expected(
$job,
$forAppliedEffect
);
}
return CloudOperationRecoveryIdentity::expectedFromPayload(
$job,
$payload,
$forAppliedEffect
);
}
/**
* Translate panel vocabulary only at the Agent wire boundary. The PHP job
* keeps "poweroff" so its audit label and user-facing history remain
* stable, while the Go protocol uses libvirt's explicit "destroy" action.
*/
private static function agentRequestBody(string $kind, array $body): array
{
return CloudOperationRecoveryIdentity::wireBody($kind, $body);
}
/**
* Bind every journal-backed Agent response to the claimed PHP job before
* its status or result can influence durable panel state. Missing IDs and
* cross-operation responses are ambiguous, never legacy success shortcuts.
*/
private static function operationForJob(CloudJob $job,
array $operation): array
{
$expected = (string) $job->operation_id;
$actual = $operation['operation_id'] ?? null;
if (!is_string($actual) || $expected === ''
|| !hash_equals($expected, $actual)) {
throw new CloudError(
'Agent 操作标识与当前任务不匹配,必须人工核查',
502
);
}
if (CloudOperationRecoveryIdentity::supports((string) $job->kind)) {
$expectedIdentity = CloudOperationRecoveryIdentity::expected(
$job,
strtolower((string) ($operation['status'] ?? ''))
=== 'succeeded'
);
$fingerprint = $operation['fingerprint']
?? $operation['request_fingerprint'] ?? null;
CloudOperationRecoveryIdentity::assertMatches(
$expectedIdentity,
[
'action' => $operation['action'] ?? null,
'resource_id' => $operation['resource_id'] ?? null,
'request_fingerprint' => $fingerprint,
]
);
}
return $operation;
}
private static function normalizeAgentOperation(array $operation): array
{
$causeCode = CloudOperationErrorCatalog::agentCauseCode(
$operation['cause_code'] ?? null
);
if ($causeCode === null) {
unset($operation['cause_code']);
} else {
$operation['cause_code'] = $causeCode;
}
$error = $operation['error'] ?? null;
if (is_array($error) && is_string($error['code'] ?? null)
&& preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $error['code']) === 1) {
$operation['error_code'] = $error['code'];
}
// Never persist or expose free-form Agent error messages. The local
// catalog maps the bounded code and operation kind to reviewed text.
if (is_array($error)) {
unset($operation['error']);
}
return $operation;
}
/**
* Persist only a successful, allow-listed power observation. Runtime
* cache failures are isolated from the durable job outcome: the Agent
* result and billing state remain authoritative even during a rolling
* schema upgrade where the cache table may not exist yet.
*/
private static function recordRuntime(CloudInstance $instance, array $result,
string $kind, array $payload): void
{
try {
if (in_array($kind,
['delete', 'expiry_delete', 'delete_force'], true)) {
CloudInstanceRuntimeService::forget((int) $instance->id);
return;
}
$source = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
$state = $source['status']
?? ($source['state'] ?? ($source['power_state'] ?? null));
$action = $kind === 'action' ? self::payloadAction($payload) : null;
$normalized = CloudInstanceRuntimeService::normalizeState($state);
if ($kind === 'action') {
// A successful graceful-shutdown job only confirms that the
// request was accepted. libvirt may still report running (or
// omit the state) until the guest completes ACPI shutdown.
// Preserve that distinction instead of publishing either a
// new fresh-running observation or a fabricated stopped one.
$state = CloudInstanceRuntimeService::stateAfterSuccessfulAction(
$state, $action
);
}
$instanceId = (int) $instance->id;
$hostId = (int) $instance->host_id;
$uuid = (string) $instance->uuid;
if ($kind === 'create') {
CloudInstanceRuntimeService::seed($instanceId, $hostId, $uuid);
}
if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') {
CloudInstanceRuntimeService::markObserved(
$instanceId, $hostId, $uuid, $state
);
}
} catch (\Throwable $error) {
error_log('cloud runtime observation write unavailable: ' . get_class($error));
}
}
/** Fail a malformed clean-break order before any Agent create mutation. */
private static function assertCreateTrafficPoolSnapshot(CloudJob $job): void
{
if ($job->order_id === null) {
throw new CloudError('开通任务缺少订单', 409);
}
$order = CloudOrder::find((int) $job->order_id);
$spec = $order === null
? null : json_decode((string) $order->spec_json, true);
if (!is_array($spec) || array_is_list($spec)) {
throw new CloudError('订单计费快照损坏', 409);
}
$quota = filter_var(
$spec['traffic_quota_bytes'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 0]]
);
if ($quota === false) {
throw new CloudError('订单缺少不可变的计费周期快照', 409);
}
CloudTrafficPoolPeriodBinding::orderSnapshot($spec, (int) $quota);
}
private static function exceptionDiagnostic(\Throwable $error, bool $dispatched,
?string $kind, bool $hasOrder, ?string $action): string
{
$operation = [];
if ($error instanceof CloudError) {
// CloudError text is Agent-/host-controlled too. The catalog
// keeps only exact reviewed details and otherwise returns a
// stable generic code for the durable job row.
$operation['error'] = $error->getMessage();
} elseif ($error instanceof \GuzzleHttp\Exception\TransferException) {
$operation['error'] = 'Agent 连接中断或操作超时';
}
$status = 'failed';
return CloudOperationErrorCatalog::workerSummary($kind, $status,
$operation, $hasOrder, $action)
?? CloudOperationErrorCatalog::userMessage($kind, $status, null,
$hasOrder, $action)
?? '虚拟机操作失败,请稍后重试';
}
}