You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
4045 lines
181 KiB
PHP
4045 lines
181 KiB
PHP
<?php
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudBackupSubscription;
|
|
use App\Models\CloudInvoice;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudOffer;
|
|
use App\Models\CloudOrder;
|
|
use App\Models\CloudRescueTemplateReplica;
|
|
use App\Models\CloudServicePeriod;
|
|
use App\Models\User;
|
|
use App\Services\Payment\PaymentOrderCreationException;
|
|
|
|
final class CloudWorkerService
|
|
{
|
|
private const AGENT_TIMEOUT_SECONDS = 900.0;
|
|
private const POLICY_AGENT_TIMEOUT_SECONDS = 120.0;
|
|
private const LEASE_SECONDS = 930;
|
|
private const POLICY_KINDS = ['service_suspend',
|
|
'network_suspend', 'network_resume'];
|
|
private const VPC_BATCH_KINDS = [
|
|
'vpc_host_apply', 'vpc_host_remove',
|
|
'vpc_port_attach', 'vpc_port_detach',
|
|
];
|
|
private const REINSTALL_REVIEW_CODES = [
|
|
'reinstall_cleanup_pending',
|
|
];
|
|
|
|
private bool $policyOnly;
|
|
private bool $vpcBatchOnly;
|
|
private array $observedErrors = [];
|
|
private CloudServiceCompensationExecutionService $compensations;
|
|
|
|
public function observationErrors(): array
|
|
{
|
|
return array_values(array_unique($this->observedErrors));
|
|
}
|
|
|
|
public function __construct(private ?CloudAgentClient $agent = null,
|
|
bool $policyOnly = false, bool $vpcBatchOnly = false,
|
|
?CloudServiceCompensationExecutionService $compensations = null)
|
|
{
|
|
if ($policyOnly && $vpcBatchOnly) {
|
|
throw new \InvalidArgumentException(
|
|
'Policy-only and VPC-batch-only modes are mutually exclusive'
|
|
);
|
|
}
|
|
$this->policyOnly = $policyOnly;
|
|
$this->vpcBatchOnly = $vpcBatchOnly;
|
|
$this->compensations = $compensations
|
|
?? new CloudServiceCompensationExecutionService();
|
|
// A cloud-image clone can legitimately take minutes on remote storage.
|
|
// The layered deadlines are Agent API 840s, Agent WSS control 855s,
|
|
// Gateway 870s, PHP 900s and worker lease 930s. Each outer layer has
|
|
// time to persist the inner layer's structured result before expiry.
|
|
}
|
|
|
|
private function agentForJob(CloudJob $job): CloudAgentClient
|
|
{
|
|
if ($this->agent !== null) {
|
|
return $this->agent;
|
|
}
|
|
if ((string) $job->kind === 'host_storage_configure') {
|
|
return CloudAgentSettingsService::clientForConfiguration(
|
|
(int) $job->host_id,
|
|
self::POLICY_AGENT_TIMEOUT_SECONDS
|
|
);
|
|
}
|
|
return CloudAgentSettingsService::clientForHost(
|
|
(int) $job->host_id,
|
|
$this->policyOnly
|
|
? self::POLICY_AGENT_TIMEOUT_SECONDS
|
|
: self::AGENT_TIMEOUT_SECONDS
|
|
);
|
|
}
|
|
|
|
public static function disposition(array $operation, bool $creating,
|
|
?string $kind = null): string
|
|
{
|
|
$status = is_string($operation['status'] ?? null)
|
|
? strtolower(trim($operation['status'])) : 'failed';
|
|
return $status === 'succeeded' ? 'succeeded'
|
|
: ($status === 'failed' ? 'failed' : 'running');
|
|
}
|
|
|
|
/** A create may refund only after both the VM and all managed artifacts are absent. */
|
|
private static function createFailureConfirmedAbsent(array $operation): bool
|
|
{
|
|
$result = $operation['result'] ?? null;
|
|
return is_array($result)
|
|
&& ($result['creation_absent'] ?? false) === true
|
|
&& ($result['managed_artifacts_absent'] ?? false) === true;
|
|
}
|
|
|
|
/** Exact Agent evidence required before PHP releases a deleted VM's ledger. */
|
|
private static function deleteSuccessConfirmed(array $operation,
|
|
string $kind): bool
|
|
{
|
|
$result = $operation['result'] ?? null;
|
|
return is_array($result) && !array_is_list($result)
|
|
&& ($result['deleted'] ?? null) === true
|
|
&& ($result['managed_artifacts_absent'] ?? null) === true
|
|
&& ($kind === 'delete_force'
|
|
? ($result['forced'] ?? null) === true
|
|
: ($result['forced'] ?? null) === false);
|
|
}
|
|
|
|
/** Exact evidence for a create request proven not to have crossed dispatch. */
|
|
private static function createNotDispatchedFailure(): array
|
|
{
|
|
return [
|
|
'status' => 'failed',
|
|
'result' => [
|
|
'creation_absent' => true,
|
|
'managed_artifacts_absent' => true,
|
|
],
|
|
];
|
|
}
|
|
|
|
/** Treat unconfirmed reinstall cleanup as a failed receipt, never as active work. */
|
|
private static function reinstallNeedsReview(array $operation): bool
|
|
{
|
|
$code = is_string($operation['error_code'] ?? null)
|
|
? strtolower(trim($operation['error_code'])) : '';
|
|
if ($code === '' && is_array($operation['error'] ?? null)
|
|
&& is_string($operation['error']['code'] ?? null)) {
|
|
$code = strtolower(trim($operation['error']['code']));
|
|
}
|
|
if (in_array($code, self::REINSTALL_REVIEW_CODES, true)) {
|
|
return true;
|
|
}
|
|
$result = $operation['result'] ?? null;
|
|
return is_array($result)
|
|
&& ($result['cleanup_pending'] ?? false) === true;
|
|
}
|
|
|
|
public function tick(): bool
|
|
{
|
|
$this->observedErrors = [];
|
|
$quarantined = self::quarantineInvalidExecutionModeJob();
|
|
$expired = self::expireDueJobs();
|
|
// Deadline convergence is always first. Recovery inspection runs in
|
|
// the dedicated --recovery-only worker, so this worker never waits on
|
|
// an offline recovery target before dispatching ordinary work.
|
|
$jobWorked = !$quarantined && $this->tickJob();
|
|
$compensationWorked = false;
|
|
$backupPreparationWorked = false;
|
|
$templateCleanup = false;
|
|
$templateTemporaryCleanup = false;
|
|
if (!$this->policyOnly && !$this->vpcBatchOnly) {
|
|
try {
|
|
// Backup intent commits before any provider I/O. Resume at
|
|
// most one missing multipart-plan/job binding per tick, after
|
|
// already accepted Agent work had the first dispatch chance.
|
|
$backupPreparationWorked =
|
|
(new CloudBackupPublicationPreparation())->resumeOne();
|
|
} catch (\Throwable $error) {
|
|
$this->observedErrors[] =
|
|
'backup_publication_preparation_failed';
|
|
}
|
|
try {
|
|
// This is a PHP-only ledger mutation. The execution service
|
|
// owns its bounded lease and idempotency evidence; the worker
|
|
// advances at most one item and never replays an Agent call.
|
|
$compensationWorked = $this->compensations->tick();
|
|
} catch (\Throwable $error) {
|
|
// Ordinary Agent work already advanced above and remains
|
|
// independent from a temporarily unavailable local ledger.
|
|
$this->observedErrors[] = 'service_compensation_failed';
|
|
}
|
|
try {
|
|
$templateCleanup = (new CloudTemplateImportCancellation())
|
|
->cleanupOneExpired();
|
|
} catch (\Throwable $error) {
|
|
// The intent has already moved to source_unknown when an S3
|
|
// abort cannot be proven. Surface one worker observation and
|
|
// never loop a destructive provider request blindly.
|
|
$this->observedErrors[] =
|
|
'template_import_cleanup_unresolved';
|
|
}
|
|
try {
|
|
$templateTemporaryCleanup =
|
|
CloudTemplateRemoteFetcher::cleanupOneStaleFile();
|
|
} catch (\Throwable $error) {
|
|
$this->observedErrors[] =
|
|
'template_import_temporary_cleanup_failed';
|
|
}
|
|
}
|
|
return $quarantined || $jobWorked || $compensationWorked
|
|
|| $backupPreparationWorked
|
|
|| $templateCleanup
|
|
|| $templateTemporaryCleanup || $expired > 0;
|
|
}
|
|
|
|
/**
|
|
* Fail closed when a durable job's execution-domain discriminator is
|
|
* damaged. Reinterpreting a local row as an Agent mutation (or an
|
|
* aggregate parent as a leaf) could replay the original operation. Keep
|
|
* the immutable dispatch evidence and encrypted payload untouched, publish
|
|
* one sticky PHP-only incident, and let the recovery worker recheck it
|
|
* with bounded leases forever without contacting the Agent.
|
|
*/
|
|
private static function quarantineInvalidExecutionModeJob(): bool
|
|
{
|
|
return (new CloudJob())->getConnection()->transaction(
|
|
static function (): bool {
|
|
$job = CloudJob::where('phase', '<>', 'terminal')
|
|
->where(static function ($invalid): void {
|
|
$invalid->where(static function ($leaf): void {
|
|
$leaf->whereIn(
|
|
'kind',
|
|
CloudOperationRecoveryPolicy::ledgerFinalizerKinds()
|
|
)->where(static function ($mode): void {
|
|
$mode->whereNull('execution_mode')
|
|
->orWhere('execution_mode', '<>', 'agent');
|
|
});
|
|
})->orWhere(static function ($aggregate): void {
|
|
$aggregate->whereIn(
|
|
'kind',
|
|
CloudJobService::aggregateKinds()
|
|
)->where(static function ($mode): void {
|
|
$mode->whereNull('execution_mode')
|
|
->orWhere(
|
|
'execution_mode',
|
|
'<>',
|
|
'aggregate'
|
|
);
|
|
});
|
|
});
|
|
})
|
|
->orderBy('id')
|
|
->lockForUpdate()
|
|
->first();
|
|
if ($job === null) {
|
|
return false;
|
|
}
|
|
$now = time();
|
|
$aggregate = CloudJobService::isAggregateKind(
|
|
(string) $job->kind
|
|
);
|
|
$job->phase = 'terminal';
|
|
$job->outcome = 'failed';
|
|
$job->status = 'failed';
|
|
$job->terminal_at = $now;
|
|
$job->effect_may_have_applied = 1;
|
|
$job->safe_to_resubmit = 0;
|
|
$job->effect_resolution = 'unknown';
|
|
$job->error_code = $aggregate
|
|
? 'recovery_aggregate_state_conflict'
|
|
: 'recovery_instance_state_conflict';
|
|
$job->finalization_incident = 1;
|
|
$job->finalization_error_code = $job->error_code;
|
|
$job->read_at = null;
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->available_at = $now + 300;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
return true;
|
|
},
|
|
3
|
|
);
|
|
}
|
|
|
|
/** Claim and advance at most one durable cloud job. */
|
|
private function tickJob(): bool
|
|
{
|
|
$job = (new CloudJob())->getConnection()->transaction(function () {
|
|
$now = time();
|
|
$query = self::dispatchableBeforeDeadline(
|
|
CloudJob::where('phase', 'accepted')
|
|
->where(static function ($mode): void {
|
|
$mode->where('execution_mode', 'agent')
|
|
->orWhere(static function ($local): void {
|
|
$local->where('execution_mode', 'local')
|
|
->where('kind', 'payment_create');
|
|
});
|
|
})
|
|
->where('dispatch_attempts', 0),
|
|
$now
|
|
)->where('available_at', '<=', $now)
|
|
->where('lease_until', '<=', $now);
|
|
CloudJobService::applyVpcHostDispatchFence($query);
|
|
if ($this->policyOnly) {
|
|
$query->whereIn('kind', self::POLICY_KINDS);
|
|
} elseif ($this->vpcBatchOnly) {
|
|
$query->whereNotNull('vpc_batch_id')
|
|
->whereIn('kind', self::VPC_BATCH_KINDS);
|
|
} else {
|
|
// The regular worker remains a fail-safe policy consumer when
|
|
// the dedicated workers are not installed, but always claims
|
|
// target-state policy ahead of slow image/provisioning jobs.
|
|
$quoted = "'" . implode("','", self::POLICY_KINDS) . "'";
|
|
$query->orderByRaw("CASE WHEN kind IN (" . $quoted
|
|
. ") THEN 0 WHEN kind = 'expiry_delete' THEN 1 ELSE 2 END");
|
|
}
|
|
$job = $query->orderBy('id')->lockForUpdate()->first();
|
|
if ($job === null) {
|
|
return null;
|
|
}
|
|
$job->lease_token = bin2hex(random_bytes(16));
|
|
$job->lease_until = $now + (in_array($job->kind, self::POLICY_KINDS, true)
|
|
? 150 : self::LEASE_SECONDS);
|
|
$job->save();
|
|
return $job;
|
|
}, 3);
|
|
if ($job === null) {
|
|
return false;
|
|
}
|
|
$payload = [];
|
|
$dispatched = false;
|
|
try {
|
|
$payload = CloudSecrets::decrypt($job->sealed_payload, 'job-payload');
|
|
if ((string) $job->kind === 'payment_create') {
|
|
try {
|
|
$this->runPaymentCreate($job, $payload, $dispatched);
|
|
} catch (\Throwable $error) {
|
|
$this->observedErrors[] = 'payment_task_failed';
|
|
$crossed = $dispatched;
|
|
if (!$crossed) {
|
|
$fresh = CloudJob::find((int) $job->id);
|
|
$crossed = $fresh !== null
|
|
&& CloudDispatchLedger::mayHaveDispatched($fresh);
|
|
}
|
|
self::forceTerminalFailure(
|
|
(int) $job->id,
|
|
(string) $job->lease_token,
|
|
$crossed,
|
|
!$crossed,
|
|
'payment_finalization_failed'
|
|
);
|
|
error_log('cloud payment task ' . (int) $job->id
|
|
. ' failed: ' . get_class($error));
|
|
}
|
|
return true;
|
|
}
|
|
if (!is_string($payload['method'] ?? null)
|
|
|| !in_array(strtoupper($payload['method']),
|
|
['POST', 'PUT', 'PATCH', 'DELETE'], true)
|
|
|| !is_string($payload['request_target'] ?? null)
|
|
|| !str_starts_with($payload['request_target'], '/v1/')
|
|
|| !is_array($payload['body'] ?? null)) {
|
|
throw new CloudError('Agent 任务请求格式无效', 409);
|
|
}
|
|
$policyKind = in_array($job->kind, self::POLICY_KINDS, true);
|
|
$systemPolicy = ($payload['system_policy'] ?? false) === true
|
|
&& $policyKind && self::validSystemPolicy($job, $payload);
|
|
$systemCleanup = $job->kind === 'expiry_delete'
|
|
&& ($payload['system_cleanup'] ?? false) === true
|
|
&& self::validSystemCleanup($job, $payload);
|
|
$vpcSystem = CloudVpcPortService::isSystemJob($job, $payload);
|
|
if (($policyKind && !$systemPolicy)
|
|
|| ($job->kind === 'expiry_delete' && !$systemCleanup)) {
|
|
throw new CloudError('系统策略任务与实例状态不匹配', 409);
|
|
}
|
|
$actor = ['user_id' => (int) $job->user_id, 'is_admin' => false, 'is_super_admin' => false,
|
|
'can_manage' => false, 'session_id' => '', 'operation_id' => $job->operation_id];
|
|
$user = User::find($job->user_id);
|
|
if (!$systemPolicy && !$systemCleanup && !$vpcSystem
|
|
&& ($user === null || (int) $user->enable !== 1)) {
|
|
$operation = CloudOperationRecoveryPolicy::isCreateLike(
|
|
(string) $job->kind
|
|
)
|
|
? self::createNotDispatchedFailure()
|
|
: ['status' => 'failed'];
|
|
$operation['effect_may_have_applied'] = false;
|
|
$operation['safe_to_resubmit'] = true;
|
|
$this->finish($job, $operation, $payload);
|
|
return true;
|
|
}
|
|
if ($job->kind === 'create') {
|
|
self::assertCreateTrafficPoolSnapshot($job);
|
|
$actor['can_provision'] = true;
|
|
$actor['can_manage'] = true;
|
|
} elseif ($vpcSystem) {
|
|
// Host-route projections are PHP-derived control-plane work.
|
|
// The Agent keeps its host-global VPC endpoint behind the
|
|
// super-administrator boundary, while the job remains owned
|
|
// by the tenant whose accepted port mutation it completes.
|
|
$actor['is_admin'] = true;
|
|
$actor['is_super_admin'] = true;
|
|
$actor['can_manage'] = true;
|
|
} elseif ($systemPolicy || $systemCleanup) {
|
|
$actor['can_manage'] = true;
|
|
if ($systemCleanup) {
|
|
$actor['system_cleanup'] = true;
|
|
}
|
|
} elseif (($payload['can_manage'] ?? false) === true && $user !== null
|
|
&& (bool) $user->is_admin && in_array((int) $user->id,
|
|
array_map('intval', explode(',', (string) \App\Services\Config::get('super_admin'))), true)) {
|
|
$actor['is_admin'] = $actor['is_super_admin'] = $actor['can_manage'] = true;
|
|
} elseif ((int) ($payload['owner_id'] ?? 0) === (int) $job->user_id) {
|
|
$actor['can_manage'] = true;
|
|
}
|
|
if ($job->kind !== 'create' && !$actor['can_manage']) {
|
|
$this->finish($job, ['status' => 'failed',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => true], $payload);
|
|
return true;
|
|
}
|
|
if (!CloudFirewallService::assertJobRunnable($job, $payload)) {
|
|
// A confirmed instance deletion dominates an overlapping
|
|
// tenant firewall request. Close the obsolete job without
|
|
// touching the deleted policy or contacting the Agent.
|
|
$this->finish($job, [
|
|
'operation_id' => (string) $job->operation_id,
|
|
'status' => 'failed',
|
|
'error_code' => 'firewall_apply_superseded',
|
|
'error' => '实例已删除,防火墙请求不再适用',
|
|
], $payload);
|
|
return true;
|
|
}
|
|
CloudVpcPortService::assertJobRunnable($job, $payload);
|
|
if ($systemPolicy && $job->instance_id !== null) {
|
|
$instance = CloudInstance::find($job->instance_id);
|
|
$policyMarker = in_array((string) $job->kind,
|
|
['network_suspend', 'network_resume'], true)
|
|
? CloudInstanceNetworkPolicy::payloadMarker(
|
|
$payload,
|
|
(int) $job->instance_id,
|
|
(string) $job->operation_id
|
|
) : null;
|
|
$desired = $policyMarker === null
|
|
? self::policyDesired($instance, (string) $job->kind)
|
|
: CloudInstanceNetworkPolicy::jobStillDesired(
|
|
(int) $job->instance_id,
|
|
(string) $job->kind,
|
|
(string) $job->operation_id
|
|
);
|
|
if (!$desired) {
|
|
$this->finish($job, ['status' => 'succeeded', 'result' => ['skipped' => true]], $payload);
|
|
return true;
|
|
}
|
|
}
|
|
if (!$systemPolicy && !$systemCleanup && !$vpcSystem
|
|
&& $job->kind !== 'create'
|
|
&& $job->instance_id !== null) {
|
|
$instance = CloudInstance::find($job->instance_id);
|
|
$policyTransition = $instance !== null
|
|
&& (in_array((string) $instance->service_status,
|
|
['suspending_expired', 'resume_pending', 'resuming',
|
|
'suspend_failed', 'resume_failed', 'expired_grace',
|
|
'purge_stopping', 'purge_stop_failed',
|
|
'purge_force_required',
|
|
'purge_deleting', 'purge_failed'], true)
|
|
|| in_array((string) $instance->network_status,
|
|
['blocking', 'blocking_expired', 'blocked_expired',
|
|
'restore_pending', 'restoring', 'block_failed',
|
|
'restore_failed'], true));
|
|
$globalOverride = $actor['is_super_admin'] && $actor['can_manage'];
|
|
if ($instance === null || ($job->kind !== 'delete' && !$globalOverride
|
|
&& ($policyTransition || !$instance->serviceAvailable()))) {
|
|
$this->finish($job, ['status' => 'failed',
|
|
'error' => '实例服务已到期或正在执行策略,不能执行该操作'], $payload);
|
|
return true;
|
|
}
|
|
}
|
|
if ((string) $job->kind === 'network_change'
|
|
&& $job->public_network_transition_id === null) {
|
|
throw new CloudError('旧公网网络写任务已停用', 409);
|
|
} elseif ((string) $job->kind === 'backup_restore') {
|
|
self::assertPortableRestoreDispatchReady($job, $payload);
|
|
} elseif (str_starts_with(
|
|
(string) $job->kind,
|
|
'instance_migration_'
|
|
)) {
|
|
// Canonical PeerMigration rows carry an exact recoverable
|
|
// identity. Reject stale scalar-IP or torn ledger jobs before
|
|
// the dispatch CAS so they can never contact an Agent. This
|
|
// must validate the live request directly: expected() also
|
|
// accepts a post-effect recovery escrow, which is never a
|
|
// valid shape for an accepted, not-yet-dispatched job.
|
|
CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
$job,
|
|
$payload,
|
|
false
|
|
);
|
|
} elseif (in_array(
|
|
(string) $job->kind,
|
|
['delete', 'delete_force', 'expiry_delete'],
|
|
true
|
|
) && $job->instance_id !== null) {
|
|
$deleteInstance = CloudInstance::find(
|
|
(int) $job->instance_id
|
|
);
|
|
if ($deleteInstance !== null) {
|
|
CloudInstanceResourceRelease::
|
|
assertCanDispatchDelete($deleteInstance);
|
|
}
|
|
}
|
|
$wirePayload = $payload;
|
|
$dispatchPayload = $payload;
|
|
$dispatchSealedPayload = null;
|
|
if ((string) $job->kind === 'backup_restore') {
|
|
[$dispatchPayload, $wirePayload] =
|
|
self::portableRestoreDispatchPayload($job, $payload);
|
|
$dispatchSealedPayload = CloudSecrets::encrypt(
|
|
$dispatchPayload,
|
|
'job-payload'
|
|
);
|
|
}
|
|
$now = time();
|
|
$dispatchClaim = self::dispatchableBeforeDeadline(
|
|
CloudJob::where('id', $job->id)
|
|
->where('lease_token', $job->lease_token)
|
|
->where('phase', 'accepted')
|
|
->where('dispatch_attempts', 0),
|
|
$now
|
|
);
|
|
$dispatchUpdate = [
|
|
'phase' => 'dispatching',
|
|
'status' => 'running',
|
|
'attempts' => 1,
|
|
'dispatch_attempts' => 1,
|
|
'dispatch_started_at' => $now,
|
|
'dispatched_at' => $now,
|
|
'receipt_deadline_at' => $now
|
|
+ CloudJobService::receiptDeadlineSecondsForJob($job),
|
|
'updated_at' => $now,
|
|
];
|
|
if ($dispatchSealedPayload !== null) {
|
|
// Persist only the fingerprint of the exact one-shot wire
|
|
// request. The presigned capability itself exists solely in
|
|
// $wirePayload and can never become durable job state.
|
|
$dispatchUpdate['sealed_payload'] = $dispatchSealedPayload;
|
|
}
|
|
$updated = $dispatchClaim->update($dispatchUpdate);
|
|
if ($updated !== 1) {
|
|
return true;
|
|
}
|
|
// Keep the claimed model aligned with the successful CAS. The
|
|
// synchronous response is finalized through this same object; a
|
|
// stale dispatch_attempts=0 value would incorrectly classify a
|
|
// real Agent response as a local pre-dispatch failure and omit its
|
|
// immutable identity from the terminal payload.
|
|
$job->phase = 'dispatching';
|
|
$job->status = 'running';
|
|
$job->attempts = 1;
|
|
$job->dispatch_attempts = 1;
|
|
$job->dispatch_started_at = $now;
|
|
$job->dispatched_at = $now;
|
|
$job->receipt_deadline_at = $now
|
|
+ CloudJobService::receiptDeadlineSecondsForJob($job);
|
|
if ($dispatchSealedPayload !== null) {
|
|
$job->sealed_payload = $dispatchSealedPayload;
|
|
$payload = $dispatchPayload;
|
|
}
|
|
CloudFirewallService::markJobRunning($job);
|
|
CloudVpcPortService::markJobRunning($job);
|
|
CloudPortableRestoreFinalizer::markDispatched($job);
|
|
if ((string) $job->kind === 'network_change'
|
|
&& $job->public_network_transition_id !== null) {
|
|
CloudPublicNetworkTransitionWorkflow::markDispatched($job);
|
|
}
|
|
$dispatched = true;
|
|
$operation = self::operationForJob(
|
|
$job,
|
|
$this->agentForJob($job)->request(
|
|
$payload['method'],
|
|
$payload['request_target'],
|
|
$actor,
|
|
self::agentRequestBody((string) $job->kind,
|
|
$wirePayload['body'])
|
|
)
|
|
);
|
|
$operation = self::normalizeAgentOperation($operation);
|
|
$agentStatus = strtolower((string) ($operation['status'] ?? ''));
|
|
if (in_array($agentStatus, ['succeeded', 'failed'], true)) {
|
|
$operation['effect_may_have_applied'] =
|
|
$operation['effect_may_have_applied'] ?? true;
|
|
$operation['safe_to_resubmit'] =
|
|
$operation['safe_to_resubmit'] ?? false;
|
|
$this->finish($job, $operation, $payload);
|
|
} elseif (in_array($agentStatus,
|
|
['accepted', 'reserved', 'dispatching', 'pending', 'running'],
|
|
true)) {
|
|
CloudJob::where('id', $job->id)
|
|
->where('lease_token', $job->lease_token)
|
|
->where('phase', 'dispatching')->update([
|
|
'phase' => 'awaiting_receipt',
|
|
'agent_accepted_at' => time(),
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'updated_at' => time(),
|
|
]);
|
|
$fresh = CloudJob::find((int) $job->id);
|
|
$event = $fresh === null ? null
|
|
: CloudOperationCompletionService::pendingForJob($fresh);
|
|
if ($event !== null) {
|
|
$this->settleCompletionReceipt((int) $job->id,
|
|
(string) $event->event_id);
|
|
}
|
|
} else {
|
|
throw new CloudError('Agent 返回了无效的任务状态', 502);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
// Diagnostics deliberately contain no raw payloads, credentials or upstream tracebacks.
|
|
$this->observedErrors[] = 'worker_tick_failed';
|
|
$diagnostic = self::exceptionDiagnostic($error, $dispatched,
|
|
(string) $job->kind, $job->order_id !== null, self::payloadAction($payload));
|
|
$log = 'cloud worker job ' . $job->id . ' failed: ' . get_class($error);
|
|
if ($error instanceof CloudError) {
|
|
$log .= ' status=' . $error->httpStatus . ' message=' . $diagnostic;
|
|
}
|
|
error_log($log);
|
|
$safe = !$dispatched || ($error instanceof CloudAgentError
|
|
&& ($error->safeValidationFailure
|
|
|| $error->safePreDispatchFailure));
|
|
$operation = $safe && CloudOperationRecoveryPolicy::isCreateLike(
|
|
(string) $job->kind
|
|
)
|
|
? self::createNotDispatchedFailure() : ['status' => 'failed'];
|
|
$operation['operation_id'] = (string) $job->operation_id;
|
|
$operation['error_code'] = $error instanceof CloudAgentError
|
|
? $error->agentCode : 'dispatch_failed';
|
|
$operation['effect_may_have_applied'] = !$safe;
|
|
$operation['safe_to_resubmit'] = $safe;
|
|
try {
|
|
$this->finish($job, $operation, $payload);
|
|
} catch (\Throwable $finishError) {
|
|
self::forceTerminalFailure((int) $job->id,
|
|
(string) $job->lease_token, !$safe,
|
|
$safe, 'local_finalization_failed', $safe);
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Keep both the queue claim and the accepted-to-dispatching CAS behind the
|
|
* same absolute deadline. The second check closes the boundary race where
|
|
* an old storage task was scanned just before created_at+60 but would
|
|
* otherwise cross dispatch using its former persisted 300-second value.
|
|
*/
|
|
private static function dispatchableBeforeDeadline($query, int $now)
|
|
{
|
|
$storageWindow = CloudJobService::dispatchDeadlineSeconds(
|
|
'host_storage_configure'
|
|
);
|
|
return $query->where(static function ($safe): void {
|
|
$safe->where(static function ($payment): void {
|
|
$payment->where('execution_mode', 'local')
|
|
->where('kind', 'payment_create');
|
|
})->orWhere(static function ($agent): void {
|
|
$agent->where('execution_mode', 'agent')
|
|
->whereNotExists(static function ($events): void {
|
|
$events->selectRaw('1')
|
|
->from(
|
|
'cloud_agent_completion_events as dispatch_events'
|
|
)->whereColumn(
|
|
'dispatch_events.host_id',
|
|
'cloud_jobs.host_id'
|
|
)->whereColumn(
|
|
'dispatch_events.operation_id',
|
|
'cloud_jobs.operation_id'
|
|
);
|
|
});
|
|
});
|
|
})->where('dispatch_deadline_at', '>', $now)
|
|
->where(static function ($eligible) use (
|
|
$now, $storageWindow
|
|
): void {
|
|
$eligible->where('kind', '!=', 'host_storage_configure')
|
|
->orWhere('created_at', '>', $now - $storageWindow);
|
|
});
|
|
}
|
|
|
|
private function runPaymentCreate(CloudJob $job, array $payload,
|
|
bool &$dispatched): void
|
|
{
|
|
$input = $payload['input'] ?? null;
|
|
if ($job->invoice_id === null || !is_array($input)
|
|
|| !is_string($input['idempotency_key'] ?? null)
|
|
|| !is_string($input['method'] ?? null)) {
|
|
$this->finishPaymentCreate($job, false, null, false, true,
|
|
'payment_request_invalid');
|
|
return;
|
|
}
|
|
$now = time();
|
|
$dispatchClaim = self::dispatchableBeforeDeadline(
|
|
CloudJob::where('id', (int) $job->id)
|
|
->where('lease_token', (string) $job->lease_token)
|
|
->where('phase', 'accepted')
|
|
->where('dispatch_attempts', 0),
|
|
$now
|
|
);
|
|
$claimed = $dispatchClaim->update([
|
|
'phase' => 'dispatching', 'status' => 'running',
|
|
'attempts' => 1, 'dispatch_attempts' => 1,
|
|
'dispatch_started_at' => $now, 'dispatched_at' => $now,
|
|
'receipt_deadline_at' => $now
|
|
+ CloudJobService::receiptDeadlineSeconds(
|
|
(string) $job->kind
|
|
),
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($claimed !== 1) {
|
|
return;
|
|
}
|
|
$dispatched = true;
|
|
try {
|
|
$invoice = CloudInvoice::where('id', (int) $job->invoice_id)
|
|
->where('user_id', (int) $job->user_id)->first();
|
|
$user = User::find((int) $job->user_id);
|
|
if ($invoice === null || $user === null) {
|
|
throw new CloudError('账单或账户不存在', 404);
|
|
}
|
|
$payment = CloudInvoicePaymentService::create(
|
|
$invoice,
|
|
$user,
|
|
$input,
|
|
(string) ($payload['client_ip'] ?? ''),
|
|
(string) ($payload['user_agent'] ?? '')
|
|
);
|
|
$this->finishPaymentCreate($job, true, $payment, false, false, '');
|
|
} catch (\Throwable $error) {
|
|
$attempt = \App\Models\CloudInvoicePayment::where(
|
|
'user_id', (int) $job->user_id
|
|
)->where('idempotency_key', (string) $input['idempotency_key'])
|
|
->first();
|
|
if ($error instanceof PaymentOrderCreationException) {
|
|
$effect = $error->effectMayHaveApplied;
|
|
$safe = $error->safeToResubmit;
|
|
$code = $error->failureCode;
|
|
} elseif ((string) $input['method'] === 'usdt') {
|
|
// USDT order creation is one local database transaction; it
|
|
// does not submit a non-idempotent provider mutation here.
|
|
$effect = false;
|
|
$safe = true;
|
|
$code = 'payment_creation_failed';
|
|
} else {
|
|
$status = (string) ($attempt?->status ?? '');
|
|
$effect = $attempt !== null
|
|
&& $status !== 'not_created';
|
|
$safe = !$effect;
|
|
$code = $effect ? 'payment_creation_unknown'
|
|
: 'payment_creation_failed';
|
|
}
|
|
$this->finishPaymentCreate($job, false, null, $effect,
|
|
$safe, $code);
|
|
}
|
|
}
|
|
|
|
private function finishPaymentCreate(CloudJob $job, bool $success,
|
|
?array $result, bool $effect, bool $safe, string $code): void
|
|
{
|
|
CloudJob::where('id', (int) $job->id)
|
|
->where('lease_token', (string) $job->lease_token)
|
|
->where('phase', '<>', 'terminal')->update([
|
|
'phase' => 'terminal',
|
|
'outcome' => $success ? 'succeeded' : 'failed',
|
|
'status' => $success ? 'succeeded' : 'failed',
|
|
'terminal_at' => time(),
|
|
'effect_may_have_applied' => $effect ? 1 : 0,
|
|
'safe_to_resubmit' => $safe ? 1 : 0,
|
|
'effect_resolution' => $success ? 'applied'
|
|
: ($effect ? 'unknown' : 'not_applied'),
|
|
'error_code' => $success ? null : $code,
|
|
'result_json' => $result === null ? null
|
|
: json_encode(['payment_id' => (string) ($result['id'] ?? '')], JSON_THROW_ON_ERROR
|
|
| JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE),
|
|
'sealed_payload' => CloudSecrets::encrypt([], 'job-payload'),
|
|
'lease_token' => null, 'lease_until' => 0,
|
|
'available_at' => 0, 'updated_at' => time(),
|
|
]);
|
|
if (!$success) {
|
|
$this->observedErrors[] = 'task_failed';
|
|
}
|
|
}
|
|
|
|
/** Expire accepted or dispatched work into a local terminal failure. */
|
|
public static function expireDueJobs(int $limit = 100): int
|
|
{
|
|
return self::expireDueJobsMatching(null, null, $limit);
|
|
}
|
|
|
|
/**
|
|
* Close only one owner's public tasks when the worker daemon is absent.
|
|
* This path consumes durable receipts and performs local finalization; it
|
|
* never opens an Agent connection or dispatches a mutation.
|
|
*/
|
|
public static function expireDueTasksForOwner(int $userId,
|
|
?int $taskId = null, int $limit = 100): int
|
|
{
|
|
if ($userId < 1 || ($taskId !== null && $taskId < 1)) {
|
|
return 0;
|
|
}
|
|
return self::expireDueJobsMatching($userId, $taskId, $limit);
|
|
}
|
|
|
|
/**
|
|
* Close one already-due task from an administrator read path.
|
|
* This never contacts an Agent and never dispatches a mutation.
|
|
*/
|
|
public static function expireDueTaskForAdmin(int $taskId): int
|
|
{
|
|
if ($taskId < 1) {
|
|
return 0;
|
|
}
|
|
return self::expireDueJobsMatching(null, $taskId, 1);
|
|
}
|
|
|
|
private static function expireDueJobsMatching(?int $userId,
|
|
?int $taskId, int $limit): int
|
|
{
|
|
$now = time();
|
|
$storageDispatchWindow = CloudJobService::dispatchDeadlineSeconds(
|
|
'host_storage_configure'
|
|
);
|
|
$storageWindow = CloudJobService::receiptDeadlineSeconds(
|
|
'host_storage_configure'
|
|
);
|
|
$query = CloudJob::where(static function ($query) use (
|
|
$now, $storageDispatchWindow, $storageWindow
|
|
): void {
|
|
$query->where(static function ($accepted) use (
|
|
$now, $storageDispatchWindow
|
|
): void {
|
|
$accepted->where('phase', 'accepted')
|
|
->where(static function ($deadline) use (
|
|
$now, $storageDispatchWindow
|
|
): void {
|
|
$deadline->where('dispatch_deadline_at', '<=', $now)
|
|
->orWhereNull('dispatch_deadline_at')
|
|
->orWhere(static function ($storage) use (
|
|
$now, $storageDispatchWindow
|
|
): void {
|
|
$storage->where(
|
|
'kind', 'host_storage_configure'
|
|
)->where('created_at', '<=',
|
|
$now - $storageDispatchWindow);
|
|
});
|
|
});
|
|
})->orWhere(static function ($sent) use (
|
|
$now, $storageWindow
|
|
): void {
|
|
$sent->whereIn('phase', ['dispatching', 'awaiting_receipt'])
|
|
->where(static function ($deadline) use (
|
|
$now, $storageWindow
|
|
): void {
|
|
$deadline->where(static function ($due) use (
|
|
$now
|
|
): void {
|
|
$due->whereNotNull('receipt_deadline_at')
|
|
->where('receipt_deadline_at', '<=', $now);
|
|
})->orWhereNull('receipt_deadline_at')
|
|
->orWhere(static function ($storage) use (
|
|
$now, $storageWindow
|
|
): void {
|
|
$storage->where(
|
|
'kind', 'host_storage_configure'
|
|
)->whereNotNull('dispatched_at')
|
|
->where('dispatched_at', '<=',
|
|
$now - $storageWindow);
|
|
});
|
|
});
|
|
});
|
|
})->where(static function ($claimable): void {
|
|
// A VPC batch parent is a PHP-local coordination ledger. It never
|
|
// crosses an Agent dispatch deadline; only its durable child
|
|
// finalizers may complete it.
|
|
$claimable->whereNotIn('kind', [
|
|
'vpc_membership_batch',
|
|
'vpc_batch_instance_fence',
|
|
'public_ip_pool_replacement_batch',
|
|
])
|
|
->orWhere('execution_mode', '<>', 'local');
|
|
});
|
|
if ($userId !== null) {
|
|
// Child jobs are not publicly listed, but the owner-scoped
|
|
// daemon-loss fallback must still terminalize them before their
|
|
// aggregate parent can close.
|
|
$query->where('user_id', $userId);
|
|
}
|
|
if ($taskId !== null) {
|
|
$query->where('id', $taskId);
|
|
}
|
|
$rows = $query
|
|
->orderByRaw('CASE WHEN parent_job_id IS NULL THEN 1 ELSE 0 END')
|
|
->orderBy('id')
|
|
->limit(max(1, min(100, $limit)))->get();
|
|
$expired = 0;
|
|
foreach ($rows as $row) {
|
|
$dispatchConflict = CloudDispatchLedger::isInvalid($row);
|
|
$effect = CloudDispatchLedger::mayHaveDispatched($row);
|
|
$failureCode = $dispatchConflict
|
|
? CloudDispatchLedger::CONFLICT_CODE
|
|
: ($effect
|
|
? 'completion_deadline_exceeded'
|
|
: 'dispatch_deadline_exceeded');
|
|
$token = bin2hex(random_bytes(16));
|
|
$claim = CloudJob::where('id', (int) $row->id)
|
|
->where('phase', (string) $row->phase)
|
|
->where('dispatch_attempts', (int) $row->dispatch_attempts);
|
|
if ($row->lease_token === null) {
|
|
$claim->whereNull('lease_token');
|
|
} else {
|
|
$claim->where('lease_token', (string) $row->lease_token);
|
|
}
|
|
// An absolute deadline wins over a stale worker lease. The CAS on
|
|
// phase/attempt/token prevents two local expiry paths from owning
|
|
// the same finalization.
|
|
$updated = $claim->update([
|
|
'lease_token' => $token,
|
|
'lease_until' => $now + self::LEASE_SECONDS,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
continue;
|
|
}
|
|
$expired++;
|
|
$claimed = CloudJob::find((int) $row->id);
|
|
try {
|
|
if ($claimed !== null
|
|
&& CloudDispatchLedger::isInvalid($claimed)) {
|
|
self::forceTerminalFailure(
|
|
(int) $row->id,
|
|
$token,
|
|
true,
|
|
false,
|
|
CloudDispatchLedger::CONFLICT_CODE
|
|
);
|
|
continue;
|
|
}
|
|
if ($claimed !== null && $claimed->host_id !== null
|
|
&& CloudDispatchLedger::isDispatched($claimed)) {
|
|
$event = CloudOperationCompletionService::pendingForJob(
|
|
$claimed
|
|
);
|
|
if ($event !== null) {
|
|
(new self())->settleCompletionReceipt(
|
|
(int) $claimed->id,
|
|
(string) $event->event_id
|
|
);
|
|
continue;
|
|
}
|
|
}
|
|
if ($claimed !== null
|
|
&& (string) $claimed->execution_mode === 'local'
|
|
&& (string) $claimed->kind === 'payment_create') {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $claimed->sealed_payload, 'job-payload'
|
|
);
|
|
[$success, $result, $effect, $safe, $code]
|
|
= self::expiredPaymentResult($claimed, $payload);
|
|
(new self())->finishPaymentCreate(
|
|
$claimed, $success, $result, $effect, $safe, $code
|
|
);
|
|
continue;
|
|
}
|
|
if ($claimed !== null && $claimed->host_id === null
|
|
&& $claimed->parent_job_id === null
|
|
&& (string) $claimed->execution_mode === 'aggregate') {
|
|
self::expireAggregate($claimed, $token);
|
|
continue;
|
|
}
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $claimed->sealed_payload, 'job-payload'
|
|
);
|
|
$operation = !$effect && CloudOperationRecoveryPolicy::isCreateLike(
|
|
(string) $claimed->kind
|
|
)
|
|
? self::createNotDispatchedFailure()
|
|
: ['status' => 'failed'];
|
|
$operation['operation_id'] = (string) $claimed->operation_id;
|
|
$operation['error_code'] = $failureCode;
|
|
$operation['effect_may_have_applied'] = $effect;
|
|
$operation['safe_to_resubmit'] = !$effect;
|
|
(new self())->finish($claimed, $operation, $payload);
|
|
} catch (\Throwable $error) {
|
|
self::forceTerminalFailure((int) $row->id, $token,
|
|
$effect, !$effect, $failureCode);
|
|
}
|
|
}
|
|
return $expired;
|
|
}
|
|
|
|
/**
|
|
* Derive a local payment task's deadline result from its durable attempt.
|
|
* Claiming the worker is not proof that the provider POST was sent.
|
|
* This read never creates or retransmits a provider order.
|
|
*
|
|
* @return array{0:bool,1:?array,2:bool,3:bool,4:string}
|
|
*/
|
|
private static function expiredPaymentResult(CloudJob $job,
|
|
array $payload): array
|
|
{
|
|
$key = $payload['input']['idempotency_key'] ?? null;
|
|
if (!is_string($key)
|
|
|| preg_match('/^[A-Za-z0-9_.-]{16,128}$/D', $key) !== 1) {
|
|
return [false, null, true, false, 'payment_creation_unknown'];
|
|
}
|
|
$attempt = \App\Models\CloudInvoicePayment::where(
|
|
'user_id', (int) $job->user_id
|
|
)->where('invoice_id', (int) $job->invoice_id)
|
|
->where('idempotency_key', $key)->first();
|
|
if ($attempt === null) {
|
|
return [false, null, false, true,
|
|
'payment_order_not_dispatched'];
|
|
}
|
|
$status = (string) $attempt->status;
|
|
if (in_array($status,
|
|
[
|
|
'pending', 'failed', 'expired',
|
|
'paid_invoice', 'credited_balance',
|
|
], true)) {
|
|
return [true, ['id' => (string) $attempt->id], false, false, ''];
|
|
}
|
|
if ($status === 'not_created') {
|
|
return [false, null, false, true, 'payment_order_rejected'];
|
|
}
|
|
return [false, null, true, false, 'payment_creation_unknown'];
|
|
}
|
|
|
|
public function settleCompletionReceipt(int $jobId,
|
|
string $eventId): CloudJob
|
|
{
|
|
$event = \App\Models\CloudAgentCompletionEvent::where(
|
|
'event_id', $eventId
|
|
)->first();
|
|
if ($event === null) {
|
|
throw new CloudError('完成回执不存在', 404);
|
|
}
|
|
$source = CloudJob::find($jobId);
|
|
if ($source === null) {
|
|
throw new CloudError('任务不存在', 404);
|
|
}
|
|
if ((string) $source->execution_mode !== 'agent'
|
|
|| $source->host_id === null
|
|
|| (int) $event->host_id !== (int) $source->host_id
|
|
|| !hash_equals((string) $source->operation_id,
|
|
(string) $event->operation_id)) {
|
|
CloudOperationCompletionService::mark($event, 'rejected');
|
|
throw new CloudError('完成回执与任务不匹配', 409);
|
|
}
|
|
if (CloudDispatchLedger::isInvalid($source)) {
|
|
CloudOperationCompletionService::deferDispatchLedgerConflict(
|
|
$source
|
|
);
|
|
return CloudJob::find($jobId) ?? $source;
|
|
}
|
|
if ((string) $source->phase === 'terminal') {
|
|
if (CloudOperationCompletionService::settledTaskWithEvent(
|
|
$source,
|
|
$event
|
|
)) {
|
|
CloudOperationCompletionService::mark($event, 'applied');
|
|
} else {
|
|
CloudOperationRecoveryService::settleLateReceipt(
|
|
$source,
|
|
$event
|
|
);
|
|
}
|
|
return CloudJob::find($jobId) ?? $source;
|
|
}
|
|
if ((string) $event->disposition !== 'received') {
|
|
throw new CloudError('完成回执已经处理', 409);
|
|
}
|
|
$receiptDeadline = CloudJobService::effectiveReceiptDeadline($source);
|
|
if ($receiptDeadline < 1
|
|
|| (int) $event->received_at >= $receiptDeadline) {
|
|
// The durable receive timestamp, rather than PHP processing time,
|
|
// decides the boundary. A receipt persisted before the absolute
|
|
// deadline can still settle later; one received after it is audit
|
|
// evidence only and can never revive or succeed the generation.
|
|
$token = bin2hex(random_bytes(16));
|
|
$claim = CloudJob::where('id', $jobId)
|
|
->whereIn('phase', ['dispatching', 'awaiting_receipt'])
|
|
->where('dispatch_attempts', 1);
|
|
if ($source->lease_token === null) {
|
|
$claim->whereNull('lease_token');
|
|
} else {
|
|
$claim->where('lease_token', (string) $source->lease_token);
|
|
}
|
|
$claimed = $claim->update([
|
|
'lease_token' => $token,
|
|
'lease_until' => time() + self::LEASE_SECONDS,
|
|
'updated_at' => time(),
|
|
]);
|
|
if ($claimed === 1) {
|
|
self::forceTerminalFailure(
|
|
$jobId,
|
|
$token,
|
|
true,
|
|
false,
|
|
'completion_deadline_exceeded'
|
|
);
|
|
}
|
|
$current = CloudJob::find($jobId) ?? $source;
|
|
if ((string) $current->phase === 'terminal') {
|
|
CloudOperationRecoveryService::settleLateReceipt(
|
|
$current,
|
|
$event
|
|
);
|
|
}
|
|
return $current;
|
|
}
|
|
$operation = CloudOperationCompletionService::operation($event);
|
|
$payload = CloudSecrets::decrypt((string) $source->sealed_payload,
|
|
'job-payload');
|
|
$token = bin2hex(random_bytes(16));
|
|
$claim = CloudJob::where('id', $jobId)
|
|
->whereIn('phase', ['dispatching', 'awaiting_receipt'])
|
|
->where('dispatch_attempts', 1);
|
|
if ($source->lease_token === null) {
|
|
$claim->whereNull('lease_token');
|
|
} else {
|
|
$claim->where('lease_token', (string) $source->lease_token);
|
|
}
|
|
$claimed = $claim->update([
|
|
'lease_token' => $token,
|
|
'lease_until' => time() + self::LEASE_SECONDS,
|
|
'completion_event_id' => $eventId,
|
|
'updated_at' => time(),
|
|
]);
|
|
if ($claimed !== 1) {
|
|
$current = CloudJob::find($jobId) ?? $source;
|
|
if ((string) $current->phase === 'terminal') {
|
|
if (CloudOperationCompletionService::settledTaskWithEvent(
|
|
$current,
|
|
$event
|
|
)) {
|
|
CloudOperationCompletionService::mark($event, 'applied');
|
|
} else {
|
|
CloudOperationRecoveryService::settleLateReceipt(
|
|
$current,
|
|
$event
|
|
);
|
|
}
|
|
}
|
|
return $current;
|
|
}
|
|
$job = CloudJob::find($jobId);
|
|
$normalized = self::normalizeAgentOperation(
|
|
self::operationForJob($job, $operation)
|
|
);
|
|
try {
|
|
$this->finish($job, $normalized, $payload);
|
|
} catch (\Throwable $error) {
|
|
$createNotApplied = CloudOperationRecoveryPolicy::isCreateLike(
|
|
(string) $job->kind
|
|
)
|
|
&& strtolower((string) ($normalized['status'] ?? ''))
|
|
=== 'failed'
|
|
&& self::createFailureConfirmedAbsent($normalized);
|
|
$safeToResubmit = $createNotApplied
|
|
&& ($normalized['safe_to_resubmit'] ?? false) === true;
|
|
self::forceTerminalFailure(
|
|
$jobId,
|
|
$token,
|
|
!$createNotApplied,
|
|
$safeToResubmit,
|
|
'completion_finalization_failed'
|
|
);
|
|
}
|
|
$current = CloudJob::find($jobId) ?? $source;
|
|
if (CloudOperationCompletionService::settledTaskWithEvent(
|
|
$current,
|
|
$event
|
|
)) {
|
|
CloudOperationCompletionService::mark($event, 'applied');
|
|
} elseif ((string) $current->phase === 'terminal') {
|
|
CloudOperationRecoveryService::settleLateReceipt(
|
|
$current,
|
|
$event
|
|
);
|
|
}
|
|
return $current;
|
|
}
|
|
|
|
private static function forceTerminalFailure(int $jobId, string $token,
|
|
bool $effect, bool $safe, string $code,
|
|
bool $confirmedNotDispatched = false): void
|
|
{
|
|
$claimed = CloudJob::where('id', $jobId)
|
|
->where('lease_token', $token)->where('phase', '<>', 'terminal')
|
|
->first();
|
|
$recoverableFinalizer = $claimed !== null
|
|
&& (CloudOperationRecoveryPolicy::supports(
|
|
(string) $claimed->kind
|
|
) || CloudOperationRecoveryPolicy::isCreateLike(
|
|
(string) $claimed->kind
|
|
));
|
|
$preserveRecoveryEnvelope = $recoverableFinalizer
|
|
|| ($claimed !== null
|
|
&& CloudOperationRecoveryPolicy::isCreateLike(
|
|
(string) $claimed->kind
|
|
));
|
|
// Preserve the opaque escrow before attempting decryption. A temporary
|
|
// key/configuration failure is itself recoverable; replacing the only
|
|
// encrypted request with an empty string would make the PHP-only
|
|
// finalizer impossible to resume after the environment is repaired.
|
|
$recoveryEnvelope = $preserveRecoveryEnvelope
|
|
? (string) $claimed->sealed_payload : '';
|
|
$incidentCode = $code;
|
|
$dispatchConflict = $claimed !== null
|
|
&& CloudDispatchLedger::isInvalid($claimed);
|
|
if ($dispatchConflict) {
|
|
// A corrupt counter can never prove non-dispatch. Do not invoke a
|
|
// normal success/not-applied finalizer or consume Agent evidence;
|
|
// preserve the encrypted escrow so repair can resume this exact
|
|
// operation without replaying it.
|
|
$effect = true;
|
|
$safe = false;
|
|
$code = CloudDispatchLedger::CONFLICT_CODE;
|
|
$recoveryEnvelope = (string) $claimed->sealed_payload;
|
|
} elseif ($claimed !== null) {
|
|
try {
|
|
if ((string) $claimed->execution_mode === 'aggregate') {
|
|
if (!self::forceAggregateFailure($claimed, $token)) {
|
|
return;
|
|
}
|
|
} elseif ((string) $claimed->kind === 'payment_create') {
|
|
[$success, $result, $paymentEffect, $paymentSafe,
|
|
$paymentCode] = self::expiredPaymentResult(
|
|
$claimed,
|
|
['input' => ['idempotency_key' =>
|
|
(string) $claimed->request_idempotency_key]]
|
|
);
|
|
(new self())->finishPaymentCreate(
|
|
$claimed,
|
|
$success,
|
|
$result,
|
|
$paymentEffect,
|
|
$paymentSafe,
|
|
$paymentCode
|
|
);
|
|
} else {
|
|
try {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $claimed->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
} catch (\Throwable $error) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'任务加密恢复数据暂时无法读取',
|
|
409
|
|
);
|
|
}
|
|
// Every explicitly registered recovery kind retains its
|
|
// encrypted request until the exact Agent operation is
|
|
// reconciled. The recovery worker never re-dispatches
|
|
// this payload; it is needed only by PHP's existing
|
|
// kind-specific ledger finalizers.
|
|
if (CloudOperationRecoveryPolicy::supports(
|
|
(string) $claimed->kind
|
|
) || CloudOperationRecoveryPolicy::isCreateLike(
|
|
(string) $claimed->kind
|
|
)) {
|
|
// An exact create success receipt can finish the paid
|
|
// order only while the encrypted wire request and
|
|
// credential escrow remain available. Keep it across
|
|
// a local finalizer crash just like policy recovery
|
|
// payloads; successful convergence scrubs it.
|
|
$recoveryEnvelope = (string) $claimed->sealed_payload;
|
|
} else {
|
|
$safePayload = CloudInput::safeResult($payload);
|
|
if (CloudDispatchLedger::isDispatched($claimed)
|
|
&& CloudOperationRecoveryIdentity::supports(
|
|
(string) $claimed->kind
|
|
)) {
|
|
$identity = CloudOperationRecoveryIdentity::
|
|
expectedFromPayload($claimed, $payload, false);
|
|
$safePayload[
|
|
CloudOperationRecoveryIdentity::payloadKey()
|
|
] = CloudOperationRecoveryIdentity::persisted(
|
|
$identity
|
|
);
|
|
}
|
|
$recoveryEnvelope = CloudSecrets::encrypt(
|
|
$safePayload,
|
|
'job-payload'
|
|
);
|
|
}
|
|
(new self())->finish($claimed, [
|
|
'operation_id' => (string) $claimed->operation_id,
|
|
'status' => 'failed',
|
|
'error_code' => $code,
|
|
'effect_may_have_applied' => $effect,
|
|
'safe_to_resubmit' => $safe,
|
|
], $payload);
|
|
}
|
|
$terminal = CloudJob::where('id', $jobId)
|
|
->where('phase', 'terminal')->exists();
|
|
if ($terminal) {
|
|
return;
|
|
}
|
|
} catch (\Throwable $finalizationError) {
|
|
$incidentCode = self::localFinalizationIncidentCode(
|
|
$finalizationError
|
|
);
|
|
error_log(sprintf(
|
|
'cloud task ledger finalization failed job_id=%d kind=%s error=%s',
|
|
$jobId,
|
|
(string) $claimed->kind,
|
|
get_class($finalizationError)
|
|
));
|
|
}
|
|
}
|
|
if ($claimed !== null
|
|
&& (string) $claimed->execution_mode === 'aggregate'
|
|
&& self::releaseAggregateParentWhenChildrenActive(
|
|
(int) $claimed->id,
|
|
$token
|
|
)) {
|
|
// A child can win the lease CAS between our snapshot and claim.
|
|
// Never hide still-active child work behind a terminal parent;
|
|
// a later bounded deadline pass will finish children first.
|
|
return;
|
|
}
|
|
// An order-backed create must never fall through to a job-only write:
|
|
// its wallet, invoice, stock and placement ledgers are one atomic
|
|
// finalization unit. Ambiguous effects retain a durable placement
|
|
// fence; proven no-effect failures refund and release exactly once.
|
|
if ($claimed !== null && (string) $claimed->kind === 'create'
|
|
&& $claimed->order_id !== null) {
|
|
try {
|
|
CloudCreateFailureFinalizer::forceTerminalFailure(
|
|
$jobId,
|
|
$token,
|
|
$effect,
|
|
$safe,
|
|
$code
|
|
);
|
|
if (CloudJob::where('id', $jobId)
|
|
->where('phase', 'terminal')->exists()) {
|
|
return;
|
|
}
|
|
} catch (\Throwable $finalizationError) {
|
|
error_log(sprintf(
|
|
'cloud create ledger finalization incident job_id=%d error=%s',
|
|
$jobId,
|
|
get_class($finalizationError)
|
|
));
|
|
}
|
|
}
|
|
if ($claimed !== null) {
|
|
// Best-effort resource latches make the incident visible in the
|
|
// affected catalog. Their own failure must never keep the public
|
|
// task active; the durable finalization_incident below remains the
|
|
// authoritative operator alert.
|
|
try {
|
|
CloudSnapshotService::markJobUncertain($claimed);
|
|
} catch (\Throwable $ignored) {
|
|
}
|
|
try {
|
|
CloudBackupService::markJobUncertain($claimed);
|
|
} catch (\Throwable $ignored) {
|
|
}
|
|
try {
|
|
CloudFirewallService::markJobUncertain($claimed);
|
|
} catch (\Throwable $ignored) {
|
|
}
|
|
try {
|
|
CloudVpcPortService::markJobUncertain($claimed);
|
|
} catch (\Throwable $ignored) {
|
|
}
|
|
if ((string) $claimed->kind === 'network_change'
|
|
&& $claimed->public_network_transition_id !== null) {
|
|
try {
|
|
CloudPublicNetworkTransitionWorkflow::markJobUncertain(
|
|
$claimed
|
|
);
|
|
} catch (\Throwable $ignored) {
|
|
}
|
|
}
|
|
}
|
|
// Last-resort terminalization is deliberately business-conservative:
|
|
// never refund, release or claim success after a ledger finalizer
|
|
// failed. Retain a validated encrypted request only when it can support
|
|
// later ledger recovery, retain every resource fence, and publish an
|
|
// explicit durable incident for operators.
|
|
// Registered policy kinds and an order-backed create have PHP-only
|
|
// retry paths that can safely publish a known not-applied effect for
|
|
// generation zero or for one exact transport-level
|
|
// dispatch_state=not_dispatched proof. The immutable counter remains
|
|
// one in the latter case. Aggregate/other special finalizers retain
|
|
// the conservative unknown effect if their atomic fallback fails.
|
|
$neverDispatched = $recoverableFinalizer
|
|
&& CloudDispatchLedger::isNeverDispatched($claimed);
|
|
$transportNotDispatched = $recoverableFinalizer
|
|
&& $confirmedNotDispatched
|
|
&& !$effect
|
|
&& $safe
|
|
&& CloudDispatchLedger::isDispatched($claimed)
|
|
&& ($claimed->agent_accepted_at ?? null) === null
|
|
&& ($claimed->completion_event_id ?? null) === null;
|
|
$notDispatched = $neverDispatched || $transportNotDispatched;
|
|
$terminalUpdate = [
|
|
'phase' => 'terminal', 'outcome' => 'failed',
|
|
'status' => 'failed', 'terminal_at' => time(),
|
|
'effect_may_have_applied' => $notDispatched ? 0 : 1,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => $notDispatched
|
|
? 'not_applied' : 'unknown',
|
|
'error_code' => $dispatchConflict
|
|
? CloudDispatchLedger::CONFLICT_CODE
|
|
: 'local_finalization_failed',
|
|
'finalization_incident' => 1,
|
|
'finalization_error_code' => substr(
|
|
$dispatchConflict
|
|
? CloudDispatchLedger::CONFLICT_CODE
|
|
: $incidentCode,
|
|
0,
|
|
128
|
|
),
|
|
'sealed_payload' => $recoveryEnvelope,
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'available_at' => ($dispatchConflict
|
|
|| $recoverableFinalizer) ? time() + 300 : 0,
|
|
'updated_at' => time(),
|
|
];
|
|
if ($transportNotDispatched) {
|
|
$terminalUpdate['effect_resolution_reason'] =
|
|
CloudDispatchLedger::TRANSPORT_NOT_DISPATCHED_REASON;
|
|
}
|
|
CloudJob::where('id', $jobId)->where('lease_token', $token)
|
|
->where('phase', '<>', 'terminal')->update($terminalUpdate);
|
|
}
|
|
|
|
private static function localFinalizationIncidentCode(
|
|
\Throwable $error
|
|
): string {
|
|
return match (true) {
|
|
$error instanceof CloudRecoveryDispatchLedgerConflict =>
|
|
CloudDispatchLedger::CONFLICT_CODE,
|
|
$error instanceof CloudRecoveryAggregateStateConflict =>
|
|
'recovery_aggregate_state_conflict',
|
|
$error instanceof CloudRecoveryPayloadUnavailable =>
|
|
'recovery_payload_unavailable',
|
|
$error instanceof CloudRecoveryOperationIdentityConflict =>
|
|
'recovery_operation_identity_conflict',
|
|
$error instanceof CloudRecoveryReceiptInvalid =>
|
|
'recovery_receipt_invalid',
|
|
$error instanceof CloudRecoveryInstanceStateConflict =>
|
|
'recovery_instance_state_conflict',
|
|
$error instanceof CloudError => 'recovery_ledger_conflict',
|
|
default => 'recovery_internal_error',
|
|
};
|
|
}
|
|
|
|
/** Close aggregate children first, then atomically converge its catalog. */
|
|
private static function forceAggregateFailure(CloudJob $parent,
|
|
string $token): bool
|
|
{
|
|
$children = CloudJob::where('parent_job_id', (int) $parent->id)
|
|
->where('phase', '<>', 'terminal')->orderBy('id')->get();
|
|
foreach ($children as $child) {
|
|
$childToken = bin2hex(random_bytes(16));
|
|
$claim = CloudJob::where('id', (int) $child->id)
|
|
->where('phase', (string) $child->phase)
|
|
->where('dispatch_attempts', (int) $child->dispatch_attempts);
|
|
if ($child->lease_token === null) {
|
|
$claim->whereNull('lease_token');
|
|
} else {
|
|
$claim->where('lease_token', (string) $child->lease_token);
|
|
}
|
|
if ($claim->update([
|
|
'lease_token' => $childToken,
|
|
'lease_until' => time() + self::LEASE_SECONDS,
|
|
'updated_at' => time(),
|
|
]) === 1) {
|
|
$crossed = CloudDispatchLedger::mayHaveDispatched($child);
|
|
$failureCode = CloudDispatchLedger::isInvalid($child)
|
|
? CloudDispatchLedger::CONFLICT_CODE
|
|
: 'aggregate_finalization_failed';
|
|
self::forceTerminalFailure(
|
|
(int) $child->id,
|
|
$childToken,
|
|
$crossed,
|
|
!$crossed,
|
|
$failureCode
|
|
);
|
|
}
|
|
}
|
|
if (self::releaseAggregateParentWhenChildrenActive(
|
|
(int) $parent->id,
|
|
$token
|
|
)) {
|
|
return false;
|
|
}
|
|
(new CloudJob())->getConnection()->transaction(
|
|
static function () use ($parent, $token): void {
|
|
$fresh = CloudJob::where('id', (int) $parent->id)
|
|
->where('lease_token', $token)
|
|
->lockForUpdate()->first();
|
|
if ($fresh === null
|
|
|| (string) $fresh->phase === 'terminal') {
|
|
return;
|
|
}
|
|
CloudUserIsoService::expireAggregateLocked($fresh);
|
|
CloudTemplateCatalogService::expireAggregateLocked($fresh);
|
|
$fresh = CloudJob::where('id', (int) $parent->id)
|
|
->where('lease_token', $token)->lockForUpdate()->first();
|
|
if ($fresh !== null
|
|
&& (string) $fresh->phase !== 'terminal') {
|
|
CloudJobService::completeAggregateLocked(
|
|
$fresh,
|
|
'failed',
|
|
true,
|
|
false,
|
|
'aggregate_finalization_failed'
|
|
);
|
|
}
|
|
},
|
|
3
|
|
);
|
|
return true;
|
|
}
|
|
|
|
/** Release the parent lease while any child can still execute. */
|
|
private static function releaseAggregateParentWhenChildrenActive(
|
|
int $parentId,
|
|
string $token
|
|
): bool {
|
|
if (!CloudJob::where('parent_job_id', $parentId)
|
|
->where('phase', '<>', 'terminal')->exists()) {
|
|
return false;
|
|
}
|
|
CloudJob::where('id', $parentId)
|
|
->where('lease_token', $token)
|
|
->where('phase', '<>', 'terminal')->update([
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'updated_at' => time(),
|
|
]);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* A parent deadline is an absolute local boundary. Close every remaining
|
|
* child without contacting the Agent, let resource finalizers consume
|
|
* those failures, then close any aggregate that still has no result.
|
|
*/
|
|
private static function expireAggregate(CloudJob $parent,
|
|
string $token): void
|
|
{
|
|
$children = CloudJob::where('parent_job_id', (int) $parent->id)
|
|
->where('phase', '<>', 'terminal')->orderBy('id')->get();
|
|
foreach ($children as $child) {
|
|
if ($child->host_id !== null
|
|
&& CloudDispatchLedger::isDispatched($child)) {
|
|
$event = CloudOperationCompletionService::pendingForJob(
|
|
$child
|
|
);
|
|
if ($event !== null) {
|
|
try {
|
|
(new self())->settleCompletionReceipt(
|
|
(int) $child->id,
|
|
(string) $event->event_id
|
|
);
|
|
} catch (\Throwable $ignored) {
|
|
// A malformed/unapplicable receipt cannot keep the
|
|
// public parent active past its absolute deadline.
|
|
}
|
|
$child = CloudJob::find((int) $child->id) ?? $child;
|
|
if ((string) $child->phase === 'terminal') {
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
$childToken = bin2hex(random_bytes(16));
|
|
$claim = CloudJob::where('id', (int) $child->id)
|
|
->where('phase', (string) $child->phase)
|
|
->where('dispatch_attempts', (int) $child->dispatch_attempts);
|
|
if ($child->lease_token === null) {
|
|
$claim->whereNull('lease_token');
|
|
} else {
|
|
$claim->where('lease_token', (string) $child->lease_token);
|
|
}
|
|
$claimed = $claim->update([
|
|
'lease_token' => $childToken,
|
|
'lease_until' => time() + self::LEASE_SECONDS,
|
|
'updated_at' => time(),
|
|
]);
|
|
if ($claimed !== 1) {
|
|
continue;
|
|
}
|
|
$crossed = CloudDispatchLedger::mayHaveDispatched($child);
|
|
$failureCode = CloudDispatchLedger::isInvalid($child)
|
|
? CloudDispatchLedger::CONFLICT_CODE
|
|
: 'aggregate_deadline_exceeded';
|
|
self::forceTerminalFailure((int) $child->id, $childToken,
|
|
$crossed, !$crossed, $failureCode);
|
|
}
|
|
if (CloudJob::where('parent_job_id', (int) $parent->id)
|
|
->where('phase', '<>', 'terminal')->exists()) {
|
|
// A child changed concurrently after our snapshot. Keep the parent
|
|
// active and release this local lease; a later bounded fallback
|
|
// pass will close the child first. Never hide active child work
|
|
// behind a terminal parent.
|
|
CloudJob::where('id', (int) $parent->id)
|
|
->where('lease_token', $token)
|
|
->where('phase', '<>', 'terminal')->update([
|
|
'lease_token' => null,
|
|
'lease_until' => 0,
|
|
'updated_at' => time(),
|
|
]);
|
|
return;
|
|
}
|
|
(new CloudJob())->getConnection()->transaction(
|
|
static function () use ($parent, $token): void {
|
|
$fresh = CloudJob::where('id', (int) $parent->id)
|
|
->where('lease_token', $token)
|
|
->lockForUpdate()->first();
|
|
if ($fresh === null
|
|
|| (string) $fresh->phase === 'terminal') {
|
|
return;
|
|
}
|
|
CloudUserIsoService::expireAggregateLocked($fresh);
|
|
CloudTemplateCatalogService::expireAggregateLocked($fresh);
|
|
$fresh = CloudJob::where('id', (int) $parent->id)
|
|
->where('lease_token', $token)->lockForUpdate()->first();
|
|
if ($fresh !== null
|
|
&& (string) $fresh->phase !== 'terminal') {
|
|
CloudJobService::completeAggregateLocked(
|
|
$fresh, 'failed', true, false,
|
|
'aggregate_deadline_exceeded'
|
|
);
|
|
}
|
|
}, 3
|
|
);
|
|
}
|
|
|
|
private static function policyDesired(?CloudInstance $instance, string $kind): bool
|
|
{
|
|
if ($instance === null) {
|
|
return false;
|
|
}
|
|
$status = (string) $instance->status;
|
|
return match ($kind) {
|
|
'service_suspend' => in_array($status, ['active', 'deleting'], true)
|
|
&& (string) $instance->service_status === 'purge_stopping',
|
|
'network_suspend' => $status === 'active'
|
|
&& in_array((string) $instance->network_status,
|
|
['blocking', 'blocking_expired'], true),
|
|
'network_resume' => $status === 'active'
|
|
&& in_array((string) $instance->network_status,
|
|
['restore_pending', 'restoring'], true),
|
|
default => false,
|
|
};
|
|
}
|
|
|
|
private static function validPreparedRecoveryMarker(string $marker): bool
|
|
{
|
|
if (strlen($marker) > 500) {
|
|
return false;
|
|
}
|
|
return preg_match(
|
|
'/^prepared:[a-z][a-z0-9_]{2,63}:[A-Za-z0-9_.-]{1,160}:[a-f0-9]{64}$/D',
|
|
$marker
|
|
) === 1 || preg_match(
|
|
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:[1-9][0-9]{0,9}:prepared:(?:automatic_(?:applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}|late_completion_receipt:[A-Za-z0-9_.-]{1,160}):[a-f0-9]{64}$/D',
|
|
$marker
|
|
) === 1;
|
|
}
|
|
|
|
/**
|
|
* Return the immutable prepared-evidence binding without the mutable
|
|
* automatic retry counter. Older prepared envelopes stored the complete
|
|
* automatic marker, while new envelopes store only this stable suffix.
|
|
*/
|
|
private static function preparedRecoveryBinding(string $marker): ?string
|
|
{
|
|
if (preg_match(
|
|
'/^(?:automatic_recovery:v1:[1-9][0-9]{0,10}:'
|
|
. '[1-9][0-9]{0,9}:)?'
|
|
. '(prepared:[a-z][a-z0-9_]{2,63}:'
|
|
. '[A-Za-z0-9_.-]{1,160}:[a-f0-9]{64})$/D',
|
|
$marker,
|
|
$matches
|
|
) !== 1) {
|
|
return null;
|
|
}
|
|
return $matches[1];
|
|
}
|
|
|
|
/**
|
|
* Capability URLs are needed only for the one Agent dispatch. Once exact
|
|
* operation identity and recovery evidence have been validated, retaining
|
|
* hundreds of expired multipart URLs would make the prepared recovery
|
|
* envelope unnecessarily large. The persisted request fingerprint still
|
|
* binds the complete original wire request.
|
|
*/
|
|
private static function compactPreparedRecoveryPayload(
|
|
string $kind,
|
|
array $payload
|
|
): array {
|
|
$identity = $payload[
|
|
CloudOperationRecoveryIdentity::payloadKey()
|
|
] ?? null;
|
|
if ($kind === 'backup_create') {
|
|
return [
|
|
CloudOperationRecoveryIdentity::payloadKey() => $identity,
|
|
'owner_id' => $payload['owner_id'] ?? null,
|
|
'can_manage' => $payload['can_manage'] ?? null,
|
|
'backup_key' => $payload['backup_key'] ?? null,
|
|
'maximum_size_bytes' =>
|
|
$payload['maximum_size_bytes'] ?? null,
|
|
];
|
|
}
|
|
if ($kind === 'backup_restore') {
|
|
return CloudPortableRestoreJobEnvelope::compactRecoveryPayload(
|
|
$payload
|
|
);
|
|
}
|
|
if ($kind === 'instance_migration_prepare') {
|
|
$body = is_array($payload['body'] ?? null)
|
|
? $payload['body'] : [];
|
|
return [
|
|
CloudOperationRecoveryIdentity::payloadKey() => $identity,
|
|
'owner_id' => $payload['owner_id'] ?? null,
|
|
'can_manage' => $payload['can_manage'] ?? null,
|
|
'maintenance_migration_id' =>
|
|
$payload['maintenance_migration_id'] ?? null,
|
|
'body' => [
|
|
'ticket' => $body['ticket'] ?? null,
|
|
],
|
|
];
|
|
}
|
|
if ($kind === 'instance_migration_transfer') {
|
|
$body = is_array($payload['body'] ?? null)
|
|
? $payload['body'] : [];
|
|
return [
|
|
CloudOperationRecoveryIdentity::payloadKey() => $identity,
|
|
'owner_id' => $payload['owner_id'] ?? null,
|
|
'can_manage' => $payload['can_manage'] ?? null,
|
|
'reservation_id' => $payload['reservation_id'] ?? null,
|
|
'maintenance_migration_id' =>
|
|
$payload['maintenance_migration_id'] ?? null,
|
|
'body' => [
|
|
'vcpu' => $body['vcpu'] ?? null,
|
|
'memory_mib' => $body['memory_mib'] ?? null,
|
|
'disk_gib' => $body['disk_gib'] ?? null,
|
|
'bandwidth_mbps' => $body['bandwidth_mbps'] ?? null,
|
|
'public_network' => $body['public_network'] ?? null,
|
|
'vpc_port' => $body['vpc_port'] ?? null,
|
|
],
|
|
];
|
|
}
|
|
if ($kind === 'backup_recover') {
|
|
$body = is_array($payload['body'] ?? null)
|
|
? $payload['body'] : [];
|
|
return [
|
|
CloudOperationRecoveryIdentity::payloadKey() => $identity,
|
|
'owner_id' => $payload['owner_id'] ?? null,
|
|
'can_manage' => $payload['can_manage'] ?? null,
|
|
'backup_key' => $payload['backup_key'] ?? null,
|
|
'recovery_key' => $payload['recovery_key'] ?? null,
|
|
'body' => [
|
|
'backup_key' => $body['backup_key'] ?? null,
|
|
'source_size_bytes' =>
|
|
$body['source_size_bytes'] ?? null,
|
|
'source_sha256' => $body['source_sha256'] ?? null,
|
|
],
|
|
];
|
|
}
|
|
return $payload;
|
|
}
|
|
|
|
private static function lateReceiptRecoveryMarker(string $marker): bool
|
|
{
|
|
return preg_match(
|
|
'/^(?:automatic_recovery:v1:[1-9][0-9]{0,10}:[1-9][0-9]{0,9}:)?prepared:late_completion_receipt:[A-Za-z0-9_.-]{1,160}:[a-f0-9]{64}$/D',
|
|
$marker
|
|
) === 1;
|
|
}
|
|
|
|
private static function preparedRecoveryTransitionAllowed(
|
|
string $current,
|
|
string $prepared,
|
|
array $operation
|
|
): bool {
|
|
if ($current === '' || hash_equals($prepared, $current)) {
|
|
return true;
|
|
}
|
|
$legacyOperationHash = CloudOperationRecoveryService::
|
|
legacyCompletionOperationHash($operation);
|
|
$currentLate = [];
|
|
$preparedLate = [];
|
|
if ($legacyOperationHash !== null
|
|
&& preg_match(
|
|
'/^(?:automatic_recovery:v1:[1-9][0-9]{0,10}:'
|
|
. '[1-9][0-9]{0,9}:)?'
|
|
. 'prepared:late_completion_receipt:'
|
|
. '([A-Za-z0-9_.-]{1,160}):([a-f0-9]{64})$/D',
|
|
$current,
|
|
$currentLate
|
|
) === 1
|
|
&& preg_match(
|
|
'/^prepared:late_completion_receipt:'
|
|
. '([A-Za-z0-9_.-]{1,160}):([a-f0-9]{64})$/D',
|
|
$prepared,
|
|
$preparedLate
|
|
) === 1
|
|
&& hash_equals($currentLate[1], $preparedLate[1])
|
|
&& hash_equals($currentLate[2], $legacyOperationHash)
|
|
&& hash_equals(
|
|
$preparedLate[2],
|
|
CloudOperationRecoveryService::preparedOperationHash(
|
|
$operation
|
|
)
|
|
)) {
|
|
return true;
|
|
}
|
|
if (preg_match(
|
|
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:[1-9][0-9]{0,9}:pending$/D',
|
|
$current
|
|
) === 1) {
|
|
if (preg_match(
|
|
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:1:prepared:automatic_(applied|not_applied)_(?:succeeded|failed)_(?:fence|nofence):[a-f0-9]{64}:[a-f0-9]{64}$/D',
|
|
$prepared,
|
|
$next
|
|
) === 1) {
|
|
return ($next[1] === 'applied'
|
|
&& ($operation['status'] ?? null) === 'succeeded')
|
|
|| ($next[1] === 'not_applied'
|
|
&& ($operation['status'] ?? null) === 'failed');
|
|
}
|
|
return preg_match(
|
|
'/^prepared:late_completion_receipt:[A-Za-z0-9_.-]{1,160}:[a-f0-9]{64}$/D',
|
|
$prepared
|
|
) === 1;
|
|
}
|
|
if (preg_match(
|
|
'/^automatic_recovery:v1:[1-9][0-9]{0,10}:'
|
|
. '[1-9][0-9]{0,9}:prepared:automatic_'
|
|
. '(applied|not_applied)_(succeeded|failed)'
|
|
. '_(fence|nofence):[a-f0-9]{64}:([a-f0-9]{64})$/D',
|
|
$current,
|
|
$existing
|
|
) !== 1) {
|
|
return false;
|
|
}
|
|
if (preg_match(
|
|
'/^prepared:late_completion_receipt:[A-Za-z0-9_.-]{1,160}:([a-f0-9]{64})$/D',
|
|
$prepared,
|
|
$late
|
|
) !== 1) {
|
|
return false;
|
|
}
|
|
$operationHashMatches = hash_equals($existing[4], $late[1]);
|
|
if (!$operationHashMatches && $legacyOperationHash !== null) {
|
|
$operationHashMatches = hash_equals(
|
|
$existing[4],
|
|
$legacyOperationHash
|
|
);
|
|
}
|
|
return $existing[3] === 'nofence'
|
|
&& $operationHashMatches
|
|
&& (($existing[1] === 'applied'
|
|
&& $existing[2] === 'succeeded'
|
|
&& ($operation['status'] ?? null) === 'succeeded')
|
|
|| ($existing[1] === 'not_applied'
|
|
&& $existing[2] === 'failed'
|
|
&& ($operation['status'] ?? null) === 'failed'));
|
|
}
|
|
|
|
private static function operationErrorCode(array $operation): string
|
|
{
|
|
$code = is_string($operation['error_code'] ?? null)
|
|
? strtolower(trim($operation['error_code'])) : '';
|
|
if ($code === '' && is_array($operation['error'] ?? null)
|
|
&& is_string($operation['error']['code'] ?? null)) {
|
|
$code = strtolower(trim($operation['error']['code']));
|
|
}
|
|
return preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $code) === 1
|
|
? $code : '';
|
|
}
|
|
|
|
/**
|
|
* Validate the common immutable job envelope used by every automatic
|
|
* recovery strategy. Kind-specific business validation remains in
|
|
* finish(), which is the one authoritative ledger finalizer.
|
|
*
|
|
* @return array{payload:array,instance:?CloudInstance}
|
|
*/
|
|
public static function recoveryContext(CloudJob $job): array
|
|
{
|
|
CloudDispatchLedger::assertValid($job);
|
|
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
(string) $job->kind
|
|
)
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|| (string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| (int) $job->dispatch_attempts !== 1
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|| (bool) $job->safe_to_resubmit
|
|
|| (string) $job->effect_resolution !== 'unknown') {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'任务不处于可自动核对状态',
|
|
409
|
|
);
|
|
}
|
|
if (!is_string($job->operation_id)
|
|
|| preg_match('/^[a-f0-9]{32}$/D', $job->operation_id) !== 1) {
|
|
throw new CloudRecoveryOperationIdentityConflict(
|
|
'任务缺少合法的 Agent 操作标识',
|
|
409
|
|
);
|
|
}
|
|
if ((int) $job->host_id < 1
|
|
|| $job->dispatched_at === null
|
|
|| (int) $job->dispatched_at < 1) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'任务缺少宿主机或派发时间',
|
|
409
|
|
);
|
|
}
|
|
try {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $job->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
} catch (\Throwable $error) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'任务恢复数据无法解密',
|
|
409
|
|
);
|
|
}
|
|
if (!is_array($payload) || array_is_list($payload)) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'任务恢复数据结构不完整',
|
|
409
|
|
);
|
|
}
|
|
$instance = null;
|
|
if ($job->instance_id !== null) {
|
|
$instance = CloudInstance::find((int) $job->instance_id);
|
|
if ($instance === null
|
|
|| (int) $instance->host_id !== (int) $job->host_id
|
|
|| !is_string($instance->uuid)
|
|
|| preg_match(
|
|
'/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/D',
|
|
strtolower((string) $instance->uuid)
|
|
) !== 1) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'任务与实例账本不匹配',
|
|
409
|
|
);
|
|
}
|
|
}
|
|
return compact('payload', 'instance');
|
|
}
|
|
|
|
/**
|
|
* Return the exact normalized operation durably stored with a prepared
|
|
* recovery. This is encrypted at rest and bound to both kind and marker.
|
|
*/
|
|
public static function preparedRecoveryOperation(CloudJob $job): ?array
|
|
{
|
|
$context = self::recoveryContext($job);
|
|
$envelope = $context['payload']['_operation_recovery'] ?? null;
|
|
if ($envelope === null) {
|
|
return null;
|
|
}
|
|
$operation = is_array($envelope)
|
|
? ($envelope['operation'] ?? null) : null;
|
|
$storedMarker = is_array($envelope)
|
|
&& is_string($envelope['prepared_marker'] ?? null)
|
|
? self::preparedRecoveryBinding($envelope['prepared_marker'])
|
|
: null;
|
|
$currentMarker = self::preparedRecoveryBinding(
|
|
(string) $job->effect_resolution_reason
|
|
);
|
|
if (!is_array($envelope) || array_is_list($envelope)
|
|
|| array_diff(array_keys($envelope), [
|
|
'version', 'kind', 'prepared_marker', 'operation',
|
|
]) !== []
|
|
|| ($envelope['version'] ?? null) !== 1
|
|
|| ($envelope['kind'] ?? null) !== (string) $job->kind
|
|
|| $storedMarker === null || $currentMarker === null
|
|
|| !hash_equals($currentMarker, $storedMarker)
|
|
|| !is_array($operation) || array_is_list($operation)
|
|
|| ($operation['operation_id'] ?? null)
|
|
!== (string) $job->operation_id
|
|
|| !in_array($operation['status'] ?? null,
|
|
['succeeded', 'failed'], true)) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'任务预提交恢复数据不完整',
|
|
409
|
|
);
|
|
}
|
|
return $operation;
|
|
}
|
|
|
|
/**
|
|
* Validate PHP's encrypted settings intent before recovery evidence can
|
|
* affect ledgers. Both live inspection and immutable receipt recovery use
|
|
* this exact contract so neither path can silently finalize an empty or
|
|
* damaged escrow.
|
|
*
|
|
* @return array{instance:CloudInstance,payload:array,body:array,fields:array}
|
|
*/
|
|
public static function settingsRecoveryContext(CloudJob $job): array
|
|
{
|
|
if ((string) $job->kind !== 'instance_settings') {
|
|
throw new CloudError('任务不处于可自动核对状态', 409);
|
|
}
|
|
$context = self::recoveryContext($job);
|
|
$instance = $context['instance'];
|
|
if ($instance === null
|
|
|| (string) $instance->status === 'deleted'
|
|
) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'任务与实例账本不匹配',
|
|
409
|
|
);
|
|
}
|
|
$payload = $context['payload'];
|
|
$body = $payload['body'] ?? null;
|
|
$allowedBody = [
|
|
'hostname', 'root_password', 'ssh_public_key',
|
|
'confirm_restart',
|
|
];
|
|
if (!is_array($body) || array_is_list($body)
|
|
|| array_diff(array_keys($body), $allowedBody) !== []
|
|
|| ($body['confirm_restart'] ?? null) !== true
|
|
|| !in_array(strtoupper((string) ($payload['method'] ?? '')),
|
|
['POST', 'PATCH'], true)
|
|
|| ($payload['request_target'] ?? null) !== '/v1/vms/'
|
|
. rawurlencode((string) $instance->uuid) . '/settings'
|
|
|| (int) ($payload['owner_id'] ?? 0)
|
|
!== (int) $instance->user_id
|
|
|| ((int) $job->user_id !== (int) $instance->user_id
|
|
&& ($payload['can_manage'] ?? null) !== true)) {
|
|
throw new CloudError('实例设置恢复请求记录无效', 409);
|
|
}
|
|
$fields = [];
|
|
foreach ([
|
|
'hostname' => 'hostname',
|
|
'root_password' => 'password',
|
|
'ssh_public_key' => 'ssh_key',
|
|
] as $bodyField => $code) {
|
|
if (array_key_exists($bodyField, $body)) {
|
|
$fields[] = $code;
|
|
}
|
|
}
|
|
if ($fields === []
|
|
|| (array_key_exists('settings_fields', $payload)
|
|
&& $payload['settings_fields'] !== $fields)) {
|
|
throw new CloudError('实例设置恢复字段记录无效', 409);
|
|
}
|
|
if (array_key_exists('hostname', $body)
|
|
&& CloudInput::hostname($body['hostname'])
|
|
!== $body['hostname']) {
|
|
throw new CloudError('实例设置恢复主机名记录无效', 409);
|
|
}
|
|
if (array_key_exists('root_password', $body)
|
|
&& (!is_string($body['root_password'])
|
|
|| CloudInput::password($body['root_password'])
|
|
!== $body['root_password'])) {
|
|
throw new CloudError('实例设置恢复密码记录无效', 409);
|
|
}
|
|
if (array_key_exists('ssh_public_key', $body)) {
|
|
if (!is_string($body['ssh_public_key'])) {
|
|
throw new CloudError('实例设置恢复 SSH 公钥记录无效', 409);
|
|
}
|
|
$parsedKey = CloudSSHKeys::parse($body['ssh_public_key']);
|
|
if (($parsedKey['public_key'] ?? null)
|
|
!== $body['ssh_public_key']) {
|
|
throw new CloudError('实例设置恢复 SSH 公钥记录无效', 409);
|
|
}
|
|
}
|
|
$credential = $payload['credential'] ?? null;
|
|
if ($credential !== null) {
|
|
if (!is_array($credential) || array_is_list($credential)
|
|
|| array_diff(
|
|
array_keys($credential),
|
|
['password', 'login_name', 'auth_mode']
|
|
) !== []
|
|
|| ($credential['login_name'] ?? null) !== 'root'
|
|
|| !in_array(
|
|
$credential['auth_mode'] ?? null,
|
|
[
|
|
'password', 'password_and_key', 'ssh_key',
|
|
'ssh_key_password',
|
|
],
|
|
true
|
|
)) {
|
|
throw new CloudError('实例设置恢复凭据记录无效', 409);
|
|
}
|
|
$savedPassword = $credential['password'] ?? null;
|
|
if ($savedPassword !== null
|
|
&& (!is_string($savedPassword)
|
|
|| CloudInput::password($savedPassword)
|
|
!== $savedPassword)) {
|
|
throw new CloudError('实例设置恢复凭据记录无效', 409);
|
|
}
|
|
$passwordMode = in_array(
|
|
$credential['auth_mode'],
|
|
['password', 'password_and_key', 'ssh_key_password'],
|
|
true
|
|
);
|
|
if (($passwordMode && $savedPassword === null)
|
|
|| (!$passwordMode && $savedPassword !== null)) {
|
|
throw new CloudError('实例设置恢复凭据记录无效', 409);
|
|
}
|
|
}
|
|
$changesCredential = array_key_exists('root_password', $body)
|
|
|| array_key_exists('ssh_public_key', $body);
|
|
if (!$changesCredential && $credential !== null) {
|
|
throw new CloudError('实例设置恢复凭据与请求字段不匹配', 409);
|
|
}
|
|
if (array_key_exists('ssh_public_key', $body)
|
|
&& is_array($credential)
|
|
&& !in_array(
|
|
$credential['auth_mode'],
|
|
['password_and_key', 'ssh_key', 'ssh_key_password'],
|
|
true
|
|
)) {
|
|
throw new CloudError('实例 SSH 公钥恢复凭据模式不匹配', 409);
|
|
}
|
|
if (array_key_exists('root_password', $body)) {
|
|
if (!is_array($credential)
|
|
|| ($credential['password'] ?? null)
|
|
!== $body['root_password']) {
|
|
throw new CloudError('实例密码恢复凭据不匹配', 409);
|
|
}
|
|
}
|
|
return compact('instance', 'payload', 'body', 'fields');
|
|
}
|
|
|
|
/**
|
|
* Apply an already-validated recovery result to PHP-owned ledgers only.
|
|
* The original Agent mutation is never dispatched from this path.
|
|
*/
|
|
public function prepareRecoveredEffect(
|
|
int $jobId,
|
|
array $operation,
|
|
string $preparedMarker
|
|
): CloudJob {
|
|
if (!self::validPreparedRecoveryMarker($preparedMarker)) {
|
|
throw new CloudError('任务效果恢复标记无效', 409);
|
|
}
|
|
$job = CloudJob::find($jobId);
|
|
if ($job === null) {
|
|
throw new CloudError('任务不存在', 404);
|
|
}
|
|
if (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
(string) $job->kind
|
|
)) {
|
|
throw new CloudError('任务类型不支持自动效果恢复', 409);
|
|
}
|
|
$context = (string) $job->kind === 'instance_settings'
|
|
? self::settingsRecoveryContext($job)
|
|
: self::recoveryContext($job);
|
|
$payload = $context['payload'];
|
|
$normalized = self::normalizeAgentOperation(
|
|
self::operationForJob($job, $operation)
|
|
);
|
|
try {
|
|
$this->finish($job, $normalized, $payload, $preparedMarker);
|
|
} catch (CloudRecoveryPayloadUnavailable
|
|
| CloudRecoveryEvidenceConflict
|
|
| CloudRecoveryReceiptInvalid
|
|
| CloudRecoveryOperationIdentityConflict
|
|
| CloudRecoveryAggregateStateConflict
|
|
| CloudRecoveryInstanceStateConflict $error) {
|
|
throw $error;
|
|
} catch (CloudError $error) {
|
|
if ($job->instance_id !== null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'实例或关联资源账本与原任务证据不一致',
|
|
409
|
|
);
|
|
}
|
|
throw $error;
|
|
}
|
|
return CloudJob::find($jobId) ?? $job;
|
|
}
|
|
|
|
/**
|
|
* Materialize one paid create from its exact immutable success receipt.
|
|
*
|
|
* This is deliberately not part of the generic recovery policy: it never
|
|
* inspects or replays an Agent mutation. The original failed task remains
|
|
* terminal for audit, while its create ledgers and receipt ACK commit as
|
|
* one local database transaction.
|
|
*/
|
|
public function settleRecoveredCreateCompletion(
|
|
int $jobId,
|
|
\App\Models\CloudAgentCompletionEvent $event
|
|
): CloudJob {
|
|
$job = CloudJob::find($jobId);
|
|
if ($job === null) {
|
|
throw new CloudError('任务不存在', 404);
|
|
}
|
|
if ((string) $job->kind !== 'create') {
|
|
throw new CloudError('任务类型不支持开通回执恢复', 409);
|
|
}
|
|
try {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $job->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
} catch (\Throwable $error) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'任务加密恢复数据暂时无法读取',
|
|
409
|
|
);
|
|
}
|
|
$operation = self::normalizeAgentOperation(
|
|
self::operationForJob(
|
|
$job,
|
|
CloudOperationCompletionService::operation($event)
|
|
)
|
|
);
|
|
$this->finish(
|
|
$job,
|
|
$operation,
|
|
$payload,
|
|
null,
|
|
(string) $event->event_id
|
|
);
|
|
return CloudJob::find($jobId) ?? $job;
|
|
}
|
|
|
|
/**
|
|
* Retry only the PHP ledger finalizer for an operation proven not sent.
|
|
* The terminal task receipt remains immutable and no Agent request occurs.
|
|
*/
|
|
public function retryUndispatchedFinalization(
|
|
int $jobId,
|
|
string $leaseToken
|
|
): CloudJob {
|
|
$job = CloudJob::find($jobId);
|
|
if ($job === null) {
|
|
throw new CloudError('任务不存在', 404);
|
|
}
|
|
if ((string) $job->kind === 'create') {
|
|
CloudCreateFailureFinalizer::retryUndispatchedFailure(
|
|
$jobId,
|
|
$leaseToken
|
|
);
|
|
return CloudJob::find($jobId) ?? $job;
|
|
}
|
|
try {
|
|
$payload = CloudSecrets::decrypt(
|
|
(string) $job->sealed_payload,
|
|
'job-payload'
|
|
);
|
|
} catch (\Throwable $error) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'任务恢复数据无法解密',
|
|
409
|
|
);
|
|
}
|
|
if (!is_array($payload) || array_is_list($payload)) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'任务恢复数据结构不完整',
|
|
409
|
|
);
|
|
}
|
|
$operation = CloudOperationRecoveryPolicy::isCreateLike(
|
|
(string) $job->kind
|
|
) ? self::createNotDispatchedFailure() : ['status' => 'failed'];
|
|
$operation['operation_id'] = (string) $job->operation_id;
|
|
$operation['error_code'] = 'dispatch_not_sent';
|
|
$operation['effect_may_have_applied'] = false;
|
|
$operation['safe_to_resubmit'] = true;
|
|
$this->finish(
|
|
$job,
|
|
$operation,
|
|
$payload,
|
|
null,
|
|
null,
|
|
$leaseToken
|
|
);
|
|
return CloudJob::find($jobId) ?? $job;
|
|
}
|
|
|
|
private function finish(
|
|
CloudJob $claimed,
|
|
array $operation,
|
|
array $payload,
|
|
?string $preparedRecoveryMarker = null,
|
|
?string $recoveredCreateEventId = null,
|
|
?string $localFinalizerToken = null
|
|
): void
|
|
{
|
|
if ((string) $claimed->execution_mode === 'agent') {
|
|
CloudDispatchLedger::assertValid($claimed);
|
|
}
|
|
$operation['operation_id'] = $operation['operation_id']
|
|
?? (string) $claimed->operation_id;
|
|
$status = self::disposition(
|
|
$operation,
|
|
CloudOperationRecoveryPolicy::isCreateLike(
|
|
(string) $claimed->kind
|
|
),
|
|
(string) $claimed->kind);
|
|
if (!in_array($status, ['succeeded', 'failed'], true)) {
|
|
throw new CloudError('任务尚未取得终态回执', 409);
|
|
}
|
|
$operationIdentity = null;
|
|
// The Agent triple exists only after the immutable mutation crossed
|
|
// the dispatch boundary. A locally rejected, never-dispatched task is
|
|
// already proven not-applied and must still reach its kind finalizer;
|
|
// requiring an Agent identity there would turn damaged local input
|
|
// into an unnecessary unknown-effect fence.
|
|
if (CloudDispatchLedger::isDispatched($claimed)
|
|
&& CloudOperationRecoveryIdentity::supports(
|
|
(string) $claimed->kind
|
|
)) {
|
|
$operationIdentity = self::operationIdentityForFinalization(
|
|
$claimed,
|
|
$payload,
|
|
$status === 'succeeded',
|
|
$preparedRecoveryMarker !== null
|
|
);
|
|
$payload[CloudOperationRecoveryIdentity::payloadKey()] =
|
|
CloudOperationRecoveryIdentity::persisted(
|
|
$operationIdentity
|
|
);
|
|
}
|
|
if ($status === 'succeeded'
|
|
&& in_array((string) $claimed->kind,
|
|
['delete', 'expiry_delete', 'delete_force'], true)
|
|
&& !self::deleteSuccessConfirmed($operation,
|
|
(string) $claimed->kind)) {
|
|
throw new CloudError('Agent 删除回执缺少资源已清理证据', 502);
|
|
}
|
|
if ((string) $claimed->kind === 'reinstall'
|
|
&& self::reinstallNeedsReview($operation)) {
|
|
throw new CloudError('Agent 重装回执未确认清理完成', 502);
|
|
}
|
|
if ($status === 'failed') {
|
|
$this->observedErrors[] = 'task_failed';
|
|
}
|
|
$forceRequired = $status === 'failed'
|
|
&& in_array((string) $claimed->kind,
|
|
['delete', 'expiry_delete', 'service_suspend'], true)
|
|
&& self::operationErrorCode($operation) === 'force_required';
|
|
$effectMayHaveApplied = $status === 'failed'
|
|
&& ($operation['effect_may_have_applied'] ?? false) === true;
|
|
$safeToResubmit = $status === 'failed'
|
|
&& ($operation['safe_to_resubmit'] ?? false) === true;
|
|
if ($forceRequired) {
|
|
$effectMayHaveApplied = false;
|
|
$safeToResubmit = false;
|
|
}
|
|
if ($effectMayHaveApplied) {
|
|
$safeToResubmit = false;
|
|
}
|
|
if ($status === 'failed'
|
|
&& CloudOperationRecoveryPolicy::isCreateLike(
|
|
(string) $claimed->kind
|
|
)) {
|
|
if (self::createFailureConfirmedAbsent($operation)) {
|
|
// Exact absence evidence is the authority for financial and
|
|
// placement rollback. safe_to_resubmit is informational only;
|
|
// this generation is never dispatched again.
|
|
$effectMayHaveApplied = false;
|
|
} else {
|
|
// Boolean transport hints alone cannot release a paid order.
|
|
// Keep every resource fenced until a fresh read-only host
|
|
// inspection proves both the guest and artifacts are absent.
|
|
$effectMayHaveApplied = true;
|
|
$safeToResubmit = false;
|
|
}
|
|
}
|
|
// Downstream error classification must use the validated PHP safety
|
|
// decision, never the Agent's original booleans. In particular an
|
|
// ambiguous create must not be labelled as refunded.
|
|
$operation['effect_may_have_applied'] = $effectMayHaveApplied;
|
|
$operation['safe_to_resubmit'] = $safeToResubmit;
|
|
$resourceStatus = $status === 'failed' && $effectMayHaveApplied
|
|
? 'uncertain' : $status;
|
|
$result = $operation['result'] ?? null;
|
|
CloudPortableRestoreFinalizer::recordEvidence(
|
|
$claimed,
|
|
$resourceStatus,
|
|
$result,
|
|
$payload
|
|
);
|
|
if ((string) $claimed->kind === 'network_change'
|
|
&& $claimed->public_network_transition_id !== null) {
|
|
CloudPublicNetworkTransitionWorkflow::recordEvidence(
|
|
$claimed,
|
|
$resourceStatus,
|
|
$result,
|
|
$payload,
|
|
$operation,
|
|
$preparedRecoveryMarker !== null
|
|
);
|
|
}
|
|
$aggregateParentId = (int) ($claimed->parent_job_id ?? 0);
|
|
(new CloudJob())->getConnection()->transaction(function () use ($claimed, $payload, $status, $resourceStatus, $effectMayHaveApplied, $safeToResubmit, $forceRequired, $result, $operation, $preparedRecoveryMarker, $operationIdentity, $recoveredCreateEventId, $localFinalizerToken, $aggregateParentId) {
|
|
// All financial transactions acquire the wallet first, avoiding wallet/order deadlocks.
|
|
$user = User::where('id', $claimed->user_id)->lockForUpdate()->first();
|
|
$order = $claimed->order_id === null ? null : CloudOrder::where('id', $claimed->order_id)->lockForUpdate()->first();
|
|
// Aggregate recovery and the legacy sweep both use the same
|
|
// parent -> children lock order. Never lock a leaf first and then
|
|
// wait for its parent: concurrent sibling completions otherwise
|
|
// form a deadlock cycle with the parent-led sweep.
|
|
$aggregateParent = null;
|
|
if ($aggregateParentId > 0) {
|
|
$aggregateParent = CloudJob::where('id', $aggregateParentId)
|
|
->lockForUpdate()->first();
|
|
if ($aggregateParent === null) {
|
|
throw new CloudRecoveryAggregateStateConflict(
|
|
'任务的汇总父任务不存在',
|
|
409
|
|
);
|
|
}
|
|
CloudTemplateCatalogService::lockAggregateCatalogLocked(
|
|
$aggregateParent
|
|
);
|
|
CloudUserIsoService::lockAggregateCatalogLocked(
|
|
$aggregateParent
|
|
);
|
|
}
|
|
$job = CloudJob::where('id', $claimed->id)->lockForUpdate()->first();
|
|
if ($job === null) {
|
|
throw new CloudError('任务不存在', 404);
|
|
}
|
|
if ((string) $job->execution_mode === 'agent') {
|
|
CloudDispatchLedger::assertValid($job);
|
|
if (CloudDispatchLedger::attempts($job)
|
|
!== CloudDispatchLedger::attempts($claimed)) {
|
|
throw new CloudRecoveryDispatchLedgerConflict(
|
|
'任务单次派发代次在收敛期间发生变化',
|
|
409
|
|
);
|
|
}
|
|
}
|
|
if ((int) ($job->parent_job_id ?? 0) !== $aggregateParentId) {
|
|
throw new CloudRecoveryAggregateStateConflict(
|
|
'任务的汇总父级关系在收敛期间发生变化',
|
|
409
|
|
);
|
|
}
|
|
$localFinalizerRecovery = $localFinalizerToken !== null;
|
|
if ($localFinalizerRecovery
|
|
&& (!CloudOperationRecoveryPolicy::supportsLedgerFinalizer(
|
|
(string) $job->kind
|
|
)
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|| (string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| !CloudDispatchLedger::
|
|
isLocallyFinalizableNotDispatched($job)
|
|
|| !(bool) $job->finalization_incident
|
|
|| !in_array((string) $job->effect_resolution,
|
|
['unknown', 'not_applied'], true)
|
|
|| !is_string($job->lease_token)
|
|
|| !hash_equals(
|
|
(string) $job->lease_token,
|
|
$localFinalizerToken
|
|
))) {
|
|
throw new CloudError(
|
|
'任务不处于可恢复的本地收尾状态',
|
|
409
|
|
);
|
|
}
|
|
if ($localFinalizerRecovery) {
|
|
if (CloudDispatchLedger::hasRepairIncident($job)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'任务仍处于派发账本修复状态',
|
|
409
|
|
);
|
|
}
|
|
CloudOperationCompletionService::
|
|
assertLocalNotDispatchedRecoveryState($job);
|
|
}
|
|
$recoveredCreateEvent = null;
|
|
if ($recoveredCreateEventId !== null) {
|
|
$recoveredCreateEvent = \App\Models\CloudAgentCompletionEvent::where(
|
|
'event_id',
|
|
$recoveredCreateEventId
|
|
)->lockForUpdate()->first();
|
|
if ($preparedRecoveryMarker !== null
|
|
|| (string) $job->kind !== 'create'
|
|
|| (string) $job->execution_mode !== 'agent'
|
|
|| (string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| (int) $job->dispatch_attempts !== 1
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|| (bool) $job->safe_to_resubmit
|
|
|| (string) $job->effect_resolution !== 'unknown'
|
|
|| $job->order_id === null
|
|
|| $order === null
|
|
|| $operationIdentity === null
|
|
|| $recoveredCreateEvent === null
|
|
|| (string) $recoveredCreateEvent->disposition !== 'received'
|
|
|| (int) $recoveredCreateEvent->host_id
|
|
!== (int) $job->host_id
|
|
|| !hash_equals(
|
|
(string) $job->operation_id,
|
|
(string) $recoveredCreateEvent->operation_id
|
|
)
|
|
|| (string) $recoveredCreateEvent->outcome !== 'succeeded'
|
|
|| (bool) $recoveredCreateEvent->effect_may_have_applied
|
|
|| (bool) $recoveredCreateEvent->safe_to_resubmit
|
|
|| !CloudOperationCompletionService::isExactStoredEvent(
|
|
$recoveredCreateEvent
|
|
)) {
|
|
throw new CloudError(
|
|
'开通完成回执不处于可安全收敛状态',
|
|
409
|
|
);
|
|
}
|
|
$lockedIdentity = CloudOperationRecoveryIdentity::
|
|
expectedFromPayload($job, $payload, true);
|
|
CloudOperationRecoveryIdentity::assertMatches(
|
|
$lockedIdentity,
|
|
CloudOperationCompletionService::identity(
|
|
$recoveredCreateEvent
|
|
)
|
|
);
|
|
} elseif ($preparedRecoveryMarker !== null) {
|
|
CloudOperationRecoveryService::
|
|
assertRecoveryIncidentAllowsPrepare($job);
|
|
if ((string) $job->phase !== 'terminal'
|
|
|| (string) $job->outcome !== 'failed'
|
|
|| (string) $job->status !== 'failed'
|
|
|| !(bool) $job->effect_may_have_applied
|
|
|| (bool) $job->safe_to_resubmit
|
|
|| (string) $job->effect_resolution !== 'unknown') {
|
|
throw new CloudError('任务已不再等待效果恢复', 409);
|
|
}
|
|
if (!self::preparedRecoveryTransitionAllowed(
|
|
(string) $job->effect_resolution_reason,
|
|
$preparedRecoveryMarker,
|
|
$operation
|
|
)) {
|
|
if (self::lateReceiptRecoveryMarker(
|
|
$preparedRecoveryMarker
|
|
)) {
|
|
throw new CloudRecoveryEvidenceConflict(
|
|
'任务已有冲突的效果恢复证据',
|
|
409
|
|
);
|
|
}
|
|
throw new CloudError('任务已有不同的效果恢复证据', 409);
|
|
}
|
|
if (hash_equals(
|
|
$preparedRecoveryMarker,
|
|
(string) $job->effect_resolution_reason
|
|
)) {
|
|
if (self::lateReceiptRecoveryMarker(
|
|
$preparedRecoveryMarker
|
|
) && in_array(
|
|
(string) ($job->finalization_error_code ?? ''),
|
|
[
|
|
'recovery_limit_reached',
|
|
'recovery_state_invalid',
|
|
],
|
|
true
|
|
)) {
|
|
$job->finalization_incident = 0;
|
|
$job->finalization_error_code = null;
|
|
$job->updated_at = time();
|
|
$job->save();
|
|
}
|
|
return;
|
|
}
|
|
} elseif (!$localFinalizerRecovery
|
|
&& ($job->lease_token !== $claimed->lease_token
|
|
|| (string) $job->phase === 'terminal')) {
|
|
return;
|
|
}
|
|
self::assertRescueResultMatchesRequest(
|
|
$job,
|
|
$status,
|
|
$result,
|
|
$payload
|
|
);
|
|
$safe = is_array($result)
|
|
? CloudOperationErrorCatalog::workerResult($result) : null;
|
|
$causeCode = CloudOperationErrorCatalog::agentCauseCode(
|
|
$operation['cause_code'] ?? null
|
|
);
|
|
if ($causeCode !== null) {
|
|
$durableResult = is_array($result) && !array_is_list($result)
|
|
? $result : [];
|
|
// Keep the stable cause beside the sanitized operation
|
|
// evidence. It is never copied into the public error_code or
|
|
// ordinary user DTOs.
|
|
$durableResult['cause_code'] = $causeCode;
|
|
$safe = CloudOperationErrorCatalog::workerResult($durableResult);
|
|
}
|
|
$catalogStatus = $resourceStatus;
|
|
// Catalog/configuration mutations use the public terminal status,
|
|
// never the legacy resource-level "uncertain" label. Their
|
|
// replica rows and aggregate parent are settled in this same
|
|
// transaction as the child job.
|
|
CloudTemplateCatalogService::finishJobLocked(
|
|
$job, $status, $result, $payload, $operation,
|
|
$preparedRecoveryMarker !== null || $localFinalizerRecovery
|
|
);
|
|
CloudUserIsoService::finishJobLocked(
|
|
$job, $status, $result, $payload, $operation,
|
|
$preparedRecoveryMarker !== null || $localFinalizerRecovery
|
|
);
|
|
CloudRescueTemplateReplicaCoordinator::finishJobLocked(
|
|
$job, $status, $result, $payload, $operation,
|
|
$preparedRecoveryMarker !== null || $localFinalizerRecovery
|
|
);
|
|
CloudRescueTemplateRetirementService::finishJobLocked(
|
|
$job, $status, $result, $payload, $operation,
|
|
$preparedRecoveryMarker !== null || $localFinalizerRecovery
|
|
);
|
|
CloudHostStorageConfigurationService::finishJobLocked(
|
|
$job, $status, $result, $operation
|
|
);
|
|
CloudSnapshotService::finishJobLocked(
|
|
$job, $catalogStatus, $result, $payload
|
|
);
|
|
CloudBackupService::finishJobLocked(
|
|
$job,
|
|
$catalogStatus,
|
|
$result,
|
|
$payload,
|
|
$preparedRecoveryMarker !== null
|
|
);
|
|
CloudSnapshotService::finishInstanceDeleteLocked(
|
|
$job, $catalogStatus
|
|
);
|
|
CloudFirewallService::finishJobLocked(
|
|
$job, $catalogStatus, $result, $payload, $operation
|
|
);
|
|
CloudVpcPortService::finishJobLocked(
|
|
$job, $catalogStatus, $result, $payload, $operation
|
|
);
|
|
CloudInstanceMigrationService::finishJobLocked(
|
|
$job, $catalogStatus, $result, $payload
|
|
);
|
|
CloudFirewallService::finishInstanceDeleteLocked(
|
|
$job, $catalogStatus
|
|
);
|
|
if ($job->kind === 'instance_upgrade'
|
|
&& in_array($catalogStatus,
|
|
['succeeded', 'failed', 'uncertain'], true)) {
|
|
CloudPlanUpgradeService::finishJobLocked(
|
|
$job,
|
|
$catalogStatus,
|
|
$result,
|
|
$payload,
|
|
$user
|
|
);
|
|
} elseif ($job->kind === 'network_change'
|
|
&& $job->public_network_transition_id !== null
|
|
&& in_array($catalogStatus,
|
|
['succeeded', 'failed', 'uncertain'], true)) {
|
|
CloudPublicNetworkTransitionWorkflow::finishJobLocked(
|
|
$job,
|
|
$catalogStatus,
|
|
$payload,
|
|
$user,
|
|
$preparedRecoveryMarker !== null,
|
|
$aggregateParent
|
|
);
|
|
} elseif ($status === 'succeeded' && $job->kind === 'create') {
|
|
$vm = is_array($result) ? ($result['vm'] ?? $result) : [];
|
|
$uuid = $vm['uuid'] ?? '';
|
|
if ($order === null || !is_string($uuid)
|
|
|| $operationIdentity === null
|
|
|| !hash_equals(
|
|
(string) $operationIdentity['resource_id'],
|
|
strtolower($uuid)
|
|
)
|
|
|| preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD', $uuid) !== 1) {
|
|
throw new CloudError('Agent 成功结果缺少虚拟机标识', 502);
|
|
}
|
|
$orderSpec = json_decode((string) $order->spec_json, true);
|
|
try {
|
|
$frozenPublicNetwork = CloudPublicNetworkSnapshot::fromArray(
|
|
is_array($orderSpec)
|
|
? ($orderSpec['public_network'] ?? null) : null
|
|
);
|
|
$requestedPublicNetwork =
|
|
CloudPublicNetworkSnapshot::fromArray(
|
|
is_array($payload['body'] ?? null)
|
|
? ($payload['body']['public_network'] ?? null)
|
|
: null
|
|
);
|
|
$confirmedPublicNetwork =
|
|
CloudPublicNetworkSnapshot::fromArray(
|
|
$vm['public_network'] ?? null
|
|
);
|
|
} catch (\InvalidArgumentException) {
|
|
throw new CloudError(
|
|
'Agent 成功结果缺少有效的双栈公网快照,必须人工核查',
|
|
502
|
|
);
|
|
}
|
|
if (!hash_equals(
|
|
$frozenPublicNetwork->fingerprint(),
|
|
$requestedPublicNetwork->fingerprint()
|
|
) || !hash_equals(
|
|
$requestedPublicNetwork->fingerprint(),
|
|
$confirmedPublicNetwork->fingerprint()
|
|
)) {
|
|
// A completion result may describe a VM whose network was
|
|
// changed out of band after dispatch. Never commit either
|
|
// original address in that case; retain the whole bundle.
|
|
throw new CloudError(
|
|
'Agent 成功结果与预留双栈公网不一致,必须人工核查',
|
|
502
|
|
);
|
|
}
|
|
$provisionInvoice = CloudInvoice::where(
|
|
'order_id',
|
|
(int) $order->id
|
|
)->lockForUpdate()->first();
|
|
if (!in_array((string) $order->status,
|
|
['queued', 'failed', 'uncertain'], true)
|
|
|| $order->refunded_at !== null
|
|
|| !self::createStockLedgerIsConsistent($order)
|
|
|| $order->instance_id !== null
|
|
|| $provisionInvoice === null
|
|
|| (string) $provisionInvoice->purpose !== 'provision'
|
|
|| (string) $provisionInvoice->status !== 'paid'
|
|
|| $provisionInvoice->paid_at === null
|
|
|| (int) $provisionInvoice->order_id !== (int) $order->id
|
|
|| (int) $provisionInvoice->user_id !== (int) $order->user_id
|
|
|| (int) $provisionInvoice->plan_id !== (int) $order->plan_id
|
|
|| (int) $provisionInvoice->offer_id !== (int) $order->offer_id
|
|
|| strtoupper((string) $provisionInvoice->currency)
|
|
!== strtoupper((string) $order->currency)
|
|
|| CloudMoney::cents((string) $provisionInvoice->amount)
|
|
!== CloudMoney::cents((string) $order->amount)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'开通成功回执与资金、库存或资源预留账本不一致',
|
|
409
|
|
);
|
|
}
|
|
$instance = CloudInstance::where('order_id', $order->id)->first();
|
|
$newInstance = $instance === null;
|
|
$instance ??= new CloudInstance();
|
|
$instance->uuid = $uuid;
|
|
$instance->host_id = $order->host_id;
|
|
$instance->user_id = $order->user_id;
|
|
$instance->order_id = $order->id;
|
|
$instance->offer_id = $order->offer_id;
|
|
$instance->resource_pool_id = $order->resource_pool_id;
|
|
$instance->reservation_id = $order->reservation_id;
|
|
$spec = $orderSpec;
|
|
$templateMetadata = self::trustedTemplateMetadata($payload);
|
|
$spec = array_replace($spec, $templateMetadata);
|
|
$now = time();
|
|
$days = filter_var($spec['validity_days'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
$traffic = filter_var($spec['traffic_quota_bytes'] ?? null,
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 0]]);
|
|
if ($days === false || $traffic === false) {
|
|
throw new CloudError('订单缺少不可变的计费周期快照', 409);
|
|
}
|
|
$trafficPoolBinding =
|
|
CloudTrafficPoolPeriodBinding::orderSnapshot(
|
|
$spec,
|
|
$traffic
|
|
);
|
|
$instancePrice = CloudMoney::cents(
|
|
$spec['instance_price'] ?? null
|
|
);
|
|
$instance->plan_id = $order->plan_id;
|
|
// A duplicate completion result must not overwrite a display
|
|
// name the owner changed after the first success committed.
|
|
if ($newInstance) {
|
|
$instance->name = $payload['body']['name'];
|
|
}
|
|
$instance->status = 'active';
|
|
$instance->service_status = 'active';
|
|
$instance->network_status = 'active';
|
|
$instance->expires_at = $now + $days * 86400;
|
|
// Optional services are renewed from their own subscription
|
|
// ledgers. Keep this field as the base compute price only.
|
|
$instance->renewal_price = CloudMoney::decimal(
|
|
$instancePrice
|
|
);
|
|
$instance->renewal_days = $days;
|
|
$instance->renewal_traffic_bytes = $traffic;
|
|
$instance->traffic_quota_bytes = $traffic;
|
|
$instance->traffic_download_bytes = 0;
|
|
$instance->traffic_upload_bytes = 0;
|
|
$instance->lifetime_download_bytes = 0;
|
|
$instance->lifetime_upload_bytes = 0;
|
|
$instance->traffic_synced_at = 0;
|
|
// Persist only the Agent's slow-changing public metadata.
|
|
// SSR pages can then render IP, operating system, login name
|
|
// and actual capacity without issuing a host read
|
|
// on every request. Billing/order fields remain authoritative
|
|
// because the allowlist cannot overwrite them.
|
|
$instanceSpec = CloudInstanceMetadata::merge($spec, $vm);
|
|
if ($templateMetadata !== null) {
|
|
// Agent observations cannot replace PHP's catalog
|
|
// snapshot, even if an older Agent happens to return
|
|
// similarly named fields.
|
|
$instanceSpec = array_replace(
|
|
$instanceSpec,
|
|
$templateMetadata
|
|
);
|
|
unset($instanceSpec['install_iso']);
|
|
}
|
|
// A VM is panel-managed only after the Agent's exact create
|
|
// operation has returned the validated success above.
|
|
$instanceSpec['public_network'] =
|
|
$confirmedPublicNetwork->toArray();
|
|
$instanceSpec['managed'] = true;
|
|
$instance->spec_json = json_encode(
|
|
$instanceSpec,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
);
|
|
if (isset($payload['backup_metadata'])) {
|
|
// The restored system disk is authoritative. The user may
|
|
// have changed the guest password after provisioning, so
|
|
// copying the panel's historical envelope would expose a
|
|
// password that is no longer valid.
|
|
$instance->sealed_credential = null;
|
|
} else {
|
|
$instance->sealed_credential = isset($payload['credential'])
|
|
? CloudSecrets::encrypt(
|
|
$payload['credential'], 'instance-credential'
|
|
) : null;
|
|
}
|
|
// The paid service begins when provisioning actually succeeds,
|
|
// not while an opening invoice is waiting for payment. A
|
|
// duplicate success must retain the original value.
|
|
if (!$instance->exists) {
|
|
$instance->created_at = $now;
|
|
}
|
|
$instance->updated_at = time();
|
|
$instance->save();
|
|
CloudPlacementService::commit(
|
|
$order,
|
|
(int) $instance->id,
|
|
$confirmedPublicNetwork->toArray()
|
|
);
|
|
// Seed the independent runtime observation immediately after
|
|
// provisioning. If the Agent returned a trusted power
|
|
// state, promote it now; otherwise the background host batch
|
|
// will provide the first observation without showing "未知".
|
|
self::recordRuntime($instance, $vm, 'create', []);
|
|
$period = CloudServicePeriod::where('order_id', $order->id)->first()
|
|
?? new CloudServicePeriod();
|
|
$period->instance_id = $instance->id;
|
|
$period->user_id = $order->user_id;
|
|
$period->plan_id = $order->plan_id;
|
|
$period->order_id = $order->id;
|
|
$period->starts_at = $now;
|
|
$period->ends_at = $instance->expires_at;
|
|
$period->traffic_quota_bytes = $traffic;
|
|
CloudTrafficPoolPeriodBinding::bindPeriod(
|
|
$period,
|
|
$trafficPoolBinding['traffic_pool_group_id'],
|
|
$traffic
|
|
);
|
|
$period->download_base_bytes = 0;
|
|
$period->upload_base_bytes = 0;
|
|
$period->download_used_bytes = 0;
|
|
$period->upload_used_bytes = 0;
|
|
$period->status = 'active';
|
|
$period->activated_at = $now;
|
|
$period->created_at = $period->created_at ?: $now;
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$instance->current_period_id = $period->id;
|
|
$instance->save();
|
|
$backupSubscription =
|
|
CloudAutomaticBackupService::activatePurchased($instance);
|
|
CloudAutomaticBackupBillingService::grantProvisionEntitlementLocked(
|
|
$instance,
|
|
$period,
|
|
$provisionInvoice,
|
|
$backupSubscription,
|
|
$now
|
|
);
|
|
$job->instance_id = $order->instance_id = $instance->id;
|
|
$provisionInvoice->instance_id = $instance->id;
|
|
$provisionInvoice->updated_at = time();
|
|
$provisionInvoice->save();
|
|
CloudInvoiceService::queueProvisionSucceeded(
|
|
$provisionInvoice, $now
|
|
);
|
|
$safe = $safe ?? [];
|
|
$safe['id'] = (string) $instance->id;
|
|
$safe['uuid'] = $uuid;
|
|
} elseif ($job->kind === 'reinstall'
|
|
&& $job->instance_id !== null
|
|
&& $status === 'succeeded') {
|
|
$instance = CloudInstance::where('id', $job->instance_id)
|
|
->lockForUpdate()->first();
|
|
if ($instance === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'重装任务关联实例已缺失',
|
|
409
|
|
);
|
|
}
|
|
$instance->sealed_credential = CloudSecrets::encrypt(
|
|
$payload['credential'], 'instance-credential'
|
|
);
|
|
// Once the Agent proves the replacement guest committed,
|
|
// its credential and template snapshot supersede the old
|
|
// guest even if deletion of the old volume still requires
|
|
// operator review. The task still ends as failed.
|
|
$decoded = json_decode((string) $instance->spec_json, true);
|
|
if (!is_array($decoded) || array_is_list($decoded)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'重装任务关联实例配置账本损坏',
|
|
409
|
|
);
|
|
}
|
|
$templateMetadata = self::trustedTemplateMetadata($payload);
|
|
$templateId = is_array($payload['body'] ?? null)
|
|
? ($payload['body']['template_id'] ?? null) : null;
|
|
if (is_string($templateId)
|
|
&& preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D',
|
|
$templateId) === 1) {
|
|
$decoded['template_id'] = $templateId;
|
|
}
|
|
$decoded = array_replace($decoded, $templateMetadata);
|
|
unset($decoded['install_iso']);
|
|
$decoded['managed'] = true;
|
|
// Force one fresh capacity/IP observation while the trusted
|
|
// PHP presentation snapshot remains usable.
|
|
unset($decoded['agent_metadata_version']);
|
|
$instance->spec_json = json_encode($decoded,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE);
|
|
self::recordRuntime($instance,
|
|
is_array($result) ? $result : [],
|
|
(string) $job->kind, $payload);
|
|
$instance->updated_at = time();
|
|
$instance->save();
|
|
} elseif ($status === 'failed' && !$effectMayHaveApplied
|
|
&& $job->kind === 'create' && $order !== null) {
|
|
if ($user === null) {
|
|
throw new CloudError('退款账户不存在', 409);
|
|
}
|
|
CloudCreateFailureFinalizer::refundAndReleaseLocked(
|
|
$job,
|
|
$user,
|
|
$order,
|
|
'create_failed'
|
|
);
|
|
} elseif ($forceRequired && $job->instance_id !== null) {
|
|
$instance = CloudInstance::where('id', $job->instance_id)
|
|
->lockForUpdate()->first();
|
|
if ($instance === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'删除任务关联实例已缺失', 409
|
|
);
|
|
}
|
|
if ((string) $job->kind === 'delete'
|
|
&& (string) $instance->status === 'active') {
|
|
// A tenant graceful-delete request deliberately leaves the
|
|
// live instance unchanged when the Agent requires a new,
|
|
// separately authorized force-delete generation.
|
|
} elseif ((string) $instance->status === 'deleting') {
|
|
$instance->service_status = 'purge_force_required';
|
|
$instance->updated_at = time();
|
|
$instance->save();
|
|
} elseif ((string) $instance->service_status
|
|
!== 'purge_force_required') {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'删除任务的实例状态与强制阶段矛盾',
|
|
409
|
|
);
|
|
}
|
|
} elseif ($status === 'failed' && $job->kind === 'expiry_delete'
|
|
&& $job->instance_id !== null) {
|
|
$instance = CloudInstance::where('id', $job->instance_id)
|
|
->lockForUpdate()->first();
|
|
if ($instance !== null && (string) $instance->status !== 'deleted') {
|
|
$instance->status = 'delete_failed';
|
|
$instance->service_status = 'purge_failed';
|
|
$instance->updated_at = time();
|
|
$instance->save();
|
|
}
|
|
} elseif ($status === 'failed' && $job->instance_id !== null
|
|
&& in_array($job->kind,
|
|
['service_suspend', 'network_suspend', 'network_resume'], true)) {
|
|
// Latch a visible policy failure. This generation is never
|
|
// queued again automatically.
|
|
$instance = CloudInstance::where('id', $job->instance_id)->lockForUpdate()->first();
|
|
if ($instance === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'系统策略任务关联实例已缺失', 409
|
|
);
|
|
}
|
|
$networkPolicyMarker = in_array((string) $job->kind,
|
|
['network_suspend', 'network_resume'], true)
|
|
? CloudInstanceNetworkPolicy::payloadMarker(
|
|
$payload,
|
|
(int) $job->instance_id,
|
|
(string) $job->operation_id
|
|
) : null;
|
|
if ($networkPolicyMarker !== null) {
|
|
CloudInstanceNetworkPolicy::finishJobLocked(
|
|
$instance,
|
|
(string) $job->kind,
|
|
(string) $job->operation_id,
|
|
'failed',
|
|
false,
|
|
$effectMayHaveApplied,
|
|
time()
|
|
);
|
|
} else {
|
|
$expected = [
|
|
'service_suspend' =>
|
|
['suspending_expired', 'purge_stopping'],
|
|
'network_suspend' =>
|
|
['blocking', 'blocking_expired'],
|
|
'network_resume' =>
|
|
['restore_pending', 'restoring'],
|
|
][$job->kind];
|
|
$already = match ((string) $job->kind) {
|
|
'service_suspend' => in_array(
|
|
(string) $instance->service_status,
|
|
['suspend_failed', 'purge_stop_failed'], true
|
|
),
|
|
'network_suspend' => (string) $instance->network_status
|
|
=== 'block_failed',
|
|
'network_resume' => (string) $instance->network_status
|
|
=== 'restore_failed',
|
|
default => false,
|
|
};
|
|
$transitioned = false;
|
|
if ($job->kind === 'service_suspend'
|
|
&& in_array((string) $instance->service_status, $expected, true)) {
|
|
if ((string) $instance->status === 'deleting'
|
|
&& (string) $instance->service_status === 'purge_stopping') {
|
|
// Preserve the irreversible deletion latch, but do
|
|
// not enqueue this failed generation again.
|
|
$instance->service_status = 'purge_stop_failed';
|
|
} else {
|
|
$instance->service_status = 'suspend_failed';
|
|
}
|
|
$transitioned = true;
|
|
} elseif ($job->kind === 'network_suspend'
|
|
&& in_array((string) $instance->network_status, $expected, true)) {
|
|
$instance->network_status = 'block_failed';
|
|
$transitioned = true;
|
|
} elseif ($job->kind === 'network_resume'
|
|
&& in_array((string) $instance->network_status, $expected, true)) {
|
|
$instance->network_status = 'restore_failed';
|
|
$transitioned = true;
|
|
}
|
|
if (!$transitioned && !$already) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'系统策略失败结论与实例当前状态矛盾',
|
|
409
|
|
);
|
|
}
|
|
}
|
|
$instance->updated_at = time();
|
|
$instance->save();
|
|
} elseif ($status === 'succeeded' && $job->instance_id !== null) {
|
|
$instance = CloudInstance::where('id', $job->instance_id)->lockForUpdate()->first();
|
|
if ($instance === null) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'任务成功结论的关联实例已缺失', 409
|
|
);
|
|
}
|
|
$networkPolicyMarker = in_array((string) $job->kind,
|
|
['network_suspend', 'network_resume'], true)
|
|
? CloudInstanceNetworkPolicy::payloadMarker(
|
|
$payload,
|
|
(int) $job->instance_id,
|
|
(string) $job->operation_id
|
|
) : null;
|
|
if ($networkPolicyMarker !== null) {
|
|
CloudInstanceNetworkPolicy::finishJobLocked(
|
|
$instance,
|
|
(string) $job->kind,
|
|
(string) $job->operation_id,
|
|
'succeeded',
|
|
is_array($result)
|
|
&& ($result['skipped'] ?? false) === true,
|
|
false,
|
|
time()
|
|
);
|
|
} elseif (in_array((string) $job->kind,
|
|
['delete', 'expiry_delete', 'delete_force'], true)) {
|
|
CloudVpcPortService::releaseAfterInstanceDeletedLocked(
|
|
$instance
|
|
);
|
|
CloudInstanceNetworkPolicy::
|
|
releaseAfterInstanceDeletedLocked($instance);
|
|
$instance->status = 'deleted';
|
|
$instance->sealed_credential = null;
|
|
CloudBackupSubscription::where(
|
|
'instance_id', (int) $instance->id
|
|
)->update([
|
|
'enabled' => 0,
|
|
'renew_next_period' => 0,
|
|
'updated_at' => time(),
|
|
]);
|
|
CloudInstanceResourceRelease::
|
|
releaseAfterConfirmedDelete(
|
|
$instance,
|
|
'instance_deleted'
|
|
);
|
|
} elseif ($job->kind === 'instance_settings') {
|
|
// The Agent must acknowledge the controlled seed
|
|
// replacement before PHP changes any cached metadata.
|
|
// Passwords and public keys are intentionally never
|
|
// copied into spec_json or result_json.
|
|
if (!is_array($result) || ($result['settings_changed'] ?? false) !== true) {
|
|
self::throwInvalidSettingsResult(
|
|
$preparedRecoveryMarker,
|
|
'Agent 成功结果缺少已确认的实例设置'
|
|
);
|
|
}
|
|
$requested = is_array($payload['body'] ?? null)
|
|
? $payload['body'] : [];
|
|
if (array_key_exists('hostname', $requested)) {
|
|
$confirmed = $result['hostname'] ?? null;
|
|
if (!is_string($confirmed)
|
|
|| $confirmed !== $requested['hostname']
|
|
|| preg_match('/^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*$/D', $confirmed) !== 1
|
|
|| strlen($confirmed) > 253) {
|
|
self::throwInvalidSettingsResult(
|
|
$preparedRecoveryMarker,
|
|
'Agent 成功结果缺少已确认的主机名'
|
|
);
|
|
}
|
|
}
|
|
if (isset($payload['credential'])
|
|
&& is_array($payload['credential'])) {
|
|
$instance->sealed_credential = CloudSecrets::encrypt(
|
|
$payload['credential'], 'instance-credential'
|
|
);
|
|
}
|
|
$decoded = json_decode((string) $instance->spec_json, true);
|
|
if (!is_array($decoded) || array_is_list($decoded)) {
|
|
throw new CloudError('虚拟机配置记录损坏,无法保存实例设置', 409);
|
|
}
|
|
if (array_key_exists('hostname', $requested)) {
|
|
$decoded['guest_hostname'] = $result['hostname'];
|
|
}
|
|
$instance->spec_json = json_encode($decoded,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE);
|
|
} elseif (in_array((string) $job->kind, [
|
|
'service_suspend', 'network_suspend', 'network_resume',
|
|
], true) && is_array($result)
|
|
&& ($result['skipped'] ?? false) === true) {
|
|
// The desired state changed before dispatch. No remote
|
|
// effect occurred, so retain the current projection and
|
|
// let lifecycle derive the next generation if needed.
|
|
} elseif ($job->kind === 'service_suspend'
|
|
&& in_array((string) $instance->service_status,
|
|
array_merge(
|
|
['purge_stopping', 'purge_force_required'],
|
|
$preparedRecoveryMarker === null
|
|
? [] : ['purge_stop_failed', 'suspend_failed']
|
|
), true)) {
|
|
if ((string) $instance->status === 'deleting'
|
|
&& in_array((string) $instance->service_status,
|
|
[
|
|
'purge_stopping', 'purge_force_required',
|
|
'purge_stop_failed',
|
|
], true)) {
|
|
$instance->service_status = 'purge_deleting';
|
|
} elseif ((string) $instance->service_status
|
|
=== 'suspend_failed') {
|
|
$instance->service_status = 'suspended_expired';
|
|
$instance->suspended_at = time();
|
|
}
|
|
} elseif ($job->kind === 'service_suspend'
|
|
&& !in_array((string) $instance->service_status,
|
|
['suspended_expired', 'purge_deleting'], true)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'服务暂停成功结论与实例状态矛盾',
|
|
409
|
|
);
|
|
} elseif ($job->kind === 'network_suspend'
|
|
&& in_array((string) $instance->network_status,
|
|
array_merge(
|
|
['blocking', 'blocking_expired'],
|
|
$preparedRecoveryMarker === null
|
|
? [] : ['block_failed']
|
|
), true)) {
|
|
$instance->network_status = ($payload['suspend_reason'] ?? '') === 'expired'
|
|
? 'blocked_expired' : 'blocked_traffic';
|
|
$instance->network_suspended_at = time();
|
|
} elseif ($job->kind === 'network_suspend'
|
|
&& !in_array((string) $instance->network_status,
|
|
['blocked_expired', 'blocked_traffic'], true)) {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'网络暂停成功结论与实例状态矛盾',
|
|
409
|
|
);
|
|
} elseif ($job->kind === 'network_resume'
|
|
&& in_array((string) $instance->network_status,
|
|
array_merge(
|
|
[
|
|
'restoring', 'restore_pending',
|
|
'blocking', 'blocking_expired',
|
|
],
|
|
$preparedRecoveryMarker === null
|
|
? [] : ['restore_failed']
|
|
), true)) {
|
|
if (((string) $instance->service_status === 'expired_grace'
|
|
&& $instance->expires_at !== null
|
|
&& (int) $instance->expires_at <= time())
|
|
|| (string) $instance->network_status
|
|
=== 'blocking_expired') {
|
|
// Expiry may race with a previously valid resume.
|
|
// The receipt proves the links are now up, so queue
|
|
// a fresh suspend generation instead of publishing
|
|
// a misleading active projection.
|
|
$instance->network_status = 'blocking_expired';
|
|
} elseif ((string) $instance->network_status
|
|
=== 'blocking') {
|
|
$instance->network_status = 'blocking';
|
|
} else {
|
|
$instance->network_status = 'active';
|
|
}
|
|
$instance->network_suspended_at = null;
|
|
} elseif ($job->kind === 'network_resume'
|
|
&& (string) $instance->network_status !== 'active') {
|
|
throw new CloudRecoveryInstanceStateConflict(
|
|
'网络恢复成功结论与实例状态矛盾',
|
|
409
|
|
);
|
|
}
|
|
// Successful power/reinstall/policy operations often
|
|
// include the resulting VM state. Persist that bounded
|
|
// observation so the next page render is immediate;
|
|
// missing state is deliberately left for vm-states.
|
|
self::recordRuntime($instance, is_array($result) ? $result : [],
|
|
(string) $job->kind, $payload);
|
|
$instance->updated_at = time();
|
|
$instance->save();
|
|
}
|
|
if ($localFinalizerRecovery) {
|
|
// The kind-specific projection above is the only work being
|
|
// retried. Preserve the original terminal receipt audit
|
|
// (phase/outcome/status/terminal_at/error/result/event) and
|
|
// update only the independently recoverable effect ledger.
|
|
$now = time();
|
|
$job->effect_may_have_applied = 0;
|
|
$job->safe_to_resubmit = 1;
|
|
$job->effect_resolution = 'not_applied';
|
|
$job->effect_resolved_at = $now;
|
|
$job->effect_resolved_by = null;
|
|
$job->effect_resolution_reason =
|
|
'automatic_local_not_dispatched';
|
|
$job->sealed_payload = CloudSecrets::encrypt(
|
|
CloudInput::safeResult($payload),
|
|
'job-payload'
|
|
);
|
|
$job->finalization_incident = 0;
|
|
$job->finalization_error_code = null;
|
|
$job->read_at = null;
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->available_at = 0;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
$job,
|
|
null
|
|
);
|
|
return;
|
|
}
|
|
if ($recoveredCreateEvent !== null) {
|
|
$now = time();
|
|
$safePayload = CloudInput::safeResult($payload);
|
|
$safePayload[CloudOperationRecoveryIdentity::payloadKey()] =
|
|
CloudOperationRecoveryIdentity::persisted(
|
|
$operationIdentity
|
|
);
|
|
$job->completion_event_id =
|
|
(string) $recoveredCreateEvent->event_id;
|
|
$job->effect_resolution = 'applied';
|
|
$job->effect_resolved_at = $now;
|
|
$job->effect_resolved_by = null;
|
|
$job->effect_resolution_reason =
|
|
'late_completion_receipt:'
|
|
. (string) $recoveredCreateEvent->event_id;
|
|
$job->sealed_payload = CloudSecrets::encrypt(
|
|
$safePayload,
|
|
'job-payload'
|
|
);
|
|
$job->result_json = $safe === null ? null : json_encode(
|
|
$safe,
|
|
JSON_THROW_ON_ERROR
|
|
);
|
|
$job->finalization_incident = 0;
|
|
$job->finalization_error_code = null;
|
|
$job->read_at = null;
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->available_at = 0;
|
|
$job->updated_at = $now;
|
|
$job->save();
|
|
$order->status = 'succeeded';
|
|
$order->updated_at = $now;
|
|
$order->save();
|
|
$recoveredCreateEvent->disposition = 'applied';
|
|
$recoveredCreateEvent->applied_at = $now;
|
|
$recoveredCreateEvent->save();
|
|
return;
|
|
}
|
|
if ($preparedRecoveryMarker !== null) {
|
|
// The resource projection and this marker commit atomically.
|
|
// Keep the original encrypted request plus the exact validated
|
|
// operation until commit runs after the matching Agent fence
|
|
// has been released. This makes the prepare/fence crash window
|
|
// resumable for every registered kind, not only settings.
|
|
$preparedPayload = self::compactPreparedRecoveryPayload(
|
|
(string) $job->kind,
|
|
$payload
|
|
);
|
|
$preparedBinding = self::preparedRecoveryBinding(
|
|
$preparedRecoveryMarker
|
|
);
|
|
if ($preparedBinding === null) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'任务预提交恢复标记无效',
|
|
409
|
|
);
|
|
}
|
|
$preparedPayload['_operation_recovery'] = [
|
|
'version' => 1,
|
|
'kind' => (string) $job->kind,
|
|
'prepared_marker' => $preparedBinding,
|
|
'operation' => $operation,
|
|
];
|
|
$encodedPrepared = json_encode(
|
|
$preparedPayload,
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
| JSON_UNESCAPED_UNICODE
|
|
);
|
|
if (strlen($encodedPrepared) > 196608) {
|
|
throw new CloudRecoveryPayloadUnavailable(
|
|
'任务预提交恢复数据过大',
|
|
409
|
|
);
|
|
}
|
|
$job->effect_resolution_reason = $preparedRecoveryMarker;
|
|
$job->sealed_payload = CloudSecrets::encrypt(
|
|
$preparedPayload,
|
|
'job-payload'
|
|
);
|
|
if ($forceRequired) {
|
|
// This exact no-effect Agent conclusion authorizes a new,
|
|
// separately audited force-delete generation. Keep the
|
|
// concrete code on the immutable graceful task instead of
|
|
// its earlier transport/deadline summary.
|
|
$job->error_code = 'force_required';
|
|
}
|
|
if (self::lateReceiptRecoveryMarker(
|
|
$preparedRecoveryMarker
|
|
) && in_array(
|
|
(string) ($job->finalization_error_code ?? ''),
|
|
[
|
|
'recovery_limit_reached',
|
|
'recovery_state_invalid',
|
|
],
|
|
true
|
|
)) {
|
|
$job->finalization_incident = 0;
|
|
$job->finalization_error_code = null;
|
|
}
|
|
$job->updated_at = time();
|
|
$job->save();
|
|
if ($order !== null) {
|
|
$order->status = $status;
|
|
$order->updated_at = time();
|
|
$order->save();
|
|
}
|
|
return;
|
|
}
|
|
$job->status = $status;
|
|
$job->phase = 'terminal';
|
|
$job->outcome = $status;
|
|
$job->terminal_at = time();
|
|
$job->effect_may_have_applied = $effectMayHaveApplied ? 1 : 0;
|
|
$job->safe_to_resubmit = $safeToResubmit ? 1 : 0;
|
|
$job->effect_resolution = $status === 'succeeded' ? 'applied'
|
|
: ($effectMayHaveApplied ? 'unknown' : 'not_applied');
|
|
$retainRecoveryEscrow = $status === 'failed'
|
|
&& $effectMayHaveApplied
|
|
&& (CloudOperationRecoveryPolicy::supports(
|
|
(string) $job->kind
|
|
) || CloudOperationRecoveryPolicy::isCreateLike(
|
|
(string) $job->kind
|
|
));
|
|
if (!$retainRecoveryEscrow) {
|
|
// A resolved terminal task never needs credentials again.
|
|
// Unknown operations in the explicit recovery policy retain
|
|
// encrypted escrow only until their effect is resolved.
|
|
$safePayload = CloudInput::safeResult($payload);
|
|
if ($operationIdentity !== null) {
|
|
$safePayload[
|
|
CloudOperationRecoveryIdentity::payloadKey()
|
|
] = CloudOperationRecoveryIdentity::persisted(
|
|
$operationIdentity
|
|
);
|
|
}
|
|
$job->sealed_payload = CloudSecrets::encrypt(
|
|
$safePayload, 'job-payload'
|
|
);
|
|
}
|
|
$job->result_json = $safe === null ? null : json_encode($safe, JSON_THROW_ON_ERROR);
|
|
$job->error_code = CloudOperationErrorCatalog::workerSummary(
|
|
(string) $job->kind, $status, $operation, $order !== null,
|
|
self::payloadAction($payload)
|
|
);
|
|
$job->lease_token = null;
|
|
$job->lease_until = 0;
|
|
$job->available_at = 0;
|
|
$job->updated_at = time();
|
|
$job->save();
|
|
// Aggregate catalog parents may be failed/unknown even though all
|
|
// leaf effects are already known (for example one replica applied
|
|
// and another failed before changing the host). Publish the parent
|
|
// effect now as well as after a later recovery commit, otherwise a
|
|
// fully-known partial aggregate would retain a permanent fence.
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
$job,
|
|
null
|
|
);
|
|
if ($order !== null) {
|
|
if ($job->kind === 'create' && $effectMayHaveApplied) {
|
|
CloudPlacementService::markUncertain($order);
|
|
}
|
|
$order->status = $status;
|
|
$order->updated_at = time();
|
|
$order->save();
|
|
}
|
|
}, 3);
|
|
if ((string) $claimed->kind === 'host_storage_configure'
|
|
&& $status === 'succeeded') {
|
|
$this->scheduleTemplateBackfillAfterStorage(
|
|
(int) $claimed->host_id
|
|
);
|
|
$this->scheduleRescueBackfillAfterStorage(
|
|
(int) $claimed->host_id
|
|
);
|
|
}
|
|
if ((string) $claimed->kind
|
|
=== 'rescue_catalog_validate_replica'
|
|
&& $status === 'succeeded') {
|
|
$this->scheduleRescueDistributionAfterValidation(
|
|
$claimed
|
|
);
|
|
}
|
|
}
|
|
|
|
/** Run object-storage discovery only after the storage transaction commits. */
|
|
private function scheduleTemplateBackfillAfterStorage(int $hostId): void
|
|
{
|
|
try {
|
|
$actor = CloudDaemonActor::resolve('cloud-template-backfill');
|
|
if ($actor === null) {
|
|
throw new CloudError(
|
|
'云镜像补齐缺少可用的系统管理员执行者',
|
|
503
|
|
);
|
|
}
|
|
CloudTemplateHostBackfillService::scheduleReadyHost(
|
|
$hostId,
|
|
$actor,
|
|
'storage_ready'
|
|
);
|
|
} catch (\Throwable $error) {
|
|
error_log(sprintf(
|
|
'cloud template backfill host=%d trigger=storage_ready unavailable: %s',
|
|
$hostId,
|
|
get_class($error)
|
|
));
|
|
}
|
|
}
|
|
|
|
/** Start all-host distribution only after first-host boot validation. */
|
|
private function scheduleRescueDistributionAfterValidation(
|
|
CloudJob $job
|
|
): void {
|
|
try {
|
|
$replica = CloudRescueTemplateReplica::query()
|
|
->where('job_id', (int) $job->id)
|
|
->where('host_id', (int) $job->host_id)->first();
|
|
$templateId = $replica === null
|
|
? '' : (string) $replica->template_id;
|
|
if (preg_match('/^[a-f0-9]{32}$/D', $templateId) !== 1) {
|
|
throw new CloudError('救援 ISO 校验任务缺少目录标识', 409);
|
|
}
|
|
$actor = CloudDaemonActor::resolve(
|
|
'cloud-rescue-distribution'
|
|
);
|
|
if ($actor === null) {
|
|
throw new CloudError(
|
|
'救援 ISO 分发缺少可用的系统管理员执行者',
|
|
503
|
|
);
|
|
}
|
|
CloudRescueTemplateReplicaCoordinator::distribute(
|
|
$templateId,
|
|
$actor,
|
|
'validation_complete'
|
|
);
|
|
} catch (\Throwable $error) {
|
|
// The validation and immutable object promotion already committed.
|
|
// The periodic reconciler resumes this idempotently.
|
|
error_log(sprintf(
|
|
'cloud rescue distribution validation_job=%d deferred: %s',
|
|
(int) $job->id,
|
|
get_class($error)
|
|
));
|
|
}
|
|
}
|
|
|
|
/** Rescue replicas share the same host-readiness boundary as templates. */
|
|
private function scheduleRescueBackfillAfterStorage(int $hostId): void
|
|
{
|
|
try {
|
|
$actor = CloudDaemonActor::resolve('cloud-rescue-backfill');
|
|
if ($actor === null) {
|
|
throw new CloudError(
|
|
'救援 ISO 补齐缺少可用的系统管理员执行者',
|
|
503
|
|
);
|
|
}
|
|
CloudRescueTemplateHostBackfillService::scheduleReadyHost(
|
|
$hostId,
|
|
$actor,
|
|
'storage_ready'
|
|
);
|
|
} catch (\Throwable $error) {
|
|
error_log(sprintf(
|
|
'cloud rescue backfill host=%d trigger=storage_ready unavailable: %s',
|
|
$hostId,
|
|
get_class($error)
|
|
));
|
|
}
|
|
}
|
|
|
|
private static function throwInvalidSettingsResult(
|
|
?string $preparedRecoveryMarker,
|
|
string $message
|
|
): never {
|
|
if ($preparedRecoveryMarker !== null
|
|
&& self::lateReceiptRecoveryMarker(
|
|
$preparedRecoveryMarker
|
|
)) {
|
|
throw new CloudRecoveryReceiptInvalid($message, 502);
|
|
}
|
|
throw new CloudError($message, 502);
|
|
}
|
|
|
|
/** Rescue success is accepted only for the exact requested generation. */
|
|
private static function assertRescueResultMatchesRequest(
|
|
CloudJob $job,
|
|
string $status,
|
|
mixed $result,
|
|
array $payload
|
|
): void {
|
|
if ((string) $job->kind !== 'rescue' || $status !== 'succeeded') {
|
|
return;
|
|
}
|
|
$target = (string) ($payload['request_target'] ?? '');
|
|
$body = is_array($payload['body'] ?? null)
|
|
? $payload['body'] : [];
|
|
if (!is_array($result) || array_is_list($result)
|
|
|| !is_bool($result['active'] ?? null)
|
|
|| !is_bool($result['staged'] ?? null)
|
|
|| !is_bool($result['network_suspended'] ?? null)) {
|
|
throw new CloudError('Agent 救援操作成功回执无效', 502);
|
|
}
|
|
if (str_ends_with($target, '/rescue/enter')) {
|
|
if (!(($result['active'] ?? false)
|
|
|| ($result['staged'] ?? false))
|
|
|| ($result['template_id'] ?? null)
|
|
!== ($body['template_id'] ?? null)
|
|
|| ($result['revision'] ?? null)
|
|
!== ($body['revision'] ?? null)
|
|
|| ($result['generation_token'] ?? null)
|
|
!== ($body['generation_token'] ?? null)) {
|
|
throw new CloudError(
|
|
'Agent 救援操作回执与请求版本不匹配',
|
|
502
|
|
);
|
|
}
|
|
return;
|
|
}
|
|
if (str_ends_with($target, '/rescue/force-start')) {
|
|
if (($result['active'] ?? false) !== true
|
|
|| !is_string($result['template_id'] ?? null)
|
|
|| preg_match('/^[a-f0-9]{32}$/D',
|
|
$result['template_id']) !== 1
|
|
|| !is_int($result['revision'] ?? null)
|
|
|| (int) $result['revision'] < 1
|
|
|| !is_string($result['generation_token'] ?? null)
|
|
|| preg_match('/^[a-f0-9]{32}$/D',
|
|
$result['generation_token']) !== 1) {
|
|
throw new CloudError('Agent 救援强制启动回执无效', 502);
|
|
}
|
|
return;
|
|
}
|
|
if (!str_ends_with($target, '/rescue/exit')
|
|
|| ($result['active'] ?? true) !== false
|
|
|| ($result['staged'] ?? true) !== false
|
|
|| !in_array($result['template_id'] ?? '', ['', null], true)
|
|
|| !in_array($result['revision'] ?? 0, [0, null], true)
|
|
|| !in_array(
|
|
$result['generation_token'] ?? '',
|
|
['', null],
|
|
true
|
|
)) {
|
|
throw new CloudError('Agent 救援退出回执无效', 502);
|
|
}
|
|
}
|
|
|
|
private static function validSystemCleanup(CloudJob $job, array $payload): bool
|
|
{
|
|
if ($job->instance_id === null
|
|
|| ($payload['body'] ?? null) !== ['remove_managed_disks' => true]) {
|
|
return false;
|
|
}
|
|
$instance = CloudInstance::find($job->instance_id);
|
|
if ($instance === null || (string) $instance->status !== 'deleting'
|
|
|| (string) $instance->service_status !== 'purge_deleting'
|
|
|| (int) $job->user_id !== (int) $instance->user_id
|
|
|| (int) $job->host_id !== (int) $instance->host_id
|
|
|| (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id) {
|
|
return false;
|
|
}
|
|
return ($payload['method'] ?? null) === 'DELETE'
|
|
&& ($payload['request_target'] ?? null) === '/v1/vms/'
|
|
. rawurlencode((string) $instance->uuid);
|
|
}
|
|
|
|
private static function validSystemPolicy(CloudJob $job, array $payload): bool
|
|
{
|
|
if ($job->instance_id === null || !is_array($payload['body'] ?? null)) {
|
|
return false;
|
|
}
|
|
$instance = CloudInstance::find($job->instance_id);
|
|
if ($instance === null || (int) $job->user_id !== (int) $instance->user_id
|
|
|| (int) $job->host_id !== (int) $instance->host_id
|
|
|| (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id) {
|
|
return false;
|
|
}
|
|
$suffix = [
|
|
'service_suspend' => 'service/suspend',
|
|
'network_suspend' => 'network/suspend',
|
|
'network_resume' => 'network/resume',
|
|
][$job->kind] ?? null;
|
|
$expectedBody = match ((string) $job->kind) {
|
|
'service_suspend' => ['reason' => 'purge'],
|
|
'network_suspend' => ['reason' => (string) (
|
|
$payload['suspend_reason'] ?? ''
|
|
)],
|
|
'network_resume' => [],
|
|
default => null,
|
|
};
|
|
return $suffix !== null && $expectedBody !== null
|
|
&& ($payload['method'] ?? null) === 'POST'
|
|
&& ($payload['body'] ?? null) === $expectedBody
|
|
&& ($payload['request_target'] ?? null) === '/v1/vms/'
|
|
. rawurlencode((string) $instance->uuid) . '/' . $suffix;
|
|
}
|
|
|
|
private static function payloadAction(array $payload): ?string
|
|
{
|
|
$body = $payload['body'] ?? null;
|
|
$action = is_array($body) && is_string($body['action'] ?? null)
|
|
? $body['action'] : null;
|
|
return CloudOperationCatalog::isPowerAction($action ?? '') ? $action : null;
|
|
}
|
|
|
|
/**
|
|
* Validate the immutable purchase-time stock mode before materialising an
|
|
* exact Agent create success. Legacy finite orders retain their reservation
|
|
* marker. A markerless legacy order is accepted as unlimited only while the
|
|
* locked offer row still proves that it was unchanged before the order.
|
|
*/
|
|
private static function createStockLedgerIsConsistent(
|
|
CloudOrder $order
|
|
): bool {
|
|
$reserved = $order->stock_reserved_at !== null;
|
|
$released = $order->stock_released_at !== null;
|
|
$spec = json_decode((string) $order->spec_json, true);
|
|
if (!is_array($spec) || array_is_list($spec)) {
|
|
return false;
|
|
}
|
|
if (array_key_exists('stock_mode', $spec)) {
|
|
if ($spec['stock_mode'] === 'finite') {
|
|
return $reserved && !$released;
|
|
}
|
|
if ($spec['stock_mode'] === 'unlimited') {
|
|
return !$reserved && !$released;
|
|
}
|
|
return false;
|
|
}
|
|
if ($reserved) {
|
|
return !$released;
|
|
}
|
|
if ($released) {
|
|
return false;
|
|
}
|
|
foreach ([
|
|
'offer_id' => (int) $order->offer_id,
|
|
'plan_id' => (int) $order->plan_id,
|
|
'resource_pool_id' => (int) $order->resource_pool_id,
|
|
] as $key => $expected) {
|
|
if (($spec[$key] ?? null) !== $expected) {
|
|
return false;
|
|
}
|
|
}
|
|
$offer = CloudOffer::where('id', (int) $order->offer_id)
|
|
->lockForUpdate()->first();
|
|
return $offer !== null
|
|
&& (int) $offer->plan_id === (int) $order->plan_id
|
|
&& (int) $offer->resource_pool_id
|
|
=== (int) $order->resource_pool_id
|
|
&& (int) $offer->stock === -1
|
|
&& (int) $offer->created_at > 0
|
|
&& (int) $offer->created_at <= (int) $offer->updated_at
|
|
&& (int) $offer->updated_at < (int) $order->created_at
|
|
&& ($spec['price_version'] ?? null)
|
|
=== (int) $offer->price_version;
|
|
}
|
|
|
|
/**
|
|
* Validate the mandatory PHP-owned template snapshot without forwarding
|
|
* it to the Agent. This is a clean-break schema: create/reinstall work
|
|
* without the snapshot is invalid and finalizes as a local failure.
|
|
*/
|
|
private static function trustedTemplateMetadata(array $payload): array
|
|
{
|
|
if (array_key_exists('backup_metadata', $payload)) {
|
|
$metadata = $payload['backup_metadata'];
|
|
$body = $payload['body'] ?? null;
|
|
if (!is_array($metadata) || array_is_list($metadata)
|
|
|| !is_array($body) || array_is_list($body)
|
|
|| filter_var(
|
|
$metadata['backup_id'] ?? null,
|
|
FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 1]]
|
|
) === false
|
|
|| preg_match('/^[a-f0-9]{32}$/D',
|
|
(string) ($metadata['backup_key'] ?? '')) !== 1
|
|
|| !hash_equals(
|
|
(string) $metadata['backup_key'],
|
|
(string) ($body['backup_key'] ?? '')
|
|
)
|
|
|| ($metadata['os_source'] ?? null) !== 'cloud_template'
|
|
|| !is_string($metadata['os_name'] ?? null)
|
|
|| trim($metadata['os_name']) === ''
|
|
|| ($metadata['login_name'] ?? null) !== 'root'
|
|
|| !is_bool($metadata['debian_12_network_fix'] ?? null)
|
|
|| ($body['debian_12_network_fix'] ?? null)
|
|
!== $metadata['debian_12_network_fix']) {
|
|
throw new CloudError(
|
|
'任务中的云备份展示快照与 Agent 请求不匹配,必须人工核查',
|
|
409
|
|
);
|
|
}
|
|
return [
|
|
'os_source' => 'cloud_template',
|
|
'os_name' => $metadata['os_name'],
|
|
'login_name' => 'root',
|
|
'debian_12_network_fix' =>
|
|
$metadata['debian_12_network_fix'],
|
|
'restored_from_backup' => $metadata['backup_key'],
|
|
];
|
|
}
|
|
if (!array_key_exists('template_metadata', $payload)) {
|
|
throw new CloudError(
|
|
'任务缺少云镜像展示快照,必须终止并人工核查', 409
|
|
);
|
|
}
|
|
$metadata = $payload['template_metadata'];
|
|
$body = $payload['body'] ?? null;
|
|
if (!is_array($metadata) || array_is_list($metadata)
|
|
|| !is_array($body) || array_is_list($body)
|
|
|| !is_string($metadata['template_id'] ?? null)
|
|
|| !is_string($body['template_id'] ?? null)
|
|
|| preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D',
|
|
$metadata['template_id']) !== 1
|
|
|| !hash_equals($body['template_id'],
|
|
$metadata['template_id'])) {
|
|
throw new CloudError(
|
|
'任务中的云镜像展示快照与 Agent 请求不匹配,必须人工核查',
|
|
409
|
|
);
|
|
}
|
|
$public = CloudInstanceMetadata::publicSpec($metadata);
|
|
if (($public['os_source'] ?? null) !== 'cloud_template'
|
|
|| !is_string($public['os_name'] ?? null)
|
|
|| ($public['login_name'] ?? null) !== 'root'
|
|
|| !is_bool($metadata['debian_12_network_fix'] ?? null)) {
|
|
throw new CloudError('任务中的云镜像展示快照无效,必须人工核查',
|
|
409);
|
|
}
|
|
return [
|
|
'template_id' => $metadata['template_id'],
|
|
'os_source' => 'cloud_template',
|
|
'os_name' => $public['os_name'],
|
|
'login_name' => 'root',
|
|
'debian_12_network_fix' =>
|
|
$metadata['debian_12_network_fix'],
|
|
];
|
|
}
|
|
|
|
/** Reject retired restore jobs before the dispatch CAS can contact Agent. */
|
|
private static function assertPortableRestoreDispatchReady(
|
|
CloudJob $job,
|
|
array $payload
|
|
): void {
|
|
$attemptKey = $payload['attempt_key'] ?? null;
|
|
if (!CloudPortableRestoreFinalizer::owns($job)
|
|
|| !is_string($attemptKey)
|
|
|| preg_match('/^[a-f0-9]{32}$/D', $attemptKey) !== 1) {
|
|
throw new CloudError('旧备份恢复任务已停用', 409);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Create the one transient restore request and persist only its immutable
|
|
* Agent-journal identity. The URL is never returned in the stable payload,
|
|
* so every terminal, prepared and unknown-effect database path remains
|
|
* capability-free while still being able to reconcile the exact request.
|
|
*
|
|
* @return array{0:array,1:array} stable payload, wire payload
|
|
*/
|
|
private static function portableRestoreDispatchPayload(
|
|
CloudJob $job,
|
|
array $payload
|
|
): array {
|
|
$envelope = CloudPortableRestoreJobEnvelope::fromJobForDispatch(
|
|
$job,
|
|
$payload
|
|
);
|
|
$source = CloudBackupService::provisioningSource(
|
|
$envelope->backup()
|
|
);
|
|
$sourceUrl = $source['source_url'] ?? null;
|
|
if (!is_string($sourceUrl)) {
|
|
throw new CloudError('恢复任务临时下载能力缺失', 409);
|
|
}
|
|
$wire = $envelope->wirePayload($sourceUrl);
|
|
$identity = CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
$job,
|
|
$wire
|
|
);
|
|
$stable = $envelope->payload();
|
|
$stable[CloudOperationRecoveryIdentity::payloadKey()] =
|
|
CloudOperationRecoveryIdentity::persisted($identity);
|
|
return [$stable, $wire];
|
|
}
|
|
|
|
/**
|
|
* @return array{action:string,resource_id:string,request_fingerprint:string}
|
|
*/
|
|
private static function operationIdentityForFinalization(
|
|
CloudJob $job,
|
|
array $payload,
|
|
bool $forAppliedEffect,
|
|
bool $preparedRecovery
|
|
): array {
|
|
$portableAttempt = (string) $job->kind === 'backup_restore'
|
|
&& is_string($payload['attempt_key'] ?? null)
|
|
&& isset($payload[
|
|
CloudOperationRecoveryIdentity::payloadKey()
|
|
]);
|
|
if ($preparedRecovery || $portableAttempt) {
|
|
return CloudOperationRecoveryIdentity::expected(
|
|
$job,
|
|
$forAppliedEffect
|
|
);
|
|
}
|
|
return CloudOperationRecoveryIdentity::expectedFromPayload(
|
|
$job,
|
|
$payload,
|
|
$forAppliedEffect
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Translate panel vocabulary only at the Agent wire boundary. The PHP job
|
|
* keeps "poweroff" so its audit label and user-facing history remain
|
|
* stable, while the Go protocol uses libvirt's explicit "destroy" action.
|
|
*/
|
|
private static function agentRequestBody(string $kind, array $body): array
|
|
{
|
|
return CloudOperationRecoveryIdentity::wireBody($kind, $body);
|
|
}
|
|
|
|
/**
|
|
* Bind every journal-backed Agent response to the claimed PHP job before
|
|
* its status or result can influence durable panel state. Missing IDs and
|
|
* cross-operation responses are ambiguous, never legacy success shortcuts.
|
|
*/
|
|
private static function operationForJob(CloudJob $job,
|
|
array $operation): array
|
|
{
|
|
$expected = (string) $job->operation_id;
|
|
$actual = $operation['operation_id'] ?? null;
|
|
if (!is_string($actual) || $expected === ''
|
|
|| !hash_equals($expected, $actual)) {
|
|
throw new CloudError(
|
|
'Agent 操作标识与当前任务不匹配,必须人工核查',
|
|
502
|
|
);
|
|
}
|
|
if (CloudOperationRecoveryIdentity::supports((string) $job->kind)) {
|
|
$expectedIdentity = CloudOperationRecoveryIdentity::expected(
|
|
$job,
|
|
strtolower((string) ($operation['status'] ?? ''))
|
|
=== 'succeeded'
|
|
);
|
|
$fingerprint = $operation['fingerprint']
|
|
?? $operation['request_fingerprint'] ?? null;
|
|
CloudOperationRecoveryIdentity::assertMatches(
|
|
$expectedIdentity,
|
|
[
|
|
'action' => $operation['action'] ?? null,
|
|
'resource_id' => $operation['resource_id'] ?? null,
|
|
'request_fingerprint' => $fingerprint,
|
|
]
|
|
);
|
|
}
|
|
return $operation;
|
|
}
|
|
|
|
private static function normalizeAgentOperation(array $operation): array
|
|
{
|
|
$causeCode = CloudOperationErrorCatalog::agentCauseCode(
|
|
$operation['cause_code'] ?? null
|
|
);
|
|
if ($causeCode === null) {
|
|
unset($operation['cause_code']);
|
|
} else {
|
|
$operation['cause_code'] = $causeCode;
|
|
}
|
|
$error = $operation['error'] ?? null;
|
|
if (is_array($error) && is_string($error['code'] ?? null)
|
|
&& preg_match('/^[a-z][a-z0-9_]{0,63}$/D', $error['code']) === 1) {
|
|
$operation['error_code'] = $error['code'];
|
|
}
|
|
// Never persist or expose free-form Agent error messages. The local
|
|
// catalog maps the bounded code and operation kind to reviewed text.
|
|
if (is_array($error)) {
|
|
unset($operation['error']);
|
|
}
|
|
return $operation;
|
|
}
|
|
|
|
/**
|
|
* Persist only a successful, allow-listed power observation. Runtime
|
|
* cache failures are isolated from the durable job outcome: the Agent
|
|
* result and billing state remain authoritative even during a rolling
|
|
* schema upgrade where the cache table may not exist yet.
|
|
*/
|
|
private static function recordRuntime(CloudInstance $instance, array $result,
|
|
string $kind, array $payload): void
|
|
{
|
|
try {
|
|
if (in_array($kind,
|
|
['delete', 'expiry_delete', 'delete_force'], true)) {
|
|
CloudInstanceRuntimeService::forget((int) $instance->id);
|
|
return;
|
|
}
|
|
$source = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
|
|
$state = $source['status']
|
|
?? ($source['state'] ?? ($source['power_state'] ?? null));
|
|
$action = $kind === 'action' ? self::payloadAction($payload) : null;
|
|
$normalized = CloudInstanceRuntimeService::normalizeState($state);
|
|
if ($kind === 'action') {
|
|
// A successful graceful-shutdown job only confirms that the
|
|
// request was accepted. libvirt may still report running (or
|
|
// omit the state) until the guest completes ACPI shutdown.
|
|
// Preserve that distinction instead of publishing either a
|
|
// new fresh-running observation or a fabricated stopped one.
|
|
$state = CloudInstanceRuntimeService::stateAfterSuccessfulAction(
|
|
$state, $action
|
|
);
|
|
}
|
|
$instanceId = (int) $instance->id;
|
|
$hostId = (int) $instance->host_id;
|
|
$uuid = (string) $instance->uuid;
|
|
if ($kind === 'create') {
|
|
CloudInstanceRuntimeService::seed($instanceId, $hostId, $uuid);
|
|
}
|
|
if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') {
|
|
CloudInstanceRuntimeService::markObserved(
|
|
$instanceId, $hostId, $uuid, $state
|
|
);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
error_log('cloud runtime observation write unavailable: ' . get_class($error));
|
|
}
|
|
}
|
|
|
|
/** Fail a malformed clean-break order before any Agent create mutation. */
|
|
private static function assertCreateTrafficPoolSnapshot(CloudJob $job): void
|
|
{
|
|
if ($job->order_id === null) {
|
|
throw new CloudError('开通任务缺少订单', 409);
|
|
}
|
|
$order = CloudOrder::find((int) $job->order_id);
|
|
$spec = $order === null
|
|
? null : json_decode((string) $order->spec_json, true);
|
|
if (!is_array($spec) || array_is_list($spec)) {
|
|
throw new CloudError('订单计费快照损坏', 409);
|
|
}
|
|
$quota = filter_var(
|
|
$spec['traffic_quota_bytes'] ?? null,
|
|
FILTER_VALIDATE_INT,
|
|
['options' => ['min_range' => 0]]
|
|
);
|
|
if ($quota === false) {
|
|
throw new CloudError('订单缺少不可变的计费周期快照', 409);
|
|
}
|
|
CloudTrafficPoolPeriodBinding::orderSnapshot($spec, (int) $quota);
|
|
}
|
|
|
|
private static function exceptionDiagnostic(\Throwable $error, bool $dispatched,
|
|
?string $kind, bool $hasOrder, ?string $action): string
|
|
{
|
|
$operation = [];
|
|
if ($error instanceof CloudError) {
|
|
// CloudError text is Agent-/host-controlled too. The catalog
|
|
// keeps only exact reviewed details and otherwise returns a
|
|
// stable generic code for the durable job row.
|
|
$operation['error'] = $error->getMessage();
|
|
} elseif ($error instanceof \GuzzleHttp\Exception\TransferException) {
|
|
$operation['error'] = 'Agent 连接中断或操作超时';
|
|
}
|
|
$status = 'failed';
|
|
return CloudOperationErrorCatalog::workerSummary($kind, $status,
|
|
$operation, $hasOrder, $action)
|
|
?? CloudOperationErrorCatalog::userMessage($kind, $status, null,
|
|
$hasOrder, $action)
|
|
?? '虚拟机操作失败,请稍后重试';
|
|
}
|
|
}
|