You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudWorkerService.php

1714 lines
94 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\CloudInstance;
use App\Models\CloudInvoice;
use App\Models\CloudJob;
use App\Models\CloudOrder;
use App\Models\CloudServicePeriod;
use App\Models\CloudVpc;
use App\Models\User;
final class CloudWorkerService
{
private const ENGINE_TIMEOUT_SECONDS = 900.0;
private const POLICY_ENGINE_TIMEOUT_SECONDS = 120.0;
private const LEASE_SECONDS = 930;
private const POLICY_KINDS = ['service_suspend', 'service_resume',
'network_suspend', 'network_resume'];
private const REINSTALL_REVIEW_CODES = [
'reinstall_cleanup_pending', 'reinstall_result_uncertain',
];
private bool $policyOnly;
private array $observedErrors = [];
public function observationErrors(): array
{
return array_values(array_unique($this->observedErrors));
}
public function __construct(private ?CloudEngineClient $engine = null,
bool $policyOnly = false)
{
$this->policyOnly = $policyOnly;
// A cloud-image clone can legitimately take minutes on remote storage.
// Keep the worker lease longer than the HTTP timeout so another worker
// cannot claim the same job while the first request is still active.
$this->engine ??= new CloudEngineClient(null, $policyOnly
? self::POLICY_ENGINE_TIMEOUT_SECONDS : self::ENGINE_TIMEOUT_SECONDS);
}
public static function disposition(array $operation, bool $creating,
?string $kind = null): string
{
$status = is_string($operation['status'] ?? null)
? strtolower(trim($operation['status'])) : 'uncertain';
if ($kind === 'reinstall'
&& in_array($status, ['succeeded', 'completed', 'done', 'failed'], true)
&& self::reinstallNeedsReview($operation)) {
// A legacy/internal response can carry a post-commit marker while
// its top-level status is still failed. Keep it uncertain so the
// replacement credential/spec can be reconciled without allowing
// a destructive reinstall replay.
return 'uncertain';
}
if (in_array($status, ['succeeded', 'completed', 'done'], true)) {
return 'succeeded';
}
if ($status === 'failed') {
if (!$creating || ($operation['safe_to_refund'] ?? false) === true
|| ($operation['result']['creation_absent'] ?? false) === true) {
return 'failed';
}
return 'uncertain';
}
return in_array($status, ['pending', 'running'], true) ? 'running' : 'uncertain';
}
/**
* A reinstall can commit the replacement guest before old-volume cleanup
* is confirmed. Preserve the new credential/spec while keeping the job
* visible as uncertain; legacy engines may only provide a warning, so
* that diagnostic is accepted as the compatibility signal here.
*/
private static function reinstallNeedsReview(array $operation): bool
{
$code = is_string($operation['error_code'] ?? null)
? strtolower(trim($operation['error_code'])) : '';
if (in_array($code, self::REINSTALL_REVIEW_CODES, true)) {
return true;
}
$result = $operation['result'] ?? null;
return is_array($result)
&& (($result['reinstall_committed'] ?? false) === true
|| ($result['cleanup_pending'] ?? false) === true
|| (is_string($result['warning'] ?? null)
&& trim($result['warning']) !== ''));
}
/**
* A destructive job may capture a VPC removal tombstone just before the
* control-plane member row is finalized as deleted. The host still needs
* that immutable tombstone to remove a stale private NIC/route/peer; the
* current disabled projection is not permission to skip the host proof.
*/
private static function deleteTombstoneDetached(
string $kind, ?array $captured, array $current
): bool {
if (!in_array($kind, ['delete', 'expiry_delete'], true)
|| !is_array($captured)
|| ($captured['required'] ?? false) !== true
|| ($captured['desired_state'] ?? null) !== 'removed'
|| ($current['required'] ?? true) !== false
|| ($current['desired_state'] ?? null) !== 'disabled'
|| ($current['vpc_id'] ?? null) !== null
|| ($current['member_id'] ?? null) !== null
|| ($current['private_ip_cidr'] ?? null) !== null
|| ($current['private_mac'] ?? null) !== null
|| (int) ($current['desired_revision'] ?? -1) !== 0
|| ($current['desired_digest'] ?? null) !== null) {
return false;
}
return is_string($captured['vpc_id'] ?? null)
&& preg_match('/^[0-9a-f]{32}$/Di', $captured['vpc_id']) === 1
&& is_string($captured['member_id'] ?? null)
&& preg_match('/^[0-9a-f]{32}$/Di', $captured['member_id']) === 1
&& (int) ($captured['desired_revision'] ?? 0) > 0
&& is_string($captured['desired_digest'] ?? null)
&& preg_match('/^[0-9a-f]{64}$/Di', $captured['desired_digest']) === 1;
}
public function tick(): bool
{
$this->observedErrors = [];
$job = (new CloudJob())->getConnection()->transaction(function () {
$query = CloudJob::whereIn('status', ['queued', 'running', 'uncertain'])
->where('available_at', '<=', time())->where('lease_until', '<=', time());
if ($this->policyOnly) {
$query->whereIn('kind', self::POLICY_KINDS);
} else {
// The regular worker remains a fail-safe policy consumer when
// the dedicated workers are not installed, but always claims
// target-state policy ahead of slow image/provisioning jobs.
$quoted = "'" . implode("','", self::POLICY_KINDS) . "'";
$query->orderByRaw("CASE WHEN kind IN (" . $quoted
. ") THEN 0 WHEN kind = 'expiry_delete' THEN 1 ELSE 2 END");
}
$job = $query->orderBy('id')->lockForUpdate()->first();
if ($job === null) {
return null;
}
$job->lease_token = bin2hex(random_bytes(16));
$job->lease_until = time() + (in_array($job->kind, self::POLICY_KINDS, true)
? 150 : self::LEASE_SECONDS);
$job->attempts++;
$job->save();
return $job;
}, 3);
if ($job === null) {
return false;
}
$payload = [];
$dispatched = $job->dispatched_at !== null;
try {
$payload = CloudSecrets::decrypt($job->payload_cipher);
$policyKind = in_array($job->kind, self::POLICY_KINDS, true);
$systemPolicy = ($payload['system_policy'] ?? false) === true
&& $policyKind && self::validSystemPolicy($job, $payload);
// Expiry deletion is authorized by the server-side lifecycle
// daemon, but remains a destructive one-shot mutation. Keep it
// separate from replayable target-state policy so a lost response
// can never cause PHP to submit DELETE a second time.
$systemCleanup = $job->kind === 'expiry_delete'
&& ($payload['system_cleanup'] ?? false) === true
&& self::validSystemCleanup($job, $payload);
if (($policyKind && !$systemPolicy)
|| ($job->kind === 'expiry_delete' && !$systemCleanup)) {
throw new CloudError('系统策略任务与实例状态不匹配', 409);
}
$actor = ['user_id' => (int) $job->user_id, 'is_admin' => false, 'is_super_admin' => false,
'can_manage' => false, 'session_id' => '', 'operation_id' => $job->operation_id];
$user = User::find($job->user_id);
if (!$systemPolicy && !$systemCleanup
&& ($user === null || (int) $user->enable !== 1)) {
if ($job->dispatched_at === null) {
$this->finish($job, ['status' => 'failed', 'safe_to_refund' => true], $payload);
return true;
}
}
if ($job->kind === 'create') {
$actor['can_provision'] = true;
$actor['can_manage'] = true;
} elseif ($systemPolicy || $systemCleanup) {
$actor['can_manage'] = true;
if ($systemCleanup) {
$actor['system_cleanup'] = true;
}
} elseif (($payload['can_manage'] ?? false) === true && $user !== null
&& (bool) $user->is_admin && in_array((int) $user->id,
array_map('intval', explode(',', (string) \App\Services\Config::get('super_admin'))), true)) {
$actor['is_admin'] = $actor['is_super_admin'] = $actor['can_manage'] = true;
} elseif ((int) ($payload['owner_id'] ?? 0) === (int) $job->user_id) {
$actor['can_manage'] = true;
}
if ($job->kind !== 'create' && !$actor['can_manage'] && $job->dispatched_at === null) {
$this->finish($job, ['status' => 'failed'], $payload);
return true;
}
if (!CloudFirewallService::assertJobRunnable($job, $payload)) {
// A confirmed instance deletion dominates an overlapping
// tenant firewall request. Close the obsolete job without
// touching the deleted policy or contacting the engine.
$this->finish($job, [
'operation_id' => (string) $job->operation_id,
'status' => 'failed',
'error_code' => 'firewall_apply_superseded',
'error' => '实例已删除,防火墙请求不再适用',
], $payload);
return true;
}
if ($systemPolicy && $job->dispatched_at === null && $job->instance_id !== null) {
$instance = CloudInstance::find($job->instance_id);
$desired = self::policyDesired($instance, (string) $job->kind);
if (!$desired) {
// Even a no-op policy result crosses the instance
// lifecycle boundary: a stale/missing private readback
// must not be hidden by the early skip path.
$skipped = $this->applyPrivateLifecycleGate(
$job, ['status' => 'succeeded',
'result' => ['skipped' => true],
'_private_policy_skipped' => true],
$payload, $actor
);
$this->finish($job, $skipped, $payload);
return true;
}
}
if (!$systemPolicy && !$systemCleanup && $job->kind !== 'create'
&& $job->dispatched_at === null
&& $job->instance_id !== null) {
$instance = CloudInstance::find($job->instance_id);
$policyTransition = $instance !== null
&& (in_array((string) $instance->service_status,
['suspending_expired', 'resume_pending', 'resuming',
'suspend_failed', 'resume_failed', 'expired_grace',
'purge_stopping', 'purge_stop_failed',
'purge_deleting', 'purge_failed'], true)
|| in_array((string) $instance->network_status,
['blocking', 'blocking_expired', 'blocked_expired',
'restore_pending', 'restoring', 'block_failed',
'restore_failed'], true));
$globalOverride = $actor['is_super_admin'] && $actor['can_manage'];
if ($instance === null || ($job->kind !== 'delete' && !$globalOverride
&& ($policyTransition || !$instance->serviceAvailable()))) {
$this->finish($job, ['status' => 'failed',
'error' => '实例服务已到期或正在执行策略,不能执行该操作'], $payload);
return true;
}
}
if ($job->dispatched_at === null
&& in_array((string) $job->kind, ['delete', 'expiry_delete'], true)
&& $job->instance_id !== null) {
// Re-check the independent VPC ledger immediately before the
// irreversible engine request. This also protects legacy
// delete rows created before the enqueue admission latch.
$instance = CloudInstance::where('id', $job->instance_id)
->lockForUpdate()->first();
if ($instance === null || (string) $instance->status === 'deleted') {
throw new CloudError('虚拟机不存在,删除任务无法继续', 404);
}
if ($job->kind === 'delete') {
if ((string) $instance->status !== 'deleting') {
// Historical rows may predate the admission latch.
// Do not convert an active VM to deleting here: this
// check is outside the claim transaction and cannot
// serialize a concurrent VPC attach. Fail closed and
// require a fresh, explicitly admitted request.
throw new CloudError('旧删除任务缺少 VPC 删除门禁,请人工核查', 409);
}
CloudVpcService::assertInstanceCanBeDeleted($instance);
} else {
if ((string) $instance->status !== 'deleting'
|| (string) $instance->service_status !== 'purge_deleting') {
throw new CloudError('到期删除状态无效,请人工核查', 409);
}
CloudVpcService::assertInstanceCanBeDeleted($instance);
}
}
if ($job->dispatched_at !== null) {
// A crash after dispatch is ambiguous. Destructive/user jobs
// are never replayed. Policy lookups are explicitly read-only:
// the engine's ordinary GET reconciles target state and could
// otherwise apply a stale suspend before PHP rechecks that a
// new paid period now desires resume.
if ($systemPolicy) {
try {
$operation = $this->engine->request('GET',
'/internal/v1/operations/' . $job->operation_id
. '?reconcile=false',
$actor, null, $job->operation_id);
} catch (CloudError $error) {
if ($error->httpStatus !== 404) {
throw $error;
}
$operation = null;
}
$terminal = is_array($operation) && in_array(
(string) ($operation['status'] ?? ''), ['succeeded', 'failed'], true);
if (!$terminal) {
// Do not close or reverse a stale policy while its
// original engine executor can still touch libvirt.
// Keeping this job uncertain also prevents lifecycle
// from enqueueing the opposite target until the next
// reconciliation pass observes a terminal operation.
if (($operation['executor_active'] ?? false) === true) {
throw new CloudError('引擎仍在执行当前实例策略,请稍后核对', 503);
}
// Re-read desired state after the read-only lookup.
// Most recorded target states must converge before an
// opposite transition is sent. The retired expiry
// shutdown is different: grace now requires only link
// down, so never replay it after an online upgrade.
$current = $job->instance_id === null ? null
: CloudInstance::find($job->instance_id);
$desired = self::policyDesired($current, (string) $job->kind);
$retiredExpirySuspend = $job->kind === 'service_suspend'
&& ($payload['suspend_reason'] ?? null) === 'expired'
&& !$desired;
if (!$desired && ($operation === null || $retiredExpirySuspend)) {
$result = ['skipped' => true];
if ($job->kind === 'service_suspend'
&& ($operation['service_was_running_hint'] ?? false) === true) {
$result['service_was_running'] = true;
}
$skipped = $this->applyPrivateLifecycleGate(
$job, ['status' => 'succeeded', 'result' => $result,
'_private_policy_skipped' => true],
$payload, $actor
);
$this->finish($job, $skipped, $payload);
return true;
}
$operation = $this->engine->host((int) $job->host_id,
$payload['method'], $payload['resource'], $actor,
$payload['body'], $job->operation_id);
}
} elseif ($systemCleanup) {
// Ask the engine to reconcile its durable operation. The
// delete mutation itself is never replayed: ordinary
// ambiguous deletes become terminal failures, while a VM
// whose deletion was already proven may retry only the
// persisted public-IP claim cleanup. Missing or otherwise
// nonterminal records remain uncertain for inspection.
try {
$operation = $this->engine->request('GET',
'/internal/v1/operations/' . $job->operation_id,
$actor, null, $job->operation_id);
} catch (CloudError $error) {
if ($error->httpStatus === 404) {
throw new CloudError(
'引擎中缺少到期删除记录,必须人工核查宿主机', 503);
}
throw $error;
}
if (!in_array((string) ($operation['status'] ?? ''),
['succeeded', 'failed'], true)) {
throw new CloudError(
'到期删除结果不确定,必须人工核查宿主机', 503);
}
} else {
try {
$operation = $this->engine->request('GET',
'/internal/v1/operations/' . $job->operation_id,
$actor, null, $job->operation_id);
} catch (CloudError $error) {
if ($job->kind === 'create' || $error->httpStatus !== 404) {
throw $error;
}
// Never replay destructive/user mutations. An
// authoritative missing operation means there is no
// engine registry entry left to reconcile; terminate
// the local job visibly so it cannot block expiry.
$this->finish($job, [
'operation_id' => (string) $job->operation_id,
'status' => 'failed',
'error' => '引擎中已无此任务记录,请核查虚拟机状态后重试',
], $payload);
return true;
}
}
} else {
$updated = CloudJob::where('id', $job->id)->where('lease_token', $job->lease_token)
->whereNull('dispatched_at')->update(['dispatched_at' => time(), 'status' => 'running', 'updated_at' => time()]);
if ($updated !== 1) {
return true;
}
CloudFirewallService::markJobRunning($job);
$dispatched = true;
$operation = $this->engine->host((int) $job->host_id, $payload['method'],
$payload['resource'], $actor, $payload['body'], $job->operation_id);
}
if (isset($operation['uuid']) && !array_key_exists('result', $operation)) {
$operation = ['status' => $operation['status'] ?? 'succeeded',
'result' => $operation];
}
// Every terminal VM mutation gets the same private-network
// readback gate. The engine's normal VM payload contains only a
// domain/XML observation; an attached VPC additionally requires
// member identity, revision and digest evidence from the host
// data-plane reconciler. A missing or stale observation is kept
// uncertain before finish() can commit public/billing state.
$operation = $this->applyPrivateLifecycleGate(
$job, $operation, $payload, $actor
);
$this->finish($job, $operation, $payload);
} catch (\Throwable $error) {
// Diagnostics deliberately contain no raw payloads, credentials or upstream tracebacks.
$this->observedErrors[] = 'worker_tick_failed';
$diagnostic = self::exceptionDiagnostic($error, $dispatched,
(string) $job->kind, $job->order_id !== null, self::payloadAction($payload));
$log = 'cloud worker job ' . $job->id . ' failed: ' . get_class($error);
if ($error instanceof CloudError) {
$log .= ' status=' . $error->httpStatus . ' message=' . $diagnostic;
}
error_log($log);
if (!$dispatched) {
// Decryption/configuration/local validation failed before an
// engine request was sent, so create did not happen and an
// order refund is safe.
try {
$this->finish($job, ['status' => 'failed', 'safe_to_refund' => true,
'error' => $diagnostic], $payload);
return true;
} catch (\Throwable $finishError) {
error_log('cloud worker job ' . $job->id
. ' could not record pre-dispatch failure: ' . get_class($finishError));
}
}
$markedUncertain = CloudJob::where('id', $job->id)
->where('lease_token', $job->lease_token)->update([
'status' => 'uncertain', 'error_code' => $diagnostic, 'lease_until' => 0,
'lease_token' => null, 'available_at' => time() + 30, 'updated_at' => time()]);
// Losing the lease means another worker owns every subsequent
// state transition. Never mutate its order, reservation, or IPs.
if ($markedUncertain !== 1) {
return true;
}
if (in_array((string) $job->kind,
['snapshot_create', 'snapshot_restore', 'snapshot_delete',
'delete', 'expiry_delete'], true)) {
try {
CloudSnapshotService::markJobUncertain($job);
} catch (\Throwable $snapshotError) {
error_log('cloud worker snapshot catalog could not be marked uncertain: '
. get_class($snapshotError));
}
}
if (in_array((string) $job->kind,
['firewall_apply', 'delete', 'expiry_delete'], true)) {
try {
CloudFirewallService::markJobUncertain($job);
} catch (\Throwable $firewallError) {
error_log('cloud worker firewall state could not be marked uncertain: '
. get_class($firewallError));
}
}
if ($job->kind === 'network_change') {
try {
(new CloudJob())->getConnection()->transaction(
function () use ($job, $payload): void {
$instance = $job->instance_id === null ? null
: CloudInstance::where('id', $job->instance_id)
->lockForUpdate()->first();
$sourceIpId = filter_var(
$payload['source_public_ip_id'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$targetIpId = filter_var(
$payload['target_public_ip_id'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$networkAttachmentId = filter_var(
$payload['network_attachment_id'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$sourcePoolId = filter_var(
$payload['source_public_ip_pool_id'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
$targetPoolId = filter_var(
$payload['target_public_ip_pool_id'] ?? null,
FILTER_VALIDATE_INT,
['options' => ['min_range' => 1]]
);
if ($instance === null || $sourceIpId === false
|| $targetIpId === false
|| $networkAttachmentId === false
|| $sourcePoolId === false
|| $targetPoolId === false) {
throw new CloudError(
'公网 IP 变更任务缺少账本关联', 409
);
}
CloudPlacementService::finishNetworkChange(
$instance, (string) $job->operation_id,
(int) $sourceIpId, (int) $targetIpId,
(int) $networkAttachmentId,
(int) $sourcePoolId, (int) $targetPoolId,
'uncertain'
);
}, 3
);
} catch (\Throwable $ledgerError) {
error_log('cloud worker network ledger could not be marked uncertain: '
. get_class($ledgerError));
}
}
if ($job->order_id !== null) {
CloudOrder::where('id', $job->order_id)->whereNotIn('status', ['succeeded', 'failed'])
->update(['status' => 'uncertain', 'updated_at' => time()]);
try {
(new CloudOrder())->getConnection()->transaction(function () use ($job): void {
$order = CloudOrder::where('id', $job->order_id)
->lockForUpdate()->first();
if ($order !== null) {
CloudPlacementService::markUncertain($order);
}
}, 3);
} catch (\Throwable $reservationError) {
error_log('cloud worker reservation could not be marked uncertain: '
. get_class($reservationError));
}
}
}
return true;
}
private static function policyDesired(?CloudInstance $instance, string $kind): bool
{
if ($instance === null) {
return false;
}
$status = (string) $instance->status;
return match ($kind) {
'service_suspend' => in_array($status, ['active', 'deleting'], true)
&& (string) $instance->service_status === 'purge_stopping',
'service_resume' => $status === 'active'
&& in_array((string) $instance->service_status,
['resume_pending', 'resuming'], true),
'network_suspend' => $status === 'active'
&& in_array((string) $instance->network_status,
['blocking', 'blocking_expired'], true),
'network_resume' => $status === 'active'
&& in_array((string) $instance->network_status,
['restore_pending', 'restoring'], true),
default => false,
};
}
private function finish(CloudJob $claimed, array $operation, array $payload): void
{
$status = self::disposition($operation, $claimed->kind === 'create',
(string) $claimed->kind);
if (in_array($status, ['failed', 'uncertain'], true)) {
$this->observedErrors[] = $status === 'failed' ? 'task_failed' : 'task_uncertain';
}
$result = $operation['result'] ?? null;
(new CloudJob())->getConnection()->transaction(function () use ($claimed, $payload, &$status, &$result, &$operation) {
// All financial transactions acquire the wallet first, avoiding wallet/order deadlocks.
$user = User::where('id', $claimed->user_id)->lockForUpdate()->first();
$order = $claimed->order_id === null ? null : CloudOrder::where('id', $claimed->order_id)->lockForUpdate()->first();
$job = CloudJob::where('id', $claimed->id)->lockForUpdate()->first();
if ($job === null || $job->lease_token !== $claimed->lease_token
|| in_array($job->status, ['succeeded', 'failed', 'cancelled'], true)) {
return;
}
if ($status === 'succeeded' && $job->instance_id !== null) {
$capturedPrivateExpectation = $payload['private_expectation'] ?? null;
// VPC mutators lock VPC -> instance. Acquire the same order
// before checking the captured target so detach/attach cannot
// change a member between the readback and this commit.
$privateProbe = is_array($capturedPrivateExpectation)
? $capturedPrivateExpectation
: CloudVpcService::instancePrivateExpectation((int) $job->instance_id);
$vpcKeyToLock = is_string($privateProbe['vpc_id'] ?? null)
? (string) $privateProbe['vpc_id'] : '';
if (preg_match('/^[0-9a-f]{32}$/D', $vpcKeyToLock) === 1) {
CloudVpc::where('vpc_key', $vpcKeyToLock)
->lockForUpdate()->first();
}
$lockedInstance = CloudInstance::where('id', $job->instance_id)
->lockForUpdate()->first();
$privateTargetChanged = $lockedInstance === null;
if (!$privateTargetChanged && is_array($capturedPrivateExpectation)) {
$currentPrivateExpectation = CloudVpcService::instancePrivateExpectation(
(int) $lockedInstance->id
);
$privateTargetChanged = !CloudVpcService::privateExpectationMatches(
(int) $lockedInstance->id, $capturedPrivateExpectation
);
if ($privateTargetChanged
&& self::deleteTombstoneDetached(
(string) $job->kind,
$capturedPrivateExpectation,
$currentPrivateExpectation
)) {
// The control-plane member may have been finalized as
// deleted after this destructive job captured its
// removal tombstone. Only a confirmed host-side
// removed/absent proof can bridge that bookkeeping
// transition; a disabled ledger alone is insufficient.
$proof = is_array($operation['result']['private_state'] ?? null)
? $operation['result']['private_state'] : null;
$privateTargetChanged = !is_array($proof)
|| !is_string($lockedInstance->uuid)
|| !is_string($proof['vm_uuid'] ?? null)
|| !hash_equals(strtolower((string) $lockedInstance->uuid),
strtolower((string) $proof['vm_uuid']))
|| ($proof['required'] ?? false) !== true
|| ($proof['desired_state'] ?? null) !== 'removed'
|| !is_string($proof['vpc_id'] ?? null)
|| !hash_equals((string) $capturedPrivateExpectation['vpc_id'],
(string) $proof['vpc_id'])
|| !is_string($proof['member_id'] ?? null)
|| !hash_equals((string) $capturedPrivateExpectation['member_id'],
(string) $proof['member_id'])
|| !in_array($proof['state'] ?? null,
['disabled', 'removed'], true)
|| ($proof['readback'] ?? null) !== 'confirmed'
|| ($proof['link_state'] ?? null) !== 'absent'
|| (int) ($proof['desired_revision'] ?? 0) < 1
|| (int) ($proof['observed_revision'] ?? 0)
!== (int) ($proof['desired_revision'] ?? 0)
|| !is_string($proof['desired_digest'] ?? null)
|| !is_string($proof['observed_digest'] ?? null)
|| !hash_equals((string) $proof['desired_digest'],
(string) $proof['observed_digest']);
}
} elseif (!$privateTargetChanged) {
// Jobs created before the immutable expectation field was
// introduced cannot prove that an attached VPC target
// stayed unchanged. Allow legacy public-only jobs to
// finish, but fail closed for any live private member.
$privateTargetChanged = CloudVpcService::instancePrivateExpectation(
(int) $lockedInstance->id
)['required'] === true;
}
if ($privateTargetChanged) {
// The VPC target changed after the worker's engine
// readback. Keep the operation uncertain while holding
// the same instance row lock used by attach/detach.
$status = 'uncertain';
$operation['status'] = 'uncertain';
$operation['error_code'] = 'vpc_private_observation_mismatch';
$operation['error'] = '实例私网目标已变化,结果正在核对';
if (is_array($operation['result'] ?? null)) {
$operation['result']['private_state'] = [
'version' => 1, 'required' => true,
'state' => 'uncertain', 'desired_state' => 'active',
'readback' => 'mismatch',
'reason_code' => 'vpc_private_observation_mismatch',
'desired_revision' => 0, 'observed_revision' => 0,
'desired_digest' => null, 'observed_digest' => null,
'link_state' => 'absent', 'observed_at' => time(),
];
}
$result = $operation['result'] ?? null;
}
}
$safe = is_array($result)
? CloudOperationErrorCatalog::workerResult($result) : null;
CloudSnapshotService::finishJobLocked($job, $status, $result, $payload);
CloudSnapshotService::finishInstanceDeleteLocked($job, $status);
CloudFirewallService::finishJobLocked(
$job, $status, $result, $payload, $operation
);
CloudFirewallService::finishInstanceDeleteLocked($job, $status);
if ($job->kind === 'network_change'
&& in_array($status, ['succeeded', 'failed', 'uncertain'], true)) {
$instance = $job->instance_id === null ? null
: CloudInstance::where('id', $job->instance_id)
->lockForUpdate()->first();
$sourceIpId = filter_var($payload['source_public_ip_id'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$targetIpId = filter_var($payload['target_public_ip_id'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
$networkAttachmentId = filter_var(
$payload['network_attachment_id'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]
);
$sourcePoolId = filter_var(
$payload['source_public_ip_pool_id'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]
);
$targetPoolId = filter_var(
$payload['target_public_ip_pool_id'] ?? null,
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]
);
if ($instance === null || $sourceIpId === false
|| $targetIpId === false || $networkAttachmentId === false
|| $sourcePoolId === false || $targetPoolId === false) {
throw new CloudError('公网 IP 变更任务缺少账本关联', 409);
}
if ($status === 'succeeded') {
$requested = is_array($payload['body'] ?? null)
? ($payload['body']['public_ip_cidr'] ?? null) : null;
$requestedPoolKey = is_array($payload['body'] ?? null)
? ($payload['body']['public_pool_key'] ?? null) : null;
$confirmed = is_array($result)
? ($result['public_ip_cidr'] ?? null) : null;
$confirmedPoolKey = is_array($result)
? ($result['public_pool_key'] ?? null) : null;
if (!is_string($requested) || !is_string($confirmed)
|| !is_string($requestedPoolKey)
|| !is_string($confirmedPoolKey)
|| $requested !== $confirmed
|| !hash_equals($requestedPoolKey, $confirmedPoolKey)) {
throw new CloudError(
'引擎成功结果缺少已确认的公网 IP 或地址池', 502
);
}
$decoded = json_decode((string) $instance->spec_json, true);
if (!is_array($decoded) || array_is_list($decoded)) {
throw new CloudError('虚拟机配置记录损坏,无法保存新的公网 IP', 409);
}
CloudPlacementService::finishNetworkChange($instance,
(string) $job->operation_id, (int) $sourceIpId,
(int) $targetIpId, (int) $networkAttachmentId,
(int) $sourcePoolId, (int) $targetPoolId, 'succeeded');
$decoded['public_ip_cidr'] = $confirmed;
$instance->spec_json = json_encode($decoded,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE);
self::recordRuntime($instance, is_array($result) ? $result : [],
(string) $job->kind, $payload);
$instance->updated_at = time();
$instance->save();
} else {
CloudPlacementService::finishNetworkChange($instance,
(string) $job->operation_id, (int) $sourceIpId,
(int) $targetIpId, (int) $networkAttachmentId,
(int) $sourcePoolId, (int) $targetPoolId, $status);
}
} elseif ($status === 'succeeded' && $job->kind === 'create') {
$vm = is_array($result) ? ($result['vm'] ?? $result) : [];
$uuid = $vm['uuid'] ?? '';
if ($order === null || !is_string($uuid)
|| preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD', $uuid) !== 1) {
throw new CloudError('引擎成功结果缺少虚拟机标识', 502);
}
$requestedPublicIpCidr = is_array($payload['body'] ?? null)
? ($payload['body']['public_ip_cidr'] ?? null) : null;
$requestedPublicPoolKey = is_array($payload['body'] ?? null)
? ($payload['body']['public_pool_key'] ?? null) : null;
$confirmedPublicIpCidr = $vm['public_ip_cidr'] ?? null;
$confirmedPublicPoolKey = $vm['public_pool_key'] ?? null;
// Jobs queued before the multi-pool rollout have no key; the
// engine's unique-IP inference remains their compatibility
// path. Every newly queued placement includes a key and must
// receive the same identity back before PHP commits it.
$poolIdentityMatches = $requestedPublicPoolKey === null
|| (is_string($requestedPublicPoolKey)
&& is_string($confirmedPublicPoolKey)
&& hash_equals($requestedPublicPoolKey,
$confirmedPublicPoolKey));
if (!is_string($requestedPublicIpCidr)
|| !is_string($confirmedPublicIpCidr)
|| !hash_equals($requestedPublicIpCidr,
$confirmedPublicIpCidr)
|| !$poolIdentityMatches) {
// A recovered create result may describe a VM whose
// network was changed out of band after the original
// response was lost. Never commit the original address in
// that case: keep the order and reservation uncertain for
// an administrator to reconcile.
throw new CloudError(
'引擎成功结果与预留公网 IP 不一致,必须人工核查', 502
);
}
$instance = CloudInstance::where('order_id', $order->id)->first();
$newInstance = $instance === null;
$instance ??= new CloudInstance();
$instance->uuid = $uuid;
$instance->host_id = $order->host_id;
$instance->user_id = $order->user_id;
$instance->order_id = $order->id;
$instance->offer_id = $order->offer_id;
$instance->resource_pool_id = $order->resource_pool_id;
$instance->reservation_id = $order->reservation_id;
$spec = json_decode((string) $order->spec_json, true) ?: [];
$now = time();
$billingSnapshot = array_key_exists('validity_days', $spec)
&& array_key_exists('traffic_quota_bytes', $spec)
&& (int) $spec['validity_days'] >= 1;
$days = $billingSnapshot ? (int) $spec['validity_days'] : 0;
$traffic = $billingSnapshot
? max(0, (int) $spec['traffic_quota_bytes']) : 0;
$instance->plan_id = $order->plan_id;
// A late/replayed create result must not overwrite a display
// name the owner changed after the first success committed.
if ($newInstance) {
$instance->name = $payload['body']['name'];
}
$instance->status = 'active';
// Orders submitted by a pre-billing release do not contain an
// immutable term/quota snapshot. Never invent a 30-day expiry
// while such an order is in flight during upgrade; preserve
// its historical unlimited/exempt behavior instead.
$instance->service_status = $billingSnapshot ? 'active' : 'exempt';
$instance->network_status = 'active';
$instance->expires_at = $billingSnapshot ? $now + $days * 86400 : null;
$instance->renewal_price = $billingSnapshot ? $order->amount : null;
$instance->renewal_days = $days;
$instance->renewal_traffic_bytes = $traffic;
$instance->traffic_quota_bytes = $traffic;
$instance->traffic_download_bytes = 0;
$instance->traffic_upload_bytes = 0;
$instance->lifetime_download_bytes = 0;
$instance->lifetime_upload_bytes = 0;
$instance->traffic_synced_at = 0;
// Persist only the engine's slow-changing public metadata.
// SSR pages can then render IP, operating system, login name
// and actual capacity without issuing a full libvirt/XML read
// on every request. Billing/order fields remain authoritative
// because the allowlist cannot overwrite them.
$instance->spec_json = json_encode(
CloudInstanceMetadata::merge($spec, $vm),
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE
);
$instance->credential_cipher = isset($payload['credential']) ? CloudSecrets::encrypt($payload['credential']) : null;
// The paid service begins when provisioning actually succeeds,
// not while an opening invoice is waiting for payment. A
// reconciled/replayed success must retain the original value.
if (!$instance->exists) {
$instance->created_at = $now;
}
$instance->updated_at = time();
$instance->save();
CloudPlacementService::commit(
$order, (int) $instance->id, $confirmedPublicIpCidr
);
// Seed the independent runtime observation immediately after
// provisioning. If the engine returned a trusted power
// state, promote it now; otherwise the background host batch
// will provide the first observation without showing "未知".
self::recordRuntime($instance, $vm, 'create', []);
if ($billingSnapshot) {
$period = CloudServicePeriod::where('order_id', $order->id)->first()
?? new CloudServicePeriod();
$period->instance_id = $instance->id;
$period->user_id = $order->user_id;
$period->plan_id = $order->plan_id;
$period->order_id = $order->id;
$period->starts_at = $now;
$period->ends_at = $instance->expires_at;
$period->traffic_quota_bytes = $traffic;
$period->download_base_bytes = 0;
$period->upload_base_bytes = 0;
$period->download_used_bytes = 0;
$period->upload_used_bytes = 0;
$period->status = 'active';
$period->activated_at = $now;
$period->created_at = $period->created_at ?: $now;
$period->updated_at = $now;
$period->save();
$instance->current_period_id = $period->id;
$instance->save();
}
$job->instance_id = $order->instance_id = $instance->id;
$provisionInvoice = CloudInvoice::where('order_id', $order->id)
->lockForUpdate()->first();
if ($provisionInvoice !== null
&& (string) $provisionInvoice->purpose === 'provision') {
$provisionInvoice->instance_id = $instance->id;
$provisionInvoice->updated_at = time();
$provisionInvoice->save();
CloudInvoiceService::queueProvisionSucceeded(
$provisionInvoice, $now
);
}
$safe = $safe ?? [];
$safe['id'] = (string) $instance->id;
$safe['uuid'] = $uuid;
} elseif ($status === 'failed' && $order !== null && $order->refunded_at === null) {
if ($user === null) {
throw new CloudError('退款账户不存在', 409);
}
$user->money = CloudMoney::decimal(CloudMoney::cents((string) $user->money) + CloudMoney::cents($order->amount));
$user->save();
$invoice = CloudInvoice::where('order_id', $order->id)
->lockForUpdate()->first();
// Payment placement locks offer before host. Keep the same
// cross-resource order during refunds so a failed creation on
// one order cannot deadlock a concurrent payment for the same
// offer and host (offer -> host -> reservation -> IP).
CloudOrderService::releaseStock($order);
CloudPlacementService::release($order, 'create_failed');
if ($invoice !== null && (string) $invoice->purpose === 'provision') {
$invoice->status = 'refunded';
$invoice->updated_at = time();
$invoice->save();
}
$order->refunded_at = time();
} elseif ($status === 'failed' && $job->kind === 'expiry_delete'
&& $job->instance_id !== null) {
$instance = CloudInstance::where('id', $job->instance_id)
->lockForUpdate()->first();
if ($instance !== null && (string) $instance->status !== 'deleted') {
$instance->status = 'delete_failed';
$instance->service_status = 'purge_failed';
$instance->updated_at = time();
$instance->save();
}
} elseif ($status === 'failed' && $job->kind === 'delete'
&& $job->instance_id !== null) {
$instance = CloudInstance::where('id', $job->instance_id)
->lockForUpdate()->first();
if ($instance !== null && (string) $instance->status === 'deleting') {
// Keep a failed destructive request visible and block
// new VM/VPC mutations until an operator reconciles it.
$instance->status = 'delete_failed';
$instance->updated_at = time();
$instance->save();
}
} elseif ($status === 'failed' && $job->instance_id !== null
&& in_array($job->kind,
['service_suspend', 'service_resume', 'network_suspend', 'network_resume'], true)) {
// Record a visible policy error. The lifecycle reconciler
// retries this desired state with bounded exponential backoff
// after a temporary host/SSH failure is repaired.
$instance = CloudInstance::where('id', $job->instance_id)->lockForUpdate()->first();
if ($instance !== null) {
$expected = [
'service_suspend' => ['suspending_expired', 'purge_stopping'],
'service_resume' => ['resume_pending', 'resuming'],
'network_suspend' => ['blocking', 'blocking_expired'],
'network_resume' => ['restore_pending', 'restoring'],
][$job->kind];
if ($job->kind === 'service_suspend'
&& in_array((string) $instance->service_status, $expected, true)) {
if ((string) $instance->status === 'deleting'
&& (string) $instance->service_status === 'purge_stopping') {
// Stopping is a replayable target state. Leave the
// irreversible deletion latch in place while the
// lifecycle daemon applies its bounded retry.
$instance->service_status = 'purge_stop_failed';
} else {
$instance->service_status = 'suspend_failed';
}
} elseif ($job->kind === 'service_resume'
&& in_array((string) $instance->service_status, $expected, true)) {
$instance->service_status = 'resume_failed';
} elseif ($job->kind === 'network_suspend'
&& in_array((string) $instance->network_status, $expected, true)) {
$instance->network_status = 'block_failed';
} elseif ($job->kind === 'network_resume'
&& in_array((string) $instance->network_status, $expected, true)) {
$instance->network_status = 'restore_failed';
}
$instance->updated_at = time();
$instance->save();
}
} elseif ($job->instance_id !== null
&& ($status === 'succeeded'
|| ($status === 'uncertain' && $job->kind === 'reinstall'
&& self::reinstallNeedsReview($operation)))) {
$instance = CloudInstance::where('id', $job->instance_id)->lockForUpdate()->first();
if ($instance !== null) {
if (in_array((string) $job->kind, ['delete', 'expiry_delete'], true)) {
// VPC admission is checked while the destructive
// request is still undispatched. Once the engine has
// returned a terminal result, do not turn a confirmed
// host deletion into a second ambiguous operation;
// any legacy ledger mismatch remains visible through
// the independent VPC projection for reconciliation.
$instance->status = 'deleted';
$instance->credential_cipher = null;
if ($instance->order_id !== null) {
$provisionOrder = CloudOrder::where('id', $instance->order_id)
->lockForUpdate()->first();
if ($provisionOrder !== null) {
CloudPlacementService::release(
$provisionOrder, 'instance_deleted', true
);
}
}
} elseif ($job->kind === 'reinstall') {
if (isset($payload['credential'])
&& is_array($payload['credential'])) {
$instance->credential_cipher = CloudSecrets::encrypt(
$payload['credential']
);
}
// The replacement template changes the static OS
// metadata even though IP and disk capacity remain
// stable. Invalidate only those cached fields so the
// next detail render performs one bounded engine
// metadata read and persists the new system label.
$decoded = json_decode((string) $instance->spec_json, true);
if (is_array($decoded) && !array_is_list($decoded)) {
$templateId = is_array($payload['body'] ?? null)
? ($payload['body']['template_id'] ?? null) : null;
if (is_string($templateId)
&& preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D',
$templateId) === 1) {
$decoded['template_id'] = $templateId;
}
unset($decoded['os_source'], $decoded['os_name'],
$decoded['install_iso'], $decoded['login_name'],
$decoded['engine_metadata_version']);
$instance->spec_json = json_encode($decoded,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE);
}
} elseif ($job->kind === 'instance_settings') {
// The engine must acknowledge the controlled seed
// replacement before PHP changes any cached metadata.
// Passwords and public keys are intentionally never
// copied into spec_json or result_json.
if (!is_array($result) || ($result['settings_changed'] ?? false) !== true) {
throw new CloudError('引擎成功结果缺少已确认的实例设置', 502);
}
if (isset($payload['credential'])
&& is_array($payload['credential'])) {
$instance->credential_cipher = CloudSecrets::encrypt(
$payload['credential']
);
}
$requested = is_array($payload['body'] ?? null)
? $payload['body'] : [];
$decoded = json_decode((string) $instance->spec_json, true);
if (!is_array($decoded) || array_is_list($decoded)) {
throw new CloudError('虚拟机配置记录损坏,无法保存实例设置', 409);
}
if (array_key_exists('hostname', $requested)) {
$confirmed = $result['hostname'] ?? null;
if (!is_string($confirmed)
|| $confirmed !== $requested['hostname']
|| preg_match('/^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*$/D', $confirmed) !== 1
|| strlen($confirmed) > 253) {
throw new CloudError('引擎成功结果缺少已确认的主机名', 502);
}
$decoded['guest_hostname'] = $confirmed;
}
$instance->spec_json = json_encode($decoded,
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
| JSON_UNESCAPED_UNICODE);
} elseif ($job->kind === 'service_suspend'
&& in_array((string) $instance->service_status,
['suspending_expired', 'resume_pending', 'resuming',
'expired_grace', 'purge_stopping'], true)) {
// Preserve proof that the retired expiry policy
// touched a previously running guest. Network must be
// blocked before the lifecycle daemon safely restarts
// it; a tenant-stopped guest has no marker and stays
// stopped.
if (($result['service_was_running'] ?? false) === true
&& (string) $instance->service_status !== 'purge_stopping') {
$instance->suspended_at = time();
}
if ((string) $instance->status === 'deleting'
&& (string) $instance->service_status === 'purge_stopping') {
$instance->service_status = 'purge_deleting';
} elseif (($payload['suspend_reason'] ?? null) === 'expired') {
$expired = $instance->expires_at !== null
&& (int) $instance->expires_at <= time();
$instance->service_status = $instance->suspended_at !== null
? 'resume_pending' : ($expired ? 'expired_grace' : 'active');
} elseif ($instance->service_status === 'suspending_expired') {
$instance->service_status = 'suspended_expired';
}
} elseif ($job->kind === 'service_resume'
&& in_array((string) $instance->service_status, ['resuming', 'resume_pending'], true)) {
$instance->service_status = $instance->expires_at !== null
&& (int) $instance->expires_at <= time()
? 'expired_grace' : 'active';
$instance->suspended_at = null;
} elseif ($job->kind === 'network_suspend'
&& in_array((string) $instance->network_status,
['blocking', 'blocking_expired'], true)) {
$instance->network_status = ($payload['suspend_reason'] ?? '') === 'expired'
? 'blocked_expired' : 'blocked_traffic';
$instance->network_suspended_at = time();
} elseif ($job->kind === 'network_resume'
&& in_array((string) $instance->network_status, ['restoring', 'restore_pending'], true)) {
$instance->network_status = 'active';
$instance->network_suspended_at = null;
}
// Successful power/reinstall/policy operations often
// include the resulting VM state. Persist that bounded
// observation so the next page render is immediate;
// missing state is deliberately left for vm-states.
self::recordRuntime($instance, is_array($result) ? $result : [],
(string) $job->kind, $payload);
$instance->updated_at = time();
$instance->save();
}
}
$job->status = $status;
if (in_array($status, ['succeeded', 'failed'], true)) {
// Terminal jobs never need credentials again. Preserve only
// a sanitized audit payload so an old create/reinstall row
// cannot remain a second recoverable copy of the root password.
$job->payload_cipher = CloudSecrets::encrypt(
CloudInput::safeResult($payload)
);
}
$job->result_json = $safe === null ? null : json_encode($safe, JSON_THROW_ON_ERROR);
$job->error_code = CloudOperationErrorCatalog::workerSummary(
(string) $job->kind, $status, $operation, $order !== null,
self::payloadAction($payload)
);
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = time() + 15;
$job->updated_at = time();
$job->save();
if ($order !== null) {
if ($job->kind === 'create' && $status === 'uncertain') {
CloudPlacementService::markUncertain($order);
}
$order->status = $status;
$order->updated_at = time();
$order->save();
}
}, 3);
}
/**
* Failed/uncertain jobs normally must keep their original disposition,
* but a job that crossed the engine mutation boundary still needs the
* same private-state readback as a successful job. The durable engine
* marker avoids treating ordinary request validation failures as if they
* had changed a VM. Network changes retain their narrower checkpoint:
* the child must have claimed the target IP before this gate runs.
*
* The actual bounded private-state read happens outside finish()'s DB
* transaction below, so a slow or failed engine read never holds wallet or
* VPC rows.
*/
private static function privateMutationReadbackRequired(
CloudJob $job, array $operation, string $status
): bool {
if (!in_array($status, ['failed', 'uncertain'], true)) {
return false;
}
$kind = (string) $job->kind;
$result = is_array($operation['result'] ?? null)
? $operation['result'] : [];
$vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
// Public-IP changes already have a commit checkpoint inside the
// child. A generic invocation marker alone would gate a preflight
// rejection before any address or XML mutation took place.
if ($kind === 'network_change') {
if (($operation['network_mutation_started'] ?? false) === true) {
return true;
}
foreach (['network_changed', 'network_mutation_started'] as $key) {
if (($result[$key] ?? false) === true || ($vm[$key] ?? false) === true) {
return true;
}
}
return false;
}
if (($operation['mutation_started'] ?? false) === true
|| ($operation['network_mutation_started'] ?? false) === true) {
return true;
}
if (in_array($kind, ['delete', 'expiry_delete'], true)
&& (($operation['vm_delete_committed'] ?? false) === true
|| array_key_exists('cleanup_pending', $operation))) {
return true;
}
if ($kind === 'reinstall' && self::reinstallNeedsReview($operation)) {
return true;
}
// Preserve existing post-commit markers from older engine versions.
foreach (['reinstall_committed', 'cleanup_pending', 'network_changed',
'settings_changed', 'restored', 'deleted', 'created'] as $key) {
if (($result[$key] ?? false) === true || ($vm[$key] ?? false) === true) {
return true;
}
}
// An uncertain job dispatched by a legacy engine has no durable
// mutation marker. Its dispatched_at timestamp is the conservative
// compatibility signal; failed pre-dispatch jobs keep a false gate.
return $status === 'uncertain' && $job->dispatched_at !== null;
}
private function applyPrivateLifecycleGate(
CloudJob $job, array $operation, array $payload, array $actor
): array {
$status = is_string($operation['status'] ?? null)
? strtolower((string) $operation['status']) : 'uncertain';
$kind = (string) $job->kind;
$lifecycleKind = $kind;
$payloadBody = is_array($payload['body'] ?? null)
? $payload['body'] : [];
$powerAction = null;
if ($kind === 'action') {
// The generic job kind is not enough to decide whether a stopped
// private-state proof is an expected postcondition. Bind that
// decision to the validated encrypted request payload; never infer
// it from an engine result or from the VM's current state.
$action = is_string($payloadBody['action'] ?? null)
? (string) $payloadBody['action'] : '';
if (CloudOperationCatalog::isPowerAction($action)) {
$powerAction = $action;
}
}
$postCommitReinstall = $kind === 'reinstall'
&& $status === 'failed' && self::reinstallNeedsReview($operation);
$terminalSuccess = in_array($status, ['succeeded', 'completed', 'done'], true);
if (!$terminalSuccess && !$postCommitReinstall
&& !self::privateMutationReadbackRequired($job, $operation, $status)) {
return $operation;
}
$lifecycleKinds = [
'create', 'reinstall', 'network_change', 'instance_settings', 'rescue',
'snapshot_create', 'snapshot_restore', 'snapshot_delete', 'delete',
'expiry_delete', 'action', 'service_suspend', 'service_resume',
'network_suspend', 'network_resume', 'firewall_apply',
];
if (!in_array($kind, $lifecycleKinds, true)) {
return $operation;
}
$instance = $job->instance_id === null ? null : CloudInstance::find($job->instance_id);
// Power operations preserve the independently managed network-policy
// target. Bind private readback to the trusted database state instead
// of treating every successful start/reboot as permission to raise a
// quota- or expiry-blocked link. Transitional/failed policy states are
// deliberately unprovable until their own policy job converges.
$networkPolicy = match ((string) ($instance?->network_status ?? '')) {
'active' => 'active',
'blocked_traffic', 'blocked_expired' => 'blocked',
default => 'unready',
};
if ($powerAction !== null) {
$lifecycleKind = 'action_' . $powerAction
. '_policy_' . $networkPolicy;
} elseif ($kind === 'service_suspend') {
$lifecycleKind = 'service_suspend_policy_' . $networkPolicy;
} elseif ($kind === 'service_resume') {
// The resume endpoint defaults false, but the private-state gate
// accepts a stopped postcondition only when that target was
// explicitly captured in the encrypted job payload. A missing or
// non-boolean target remains fail-closed.
$resumeTarget = array_key_exists('start_if_suspended', $payloadBody)
&& is_bool($payloadBody['start_if_suspended'])
? ($payloadBody['start_if_suspended'] ? 'start' : 'stopped')
: 'unbound';
$lifecycleKind = 'service_resume_' . $resumeTarget
. '_policy_' . $networkPolicy;
}
$currentExpectation = CloudVpcService::instancePrivateExpectation(
$instance === null ? null : (int) $instance->id
);
$capturedExpectation = is_array($payload['private_expectation'] ?? null)
? $payload['private_expectation'] : null;
$deleteTombstoneDetached = self::deleteTombstoneDetached(
$kind, $capturedExpectation, $currentExpectation
);
$expectationStale = false;
$expectation = $currentExpectation;
if ($capturedExpectation !== null) {
$expectation = $capturedExpectation;
// `state` is an observed/transient lifecycle value. A member can
// legitimately move applying -> active while this VM job is in
// flight; desired_state plus the revision/digest still bind the
// immutable target. Removal is represented by desired_state and
// therefore remains strict.
foreach (['required', 'desired_state', 'vpc_id', 'member_id',
'private_ip_cidr', 'private_mac', 'desired_revision', 'desired_digest'] as $key) {
$left = $capturedExpectation[$key] ?? null;
$right = $currentExpectation[$key] ?? null;
if ((string) $left !== (string) $right) {
$expectationStale = true;
break;
}
}
if ($expectationStale && !$deleteTombstoneDetached) {
// Keep the original target in the evidence, but make the
// gate fail closed instead of validating it against a newer
// VPC revision that arrived while this job was running.
$expectation['required'] = true;
$expectation['reason_code'] = 'vpc_private_observation_mismatch';
} elseif ($deleteTombstoneDetached) {
// The captured removed tuple is the deliberate target for
// this destructive job. Treat the disabled current row as a
// finalized control-plane tombstone, while retaining the
// captured tuple for host-side validation.
$expectationStale = false;
}
}
$result = is_array($operation['result'] ?? null) ? $operation['result'] : [];
$vm = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
$uuid = is_string($instance?->uuid) ? (string) $instance->uuid
: (is_string($vm['uuid'] ?? null) ? $vm['uuid']
: (is_string($result['uuid'] ?? null) ? $result['uuid'] : null));
// Accept both the normal {vm:{...}} envelope and a rolling-upgrade
// response that places the bounded observation beside `vm`. A
// structured field at either level must stay strict; only a complete
// omission can use the legacy public-only compatibility path.
$hasPrivateState = array_key_exists('private_state', $vm)
|| array_key_exists('private_state', $result);
$observation = is_array($vm['private_state'] ?? null)
? $vm['private_state']
: (is_array($result['private_state'] ?? null)
? $result['private_state'] : null);
// Pre-VPC engines and the existing public-only worker fixtures do not
// return a private_state object at all. There is no private target to
// prove for those jobs, so do not issue a second host request (which
// would also consume the next response in a legacy mock queue). Once
// a VPC member is expected, the gate remains fail-closed below; and a
// structured observation supplied by any engine is still validated,
// including the public-only case where it can reveal an unexpected
// private NIC.
if (($expectation['required'] ?? false) !== true && !$hasPrivateState) {
return $operation;
}
// An attached VPC must use the mutating host adapter below as the
// authoritative observation. Do not accept a cached/XML-only GET
// from the same lifecycle boundary, even if its revision happens to
// look current. Public-only jobs retain the bounded GET path.
$privateReconcileTarget = ($expectation['required'] ?? false) === true
&& $uuid !== null && (!$expectationStale || $deleteTombstoneDetached);
// A delete may legitimately have no readable domain after the
// irreversible boundary. Only a VM without a VPC target can use the
// explicit disabled/confirmed result; a VPC tombstone is reconciled
// by the host adapter and still requires a removed/readback proof.
if ($uuid !== null && !$privateReconcileTarget) {
try {
$observed = $this->engine->host(
(int) $job->host_id, 'GET',
'vms/' . rawurlencode($uuid) . '/private-state',
$actor, null, (string) $job->operation_id
);
if (is_array($observed)) {
$observation = $observed;
}
} catch (\Throwable $error) {
$absenceProven = in_array($kind, ['delete', 'expiry_delete'], true)
&& ($expectation['required'] ?? false) !== true
&& $error instanceof CloudError
&& $error->httpStatus === 404;
if ($absenceProven) {
// A VM with no VPC member has no independent private
// teardown target. Once the engine has confirmed the
// domain deletion, a 404 is the expected absence proof.
$observation = [
'version' => 1, 'required' => false,
'vm_uuid' => $uuid, 'vpc_id' => null, 'member_id' => null,
'interface' => null, 'state' => 'disabled',
'desired_revision' => 0, 'observed_revision' => 0,
'desired_digest' => null, 'observed_digest' => null,
'readback' => 'confirmed', 'link_state' => 'absent',
'private_ip_cidr' => null, 'private_mac' => null,
'reason_code' => null, 'observed_at' => time(),
];
}
if (!$absenceProven) {
// The operation itself may already be durable. Convert a
// missing observation into a stable uncertain result
// rather than leaking a transport/host diagnostic.
$observation = null;
}
}
}
$validation = $expectationStale
? ['ok' => false, 'state' => [
'version' => 1, 'required' => true,
'vm_uuid' => $uuid, 'vpc_id' => $expectation['vpc_id'] ?? null,
'member_id' => $expectation['member_id'] ?? null,
'interface' => null, 'state' => 'uncertain',
'desired_state' => $expectation['desired_state'] ?? 'active',
'desired_revision' => (int) ($expectation['desired_revision'] ?? 0),
'observed_revision' => 0,
'desired_digest' => $expectation['desired_digest'] ?? null,
'observed_digest' => null, 'readback' => 'mismatch',
'link_state' => 'absent', 'private_ip_cidr' => null,
'private_mac' => null,
'reason_code' => 'vpc_private_observation_mismatch',
'observed_at' => time(),
], 'error_code' => 'vpc_private_observation_mismatch']
: CloudVpcService::validatePrivateObservation(
$expectation, $observation, $lifecycleKind
);
// A normal private-state GET is intentionally an observation only. A
// lifecycle mutation can have replaced the domain XML (or removed a
// domain altogether) while the host bridge, routes, WireGuard peer,
// and nftables state still need to be reconciled. Ask the engine's
// host adapter to perform that bounded apply/readback for every
// attached VPC target. Its response is authoritative: a failed or
// unsupported reconcile becomes an empty observation and therefore
// stays uncertain instead of falling back to the GET result.
if ($privateReconcileTarget) {
// Only the PHP policy short-circuit carries this private marker.
// An engine result may also expose `skipped` after it has already
// proved a target state; that must retain network_suspend/down or
// network_resume/up semantics rather than weakening the gate. A
// skipped target uses a dedicated preserve validator that accepts
// stopped only with exact persistent configuration proof.
$privateReconcileSkipped = ($operation['_private_policy_skipped'] ?? false) === true;
$reconciled = $this->reconcilePrivateState(
$job, $uuid, $lifecycleKind, $status, $expectation, $actor,
$privateReconcileSkipped
);
$validation = CloudVpcService::validatePrivateObservation(
$expectation, $reconciled,
$privateReconcileSkipped ? 'policy_skipped' : $lifecycleKind,
$uuid
);
} elseif (($expectation['required'] ?? false) === true
&& $uuid === null && !$expectationStale) {
// An attached target without a bound VM UUID cannot be proved by
// an embedded operation payload. Keep it uncertain until the
// worker can address the host-side reconcile endpoint explicitly.
$validation = CloudVpcService::validatePrivateObservation(
$expectation, null, $lifecycleKind
);
}
if ($instance !== null && ($expectation['required'] ?? false) === true
&& !$expectationStale && !$deleteTombstoneDetached) {
try {
$recorded = CloudVpcService::recordPrivateObservation(
(int) $instance->id, $validation, (string) $job->operation_id
);
if ($recorded !== true && $validation['ok'] === true) {
$validation = [
'ok' => false,
'state' => array_merge($validation['state'], [
'state' => 'uncertain', 'readback' => 'mismatch',
'reason_code' => 'vpc_private_observation_mismatch',
'observed_at' => time(),
]),
'error_code' => 'vpc_private_observation_mismatch',
];
}
} catch (\Throwable $error) {
// The operation gate remains uncertain even when the
// secondary VPC observation cache cannot be written.
error_log('cloud private observation could not be recorded: '
. get_class($error));
if ($validation['ok'] === true) {
$validation = [
'ok' => false,
'state' => array_merge($validation['state'], [
'state' => 'uncertain', 'readback' => 'missing',
'reason_code' => 'vpc_private_observation_missing',
'observed_at' => time(),
]),
'error_code' => 'vpc_private_observation_missing',
];
}
}
}
$result['private_state'] = $validation['state'];
if ($kind === 'reinstall' && !$validation['ok']) {
// Preserve the replacement credential/spec on a post-commit
// uncertain reinstall; the existing finish() compatibility path
// will not mark the task successful or replay it blindly.
$result['reinstall_committed'] = true;
}
$operation['result'] = $result;
if (!$validation['ok']) {
$operation['status'] = 'uncertain';
$operation['safe_to_refund'] = false;
$operation['error_code'] = $validation['error_code']
?? 'vpc_private_observation_missing';
$operation['error'] = '实例私网状态尚未完成实际回读,请稍后核对';
}
return $operation;
}
/**
* Request one host-side private data-plane reconcile and return only its
* bounded VM observation. The engine endpoint is deliberately separate
* from the ordinary GET: XML alone cannot prove the independent bridge,
* route, WireGuard and nftables state.
*/
private function reconcilePrivateState(
CloudJob $job, string $uuid, string $kind, string $status,
array $expectation, array $actor, bool $skipped = false
): ?array {
$body = [
'version' => 1,
'lifecycle_kind' => $kind,
'required' => true,
'desired_state' => (string) ($expectation['desired_state'] ?? 'active'),
'vpc_id' => $expectation['vpc_id'] ?? null,
'member_id' => $expectation['member_id'] ?? null,
'private_ip_cidr' => $expectation['private_ip_cidr'] ?? null,
'private_mac' => $expectation['private_mac'] ?? null,
'desired_revision' => max(0, (int) ($expectation['desired_revision'] ?? 0)),
'desired_digest' => $expectation['desired_digest'] ?? null,
'link_state' => self::privateReconcileLinkState($kind, $status, $skipped),
];
try {
$result = $this->engine->host(
(int) $job->host_id, 'POST',
'vms/' . rawurlencode($uuid) . '/private-state/reconcile',
$actor, $body, (string) $job->operation_id
);
} catch (\Throwable) {
// A rolling upgrade may not expose the endpoint yet. Treat that
// exactly like every other missing host proof; the caller keeps
// the lifecycle operation uncertain and never publishes success.
return null;
}
if (!is_array($result)) {
return null;
}
// A rolling engine may wrap the observation in a durable operation
// envelope. Never accept stale nested evidence from a failed,
// uncertain or still-running POST; and bind any echoed operation/VM
// identity before inspecting its result payload.
if (array_key_exists('status', $result)) {
$envelopeStatus = is_string($result['status'])
? strtolower(trim($result['status'])) : '';
if (!in_array($envelopeStatus, ['succeeded', 'completed', 'done'], true)) {
return null;
}
}
if (array_key_exists('operation_id', $result)
&& (!is_string($result['operation_id'])
|| !hash_equals((string) $job->operation_id, $result['operation_id']))) {
return null;
}
if (array_key_exists('vm_uuid', $result)
&& (!is_string($result['vm_uuid'])
|| !hash_equals(strtolower($uuid), strtolower($result['vm_uuid'])))) {
return null;
}
// The route is UUID-scoped, but keep the response binding explicit at
// the PHP trust boundary as well. A host adapter bug or a misrouted
// envelope must never let another VM's private proof satisfy this
// lifecycle operation. Missing/invalid vm_uuid is therefore a
// missing observation, not a legacy-compatible success.
$accept = static function ($candidate) use ($uuid): ?array {
$observedVmUuid = is_array($candidate)
? ($candidate['vm_uuid'] ?? null) : null;
if (!is_string($observedVmUuid)
|| !hash_equals(strtolower($uuid), strtolower($observedVmUuid))) {
return null;
}
return $candidate;
};
if (is_array($result['private_state'] ?? null)) {
return $accept($result['private_state']);
}
if (is_array($result['vm'] ?? null)
&& is_array($result['vm']['private_state'] ?? null)) {
return $accept($result['vm']['private_state']);
}
if (is_array($result['result'] ?? null)
&& is_array($result['result']['private_state'] ?? null)) {
return $accept($result['result']['private_state']);
}
if (is_array($result['result'] ?? null)
&& is_array($result['result']['vm'] ?? null)
&& is_array($result['result']['vm']['private_state'] ?? null)) {
return $accept($result['result']['vm']['private_state']);
}
// Permit the small direct-observation response used by the engine
// route, but never treat an operation envelope or arbitrary payload
// as private-state evidence.
return array_key_exists('readback', $result)
&& array_key_exists('version', $result) ? $accept($result) : null;
}
/** Return the only link transition a lifecycle kind is allowed to request. */
private static function privateReconcileLinkState(
string $kind, string $status, bool $skipped = false
): string
{
// A policy job can be superseded before dispatch (for example a
// quota suspend after renewal). Its `skipped` result must preserve
// the current private link; applying the old target here would either
// disconnect a renewed VM or re-enable a newly blocked one.
if ($skipped) {
return 'preserve';
}
return match ($kind) {
'network_suspend' => 'down',
'network_resume' => in_array($status, ['succeeded', 'completed', 'done'], true)
? 'up' : 'preserve',
'delete', 'expiry_delete' => 'absent',
'create', 'reinstall', 'network_change', 'instance_settings', 'rescue',
'snapshot_create', 'snapshot_restore', 'snapshot_delete', 'action',
'service_suspend', 'service_resume', 'firewall_apply' => 'preserve',
default => 'preserve',
};
}
private static function validSystemCleanup(CloudJob $job, array $payload): bool
{
if ($job->instance_id === null || ($payload['body'] ?? null) !== []) {
return false;
}
$instance = CloudInstance::find($job->instance_id);
if ($instance === null || (string) $instance->status !== 'deleting'
|| (string) $instance->service_status !== 'purge_deleting'
|| (int) $job->user_id !== (int) $instance->user_id
|| (int) $job->host_id !== (int) $instance->host_id
|| (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id) {
return false;
}
return ($payload['method'] ?? null) === 'DELETE'
&& ($payload['resource'] ?? null) === 'vms/'
. rawurlencode((string) $instance->uuid) . '?delete_disks=true';
}
private static function validSystemPolicy(CloudJob $job, array $payload): bool
{
if ($job->instance_id === null || !is_array($payload['body'] ?? null)) {
return false;
}
$instance = CloudInstance::find($job->instance_id);
if ($instance === null || (int) $job->user_id !== (int) $instance->user_id
|| (int) $job->host_id !== (int) $instance->host_id
|| (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id) {
return false;
}
$suffix = [
'service_suspend' => 'service-suspend',
'service_resume' => 'service-resume',
'network_suspend' => 'network-suspend',
'network_resume' => 'network-resume',
][$job->kind] ?? null;
return $suffix !== null && ($payload['method'] ?? null) === 'POST'
&& ($payload['resource'] ?? null) === 'vms/'
. rawurlencode((string) $instance->uuid) . '/' . $suffix;
}
private static function payloadAction(array $payload): ?string
{
$body = $payload['body'] ?? null;
$action = is_array($body) && is_string($body['action'] ?? null)
? $body['action'] : null;
return CloudOperationCatalog::isPowerAction($action ?? '') ? $action : null;
}
/**
* Persist only a successful, allow-listed power observation. Runtime
* cache failures are isolated from the durable job outcome: the engine
* result and billing state remain authoritative even during a rolling
* schema upgrade where the cache table may not exist yet.
*/
private static function recordRuntime(CloudInstance $instance, array $result,
string $kind, array $payload): void
{
try {
if (in_array($kind, ['delete', 'expiry_delete'], true)) {
CloudInstanceRuntimeService::forget((int) $instance->id);
return;
}
$source = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
$state = $source['state'] ?? ($source['power_state'] ?? null);
$action = $kind === 'action' ? self::payloadAction($payload) : null;
$normalized = CloudInstanceRuntimeService::normalizeState($state);
if ($kind === 'action') {
// A successful graceful-shutdown job only confirms that the
// request was accepted. libvirt may still report running (or
// omit the state) until the guest completes ACPI shutdown.
// Preserve that distinction instead of publishing either a
// new fresh-running observation or a fabricated stopped one.
$state = CloudInstanceRuntimeService::stateAfterSuccessfulAction(
$state, $action
);
} elseif ($normalized === 'unknown' && $kind === 'reinstall') {
// The engine starts the replacement definition before
// returning a successful reinstall result. Older engine
// responses may omit the nested vm.state, so preserve
// the same successful postcondition explicitly.
$state = 'running';
}
$instanceId = (int) $instance->id;
$hostId = (int) $instance->host_id;
$uuid = (string) $instance->uuid;
if ($kind === 'create') {
CloudInstanceRuntimeService::seed($instanceId, $hostId, $uuid);
}
if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') {
CloudInstanceRuntimeService::markObserved(
$instanceId, $hostId, $uuid, $state
);
}
} catch (\Throwable $error) {
error_log('cloud runtime observation write unavailable: ' . get_class($error));
}
}
private static function exceptionDiagnostic(\Throwable $error, bool $dispatched,
?string $kind, bool $hasOrder, ?string $action): string
{
$operation = [];
if ($error instanceof CloudError) {
// CloudError text is engine-/host-controlled too. The catalog
// keeps only exact reviewed details and otherwise returns a
// stable generic code for the durable job row.
$operation['error'] = $error->getMessage();
} elseif ($error instanceof \GuzzleHttp\Exception\TransferException) {
$operation['error'] = '引擎连接中断或操作超时,正在核对结果';
}
$status = $dispatched ? 'uncertain' : 'failed';
return CloudOperationErrorCatalog::workerSummary($kind, $status,
$operation, $hasOrder, $action)
?? CloudOperationErrorCatalog::userMessage($kind, $status, null,
$hasOrder, $action)
?? '虚拟机操作失败,请稍后重试';
}
}