|
|
|
|
<?php
|
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudInstance;
|
|
|
|
|
use App\Models\CloudInvoice;
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
use App\Models\CloudOrder;
|
|
|
|
|
use App\Models\CloudServicePeriod;
|
|
|
|
|
use App\Models\User;
|
|
|
|
|
|
|
|
|
|
final class CloudWorkerService
|
|
|
|
|
{
|
|
|
|
|
private const ENGINE_TIMEOUT_SECONDS = 900.0;
|
|
|
|
|
private const POLICY_ENGINE_TIMEOUT_SECONDS = 120.0;
|
|
|
|
|
private const LEASE_SECONDS = 930;
|
|
|
|
|
private const POLICY_KINDS = ['service_suspend', 'service_resume',
|
|
|
|
|
'network_suspend', 'network_resume'];
|
|
|
|
|
private const REINSTALL_REVIEW_CODES = [
|
|
|
|
|
'reinstall_cleanup_pending', 'reinstall_result_uncertain',
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
private bool $policyOnly;
|
|
|
|
|
private array $observedErrors = [];
|
|
|
|
|
|
|
|
|
|
public function observationErrors(): array
|
|
|
|
|
{
|
|
|
|
|
return array_values(array_unique($this->observedErrors));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function __construct(private ?CloudEngineClient $engine = null,
|
|
|
|
|
bool $policyOnly = false)
|
|
|
|
|
{
|
|
|
|
|
$this->policyOnly = $policyOnly;
|
|
|
|
|
// A cloud-image clone can legitimately take minutes on remote storage.
|
|
|
|
|
// Keep the worker lease longer than the HTTP timeout so another worker
|
|
|
|
|
// cannot claim the same job while the first request is still active.
|
|
|
|
|
$this->engine ??= new CloudEngineClient(null, $policyOnly
|
|
|
|
|
? self::POLICY_ENGINE_TIMEOUT_SECONDS : self::ENGINE_TIMEOUT_SECONDS);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static function disposition(array $operation, bool $creating,
|
|
|
|
|
?string $kind = null): string
|
|
|
|
|
{
|
|
|
|
|
$status = is_string($operation['status'] ?? null)
|
|
|
|
|
? strtolower(trim($operation['status'])) : 'uncertain';
|
|
|
|
|
if ($kind === 'reinstall'
|
|
|
|
|
&& in_array($status, ['succeeded', 'completed', 'done', 'failed'], true)
|
|
|
|
|
&& self::reinstallNeedsReview($operation)) {
|
|
|
|
|
// A legacy/internal response can carry a post-commit marker while
|
|
|
|
|
// its top-level status is still failed. Keep it uncertain so the
|
|
|
|
|
// replacement credential/spec can be reconciled without allowing
|
|
|
|
|
// a destructive reinstall replay.
|
|
|
|
|
return 'uncertain';
|
|
|
|
|
}
|
|
|
|
|
if (in_array($status, ['succeeded', 'completed', 'done'], true)) {
|
|
|
|
|
return 'succeeded';
|
|
|
|
|
}
|
|
|
|
|
if ($status === 'failed') {
|
|
|
|
|
if (!$creating || ($operation['safe_to_refund'] ?? false) === true
|
|
|
|
|
|| ($operation['result']['creation_absent'] ?? false) === true) {
|
|
|
|
|
return 'failed';
|
|
|
|
|
}
|
|
|
|
|
return 'uncertain';
|
|
|
|
|
}
|
|
|
|
|
return in_array($status, ['pending', 'running'], true) ? 'running' : 'uncertain';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* A reinstall can commit the replacement guest before old-volume cleanup
|
|
|
|
|
* is confirmed. Preserve the new credential/spec while keeping the job
|
|
|
|
|
* visible as uncertain; legacy engines may only provide a warning, so
|
|
|
|
|
* that diagnostic is accepted as the compatibility signal here.
|
|
|
|
|
*/
|
|
|
|
|
private static function reinstallNeedsReview(array $operation): bool
|
|
|
|
|
{
|
|
|
|
|
$code = is_string($operation['error_code'] ?? null)
|
|
|
|
|
? strtolower(trim($operation['error_code'])) : '';
|
|
|
|
|
if (in_array($code, self::REINSTALL_REVIEW_CODES, true)) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
$result = $operation['result'] ?? null;
|
|
|
|
|
return is_array($result)
|
|
|
|
|
&& (($result['reinstall_committed'] ?? false) === true
|
|
|
|
|
|| ($result['cleanup_pending'] ?? false) === true
|
|
|
|
|
|| (is_string($result['warning'] ?? null)
|
|
|
|
|
&& trim($result['warning']) !== ''));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public function tick(): bool
|
|
|
|
|
{
|
|
|
|
|
$this->observedErrors = [];
|
|
|
|
|
$job = (new CloudJob())->getConnection()->transaction(function () {
|
|
|
|
|
$query = CloudJob::whereIn('status', ['queued', 'running', 'uncertain'])
|
|
|
|
|
->where('available_at', '<=', time())->where('lease_until', '<=', time());
|
|
|
|
|
if ($this->policyOnly) {
|
|
|
|
|
$query->whereIn('kind', self::POLICY_KINDS);
|
|
|
|
|
} else {
|
|
|
|
|
// The regular worker remains a fail-safe policy consumer when
|
|
|
|
|
// the dedicated workers are not installed, but always claims
|
|
|
|
|
// target-state policy ahead of slow image/provisioning jobs.
|
|
|
|
|
$quoted = "'" . implode("','", self::POLICY_KINDS) . "'";
|
|
|
|
|
$query->orderByRaw("CASE WHEN kind IN (" . $quoted
|
|
|
|
|
. ") THEN 0 WHEN kind = 'expiry_delete' THEN 1 ELSE 2 END");
|
|
|
|
|
}
|
|
|
|
|
$job = $query->orderBy('id')->lockForUpdate()->first();
|
|
|
|
|
if ($job === null) {
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
$job->lease_token = bin2hex(random_bytes(16));
|
|
|
|
|
$job->lease_until = time() + (in_array($job->kind, self::POLICY_KINDS, true)
|
|
|
|
|
? 150 : self::LEASE_SECONDS);
|
|
|
|
|
$job->attempts++;
|
|
|
|
|
$job->save();
|
|
|
|
|
return $job;
|
|
|
|
|
}, 3);
|
|
|
|
|
if ($job === null) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
$payload = [];
|
|
|
|
|
$dispatched = $job->dispatched_at !== null;
|
|
|
|
|
try {
|
|
|
|
|
$payload = CloudSecrets::decrypt($job->payload_cipher);
|
|
|
|
|
$policyKind = in_array($job->kind, self::POLICY_KINDS, true);
|
|
|
|
|
$systemPolicy = ($payload['system_policy'] ?? false) === true
|
|
|
|
|
&& $policyKind && self::validSystemPolicy($job, $payload);
|
|
|
|
|
// Expiry deletion is authorized by the server-side lifecycle
|
|
|
|
|
// daemon, but remains a destructive one-shot mutation. Keep it
|
|
|
|
|
// separate from replayable target-state policy so a lost response
|
|
|
|
|
// can never cause PHP to submit DELETE a second time.
|
|
|
|
|
$systemCleanup = $job->kind === 'expiry_delete'
|
|
|
|
|
&& ($payload['system_cleanup'] ?? false) === true
|
|
|
|
|
&& self::validSystemCleanup($job, $payload);
|
|
|
|
|
if (($policyKind && !$systemPolicy)
|
|
|
|
|
|| ($job->kind === 'expiry_delete' && !$systemCleanup)) {
|
|
|
|
|
throw new CloudError('系统策略任务与实例状态不匹配', 409);
|
|
|
|
|
}
|
|
|
|
|
$actor = ['user_id' => (int) $job->user_id, 'is_admin' => false, 'is_super_admin' => false,
|
|
|
|
|
'can_manage' => false, 'session_id' => '', 'operation_id' => $job->operation_id];
|
|
|
|
|
$user = User::find($job->user_id);
|
|
|
|
|
if (!$systemPolicy && !$systemCleanup
|
|
|
|
|
&& ($user === null || (int) $user->enable !== 1)) {
|
|
|
|
|
if ($job->dispatched_at === null) {
|
|
|
|
|
$this->finish($job, ['status' => 'failed', 'safe_to_refund' => true], $payload);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if ($job->kind === 'create') {
|
|
|
|
|
$actor['can_provision'] = true;
|
|
|
|
|
$actor['can_manage'] = true;
|
|
|
|
|
} elseif ($systemPolicy || $systemCleanup) {
|
|
|
|
|
$actor['can_manage'] = true;
|
|
|
|
|
if ($systemCleanup) {
|
|
|
|
|
$actor['system_cleanup'] = true;
|
|
|
|
|
}
|
|
|
|
|
} elseif (($payload['can_manage'] ?? false) === true && $user !== null
|
|
|
|
|
&& (bool) $user->is_admin && in_array((int) $user->id,
|
|
|
|
|
array_map('intval', explode(',', (string) \App\Services\Config::get('super_admin'))), true)) {
|
|
|
|
|
$actor['is_admin'] = $actor['is_super_admin'] = $actor['can_manage'] = true;
|
|
|
|
|
} elseif ((int) ($payload['owner_id'] ?? 0) === (int) $job->user_id) {
|
|
|
|
|
$actor['can_manage'] = true;
|
|
|
|
|
}
|
|
|
|
|
if ($job->kind !== 'create' && !$actor['can_manage'] && $job->dispatched_at === null) {
|
|
|
|
|
$this->finish($job, ['status' => 'failed'], $payload);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
if (!CloudFirewallService::assertJobRunnable($job, $payload)) {
|
|
|
|
|
// A confirmed instance deletion dominates an overlapping
|
|
|
|
|
// tenant firewall request. Close the obsolete job without
|
|
|
|
|
// touching the deleted policy or contacting the engine.
|
|
|
|
|
$this->finish($job, [
|
|
|
|
|
'operation_id' => (string) $job->operation_id,
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'error_code' => 'firewall_apply_superseded',
|
|
|
|
|
'error' => '实例已删除,防火墙请求不再适用',
|
|
|
|
|
], $payload);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
if ($systemPolicy && $job->dispatched_at === null && $job->instance_id !== null) {
|
|
|
|
|
$instance = CloudInstance::find($job->instance_id);
|
|
|
|
|
$desired = self::policyDesired($instance, (string) $job->kind);
|
|
|
|
|
if (!$desired) {
|
|
|
|
|
$this->finish($job, ['status' => 'succeeded', 'result' => ['skipped' => true]], $payload);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (!$systemPolicy && !$systemCleanup && $job->kind !== 'create'
|
|
|
|
|
&& $job->dispatched_at === null
|
|
|
|
|
&& $job->instance_id !== null) {
|
|
|
|
|
$instance = CloudInstance::find($job->instance_id);
|
|
|
|
|
$policyTransition = $instance !== null
|
|
|
|
|
&& (in_array((string) $instance->service_status,
|
|
|
|
|
['suspending_expired', 'resume_pending', 'resuming',
|
|
|
|
|
'suspend_failed', 'resume_failed', 'expired_grace',
|
|
|
|
|
'purge_stopping', 'purge_stop_failed',
|
|
|
|
|
'purge_deleting', 'purge_failed'], true)
|
|
|
|
|
|| in_array((string) $instance->network_status,
|
|
|
|
|
['blocking', 'blocking_expired', 'blocked_expired',
|
|
|
|
|
'restore_pending', 'restoring', 'block_failed',
|
|
|
|
|
'restore_failed'], true));
|
|
|
|
|
$globalOverride = $actor['is_super_admin'] && $actor['can_manage'];
|
|
|
|
|
if ($instance === null || ($job->kind !== 'delete' && !$globalOverride
|
|
|
|
|
&& ($policyTransition || !$instance->serviceAvailable()))) {
|
|
|
|
|
$this->finish($job, ['status' => 'failed',
|
|
|
|
|
'error' => '实例服务已到期或正在执行策略,不能执行该操作'], $payload);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if ($job->dispatched_at !== null) {
|
|
|
|
|
// A crash after dispatch is ambiguous. Destructive/user jobs
|
|
|
|
|
// are never replayed. Policy lookups are explicitly read-only:
|
|
|
|
|
// the engine's ordinary GET reconciles target state and could
|
|
|
|
|
// otherwise apply a stale suspend before PHP rechecks that a
|
|
|
|
|
// new paid period now desires resume.
|
|
|
|
|
if ($systemPolicy) {
|
|
|
|
|
try {
|
|
|
|
|
$operation = $this->engine->request('GET',
|
|
|
|
|
'/internal/v1/operations/' . $job->operation_id
|
|
|
|
|
. '?reconcile=false',
|
|
|
|
|
$actor, null, $job->operation_id);
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
if ($error->httpStatus !== 404) {
|
|
|
|
|
throw $error;
|
|
|
|
|
}
|
|
|
|
|
$operation = null;
|
|
|
|
|
}
|
|
|
|
|
$terminal = is_array($operation) && in_array(
|
|
|
|
|
(string) ($operation['status'] ?? ''), ['succeeded', 'failed'], true);
|
|
|
|
|
if (!$terminal) {
|
|
|
|
|
// Do not close or reverse a stale policy while its
|
|
|
|
|
// original engine executor can still touch libvirt.
|
|
|
|
|
// Keeping this job uncertain also prevents lifecycle
|
|
|
|
|
// from enqueueing the opposite target until the next
|
|
|
|
|
// reconciliation pass observes a terminal operation.
|
|
|
|
|
if (($operation['executor_active'] ?? false) === true) {
|
|
|
|
|
throw new CloudError('引擎仍在执行当前实例策略,请稍后核对', 503);
|
|
|
|
|
}
|
|
|
|
|
// Re-read desired state after the read-only lookup.
|
|
|
|
|
// Most recorded target states must converge before an
|
|
|
|
|
// opposite transition is sent. The retired expiry
|
|
|
|
|
// shutdown is different: grace now requires only link
|
|
|
|
|
// down, so never replay it after an online upgrade.
|
|
|
|
|
$current = $job->instance_id === null ? null
|
|
|
|
|
: CloudInstance::find($job->instance_id);
|
|
|
|
|
$desired = self::policyDesired($current, (string) $job->kind);
|
|
|
|
|
$retiredExpirySuspend = $job->kind === 'service_suspend'
|
|
|
|
|
&& ($payload['suspend_reason'] ?? null) === 'expired'
|
|
|
|
|
&& !$desired;
|
|
|
|
|
if (!$desired && ($operation === null || $retiredExpirySuspend)) {
|
|
|
|
|
$result = ['skipped' => true];
|
|
|
|
|
if ($job->kind === 'service_suspend'
|
|
|
|
|
&& ($operation['service_was_running_hint'] ?? false) === true) {
|
|
|
|
|
$result['service_was_running'] = true;
|
|
|
|
|
}
|
|
|
|
|
$this->finish($job, ['status' => 'succeeded',
|
|
|
|
|
'result' => $result], $payload);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
$operation = $this->engine->host((int) $job->host_id,
|
|
|
|
|
$payload['method'], $payload['resource'], $actor,
|
|
|
|
|
$payload['body'], $job->operation_id);
|
|
|
|
|
}
|
|
|
|
|
} elseif ($systemCleanup) {
|
|
|
|
|
// Ask the engine to reconcile its durable operation. The
|
|
|
|
|
// delete mutation itself is never replayed: ordinary
|
|
|
|
|
// ambiguous deletes become terminal failures, while a VM
|
|
|
|
|
// whose deletion was already proven may retry only the
|
|
|
|
|
// persisted public-IP claim cleanup. Missing or otherwise
|
|
|
|
|
// nonterminal records remain uncertain for inspection.
|
|
|
|
|
try {
|
|
|
|
|
$operation = $this->engine->request('GET',
|
|
|
|
|
'/internal/v1/operations/' . $job->operation_id,
|
|
|
|
|
$actor, null, $job->operation_id);
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
if ($error->httpStatus === 404) {
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
'引擎中缺少到期删除记录,必须人工核查宿主机', 503);
|
|
|
|
|
}
|
|
|
|
|
throw $error;
|
|
|
|
|
}
|
|
|
|
|
if (!in_array((string) ($operation['status'] ?? ''),
|
|
|
|
|
['succeeded', 'failed'], true)) {
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
'到期删除结果不确定,必须人工核查宿主机', 503);
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
try {
|
|
|
|
|
$operation = $this->engine->request('GET',
|
|
|
|
|
'/internal/v1/operations/' . $job->operation_id,
|
|
|
|
|
$actor, null, $job->operation_id);
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
if ($job->kind === 'create' || $error->httpStatus !== 404) {
|
|
|
|
|
throw $error;
|
|
|
|
|
}
|
|
|
|
|
// Never replay destructive/user mutations. An
|
|
|
|
|
// authoritative missing operation means there is no
|
|
|
|
|
// engine registry entry left to reconcile; terminate
|
|
|
|
|
// the local job visibly so it cannot block expiry.
|
|
|
|
|
$this->finish($job, [
|
|
|
|
|
'operation_id' => (string) $job->operation_id,
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'error' => '引擎中已无此任务记录,请核查虚拟机状态后重试',
|
|
|
|
|
], $payload);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
$updated = CloudJob::where('id', $job->id)->where('lease_token', $job->lease_token)
|
|
|
|
|
->whereNull('dispatched_at')->update(['dispatched_at' => time(), 'status' => 'running', 'updated_at' => time()]);
|
|
|
|
|
if ($updated !== 1) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
CloudFirewallService::markJobRunning($job);
|
|
|
|
|
$dispatched = true;
|
|
|
|
|
$operation = $this->engine->host((int) $job->host_id, $payload['method'],
|
|
|
|
|
$payload['resource'], $actor, $payload['body'], $job->operation_id);
|
|
|
|
|
}
|
|
|
|
|
if (isset($operation['uuid']) && !array_key_exists('result', $operation)) {
|
|
|
|
|
$operation = ['status' => $operation['status'] ?? 'succeeded',
|
|
|
|
|
'result' => $operation];
|
|
|
|
|
}
|
|
|
|
|
$this->finish($job, $operation, $payload);
|
|
|
|
|
} catch (\Throwable $error) {
|
|
|
|
|
// Diagnostics deliberately contain no raw payloads, credentials or upstream tracebacks.
|
|
|
|
|
$this->observedErrors[] = 'worker_tick_failed';
|
|
|
|
|
$diagnostic = self::exceptionDiagnostic($error, $dispatched,
|
|
|
|
|
(string) $job->kind, $job->order_id !== null, self::payloadAction($payload));
|
|
|
|
|
$log = 'cloud worker job ' . $job->id . ' failed: ' . get_class($error);
|
|
|
|
|
if ($error instanceof CloudError) {
|
|
|
|
|
$log .= ' status=' . $error->httpStatus . ' message=' . $diagnostic;
|
|
|
|
|
}
|
|
|
|
|
error_log($log);
|
|
|
|
|
if (!$dispatched) {
|
|
|
|
|
// Decryption/configuration/local validation failed before an
|
|
|
|
|
// engine request was sent, so create did not happen and an
|
|
|
|
|
// order refund is safe.
|
|
|
|
|
try {
|
|
|
|
|
$this->finish($job, ['status' => 'failed', 'safe_to_refund' => true,
|
|
|
|
|
'error' => $diagnostic], $payload);
|
|
|
|
|
return true;
|
|
|
|
|
} catch (\Throwable $finishError) {
|
|
|
|
|
error_log('cloud worker job ' . $job->id
|
|
|
|
|
. ' could not record pre-dispatch failure: ' . get_class($finishError));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
$markedUncertain = CloudJob::where('id', $job->id)
|
|
|
|
|
->where('lease_token', $job->lease_token)->update([
|
|
|
|
|
'status' => 'uncertain', 'error_code' => $diagnostic, 'lease_until' => 0,
|
|
|
|
|
'lease_token' => null, 'available_at' => time() + 30, 'updated_at' => time()]);
|
|
|
|
|
// Losing the lease means another worker owns every subsequent
|
|
|
|
|
// state transition. Never mutate its order, reservation, or IPs.
|
|
|
|
|
if ($markedUncertain !== 1) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
if (in_array((string) $job->kind,
|
|
|
|
|
['snapshot_create', 'snapshot_restore', 'snapshot_delete',
|
|
|
|
|
'delete', 'expiry_delete'], true)) {
|
|
|
|
|
try {
|
|
|
|
|
CloudSnapshotService::markJobUncertain($job);
|
|
|
|
|
} catch (\Throwable $snapshotError) {
|
|
|
|
|
error_log('cloud worker snapshot catalog could not be marked uncertain: '
|
|
|
|
|
. get_class($snapshotError));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (in_array((string) $job->kind,
|
|
|
|
|
['firewall_apply', 'delete', 'expiry_delete'], true)) {
|
|
|
|
|
try {
|
|
|
|
|
CloudFirewallService::markJobUncertain($job);
|
|
|
|
|
} catch (\Throwable $firewallError) {
|
|
|
|
|
error_log('cloud worker firewall state could not be marked uncertain: '
|
|
|
|
|
. get_class($firewallError));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if ($job->kind === 'network_change') {
|
|
|
|
|
try {
|
|
|
|
|
(new CloudJob())->getConnection()->transaction(
|
|
|
|
|
function () use ($job, $payload): void {
|
|
|
|
|
$instance = $job->instance_id === null ? null
|
|
|
|
|
: CloudInstance::where('id', $job->instance_id)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
$sourceIpId = filter_var(
|
|
|
|
|
$payload['source_public_ip_id'] ?? null,
|
|
|
|
|
FILTER_VALIDATE_INT,
|
|
|
|
|
['options' => ['min_range' => 1]]
|
|
|
|
|
);
|
|
|
|
|
$targetIpId = filter_var(
|
|
|
|
|
$payload['target_public_ip_id'] ?? null,
|
|
|
|
|
FILTER_VALIDATE_INT,
|
|
|
|
|
['options' => ['min_range' => 1]]
|
|
|
|
|
);
|
|
|
|
|
$networkAttachmentId = filter_var(
|
|
|
|
|
$payload['network_attachment_id'] ?? null,
|
|
|
|
|
FILTER_VALIDATE_INT,
|
|
|
|
|
['options' => ['min_range' => 1]]
|
|
|
|
|
);
|
|
|
|
|
$sourcePoolId = filter_var(
|
|
|
|
|
$payload['source_public_ip_pool_id'] ?? null,
|
|
|
|
|
FILTER_VALIDATE_INT,
|
|
|
|
|
['options' => ['min_range' => 1]]
|
|
|
|
|
);
|
|
|
|
|
$targetPoolId = filter_var(
|
|
|
|
|
$payload['target_public_ip_pool_id'] ?? null,
|
|
|
|
|
FILTER_VALIDATE_INT,
|
|
|
|
|
['options' => ['min_range' => 1]]
|
|
|
|
|
);
|
|
|
|
|
if ($instance === null || $sourceIpId === false
|
|
|
|
|
|| $targetIpId === false
|
|
|
|
|
|| $networkAttachmentId === false
|
|
|
|
|
|| $sourcePoolId === false
|
|
|
|
|
|| $targetPoolId === false) {
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
'公网 IP 变更任务缺少账本关联', 409
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
CloudPlacementService::finishNetworkChange(
|
|
|
|
|
$instance, (string) $job->operation_id,
|
|
|
|
|
(int) $sourceIpId, (int) $targetIpId,
|
|
|
|
|
(int) $networkAttachmentId,
|
|
|
|
|
(int) $sourcePoolId, (int) $targetPoolId,
|
|
|
|
|
'uncertain'
|
|
|
|
|
);
|
|
|
|
|
}, 3
|
|
|
|
|
);
|
|
|
|
|
} catch (\Throwable $ledgerError) {
|
|
|
|
|
error_log('cloud worker network ledger could not be marked uncertain: '
|
|
|
|
|
. get_class($ledgerError));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if ($job->order_id !== null) {
|
|
|
|
|
CloudOrder::where('id', $job->order_id)->whereNotIn('status', ['succeeded', 'failed'])
|
|
|
|
|
->update(['status' => 'uncertain', 'updated_at' => time()]);
|
|
|
|
|
try {
|
|
|
|
|
(new CloudOrder())->getConnection()->transaction(function () use ($job): void {
|
|
|
|
|
$order = CloudOrder::where('id', $job->order_id)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
if ($order !== null) {
|
|
|
|
|
CloudPlacementService::markUncertain($order);
|
|
|
|
|
}
|
|
|
|
|
}, 3);
|
|
|
|
|
} catch (\Throwable $reservationError) {
|
|
|
|
|
error_log('cloud worker reservation could not be marked uncertain: '
|
|
|
|
|
. get_class($reservationError));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function policyDesired(?CloudInstance $instance, string $kind): bool
|
|
|
|
|
{
|
|
|
|
|
if ($instance === null) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
$status = (string) $instance->status;
|
|
|
|
|
return match ($kind) {
|
|
|
|
|
'service_suspend' => in_array($status, ['active', 'deleting'], true)
|
|
|
|
|
&& (string) $instance->service_status === 'purge_stopping',
|
|
|
|
|
'service_resume' => $status === 'active'
|
|
|
|
|
&& in_array((string) $instance->service_status,
|
|
|
|
|
['resume_pending', 'resuming'], true),
|
|
|
|
|
'network_suspend' => $status === 'active'
|
|
|
|
|
&& in_array((string) $instance->network_status,
|
|
|
|
|
['blocking', 'blocking_expired'], true),
|
|
|
|
|
'network_resume' => $status === 'active'
|
|
|
|
|
&& in_array((string) $instance->network_status,
|
|
|
|
|
['restore_pending', 'restoring'], true),
|
|
|
|
|
default => false,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private function finish(CloudJob $claimed, array $operation, array $payload): void
|
|
|
|
|
{
|
|
|
|
|
$status = self::disposition($operation, $claimed->kind === 'create',
|
|
|
|
|
(string) $claimed->kind);
|
|
|
|
|
if (in_array($status, ['failed', 'uncertain'], true)) {
|
|
|
|
|
$this->observedErrors[] = $status === 'failed' ? 'task_failed' : 'task_uncertain';
|
|
|
|
|
}
|
|
|
|
|
$result = $operation['result'] ?? null;
|
|
|
|
|
(new CloudJob())->getConnection()->transaction(function () use ($claimed, $payload, $status, $result, $operation) {
|
|
|
|
|
// All financial transactions acquire the wallet first, avoiding wallet/order deadlocks.
|
|
|
|
|
$user = User::where('id', $claimed->user_id)->lockForUpdate()->first();
|
|
|
|
|
$order = $claimed->order_id === null ? null : CloudOrder::where('id', $claimed->order_id)->lockForUpdate()->first();
|
|
|
|
|
$job = CloudJob::where('id', $claimed->id)->lockForUpdate()->first();
|
|
|
|
|
if ($job === null || $job->lease_token !== $claimed->lease_token
|
|
|
|
|
|| in_array($job->status, ['succeeded', 'failed', 'cancelled'], true)) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
$safe = is_array($result)
|
|
|
|
|
? CloudOperationErrorCatalog::workerResult($result) : null;
|
|
|
|
|
CloudSnapshotService::finishJobLocked($job, $status, $result, $payload);
|
|
|
|
|
CloudSnapshotService::finishInstanceDeleteLocked($job, $status);
|
|
|
|
|
CloudFirewallService::finishJobLocked(
|
|
|
|
|
$job, $status, $result, $payload, $operation
|
|
|
|
|
);
|
|
|
|
|
CloudFirewallService::finishInstanceDeleteLocked($job, $status);
|
|
|
|
|
if ($job->kind === 'network_change'
|
|
|
|
|
&& in_array($status, ['succeeded', 'failed', 'uncertain'], true)) {
|
|
|
|
|
$instance = $job->instance_id === null ? null
|
|
|
|
|
: CloudInstance::where('id', $job->instance_id)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
$sourceIpId = filter_var($payload['source_public_ip_id'] ?? null,
|
|
|
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
|
|
|
$targetIpId = filter_var($payload['target_public_ip_id'] ?? null,
|
|
|
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
|
|
|
|
|
$networkAttachmentId = filter_var(
|
|
|
|
|
$payload['network_attachment_id'] ?? null,
|
|
|
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]
|
|
|
|
|
);
|
|
|
|
|
$sourcePoolId = filter_var(
|
|
|
|
|
$payload['source_public_ip_pool_id'] ?? null,
|
|
|
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]
|
|
|
|
|
);
|
|
|
|
|
$targetPoolId = filter_var(
|
|
|
|
|
$payload['target_public_ip_pool_id'] ?? null,
|
|
|
|
|
FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]
|
|
|
|
|
);
|
|
|
|
|
if ($instance === null || $sourceIpId === false
|
|
|
|
|
|| $targetIpId === false || $networkAttachmentId === false
|
|
|
|
|
|| $sourcePoolId === false || $targetPoolId === false) {
|
|
|
|
|
throw new CloudError('公网 IP 变更任务缺少账本关联', 409);
|
|
|
|
|
}
|
|
|
|
|
if ($status === 'succeeded') {
|
|
|
|
|
$requested = is_array($payload['body'] ?? null)
|
|
|
|
|
? ($payload['body']['public_ip_cidr'] ?? null) : null;
|
|
|
|
|
$requestedPoolKey = is_array($payload['body'] ?? null)
|
|
|
|
|
? ($payload['body']['public_pool_key'] ?? null) : null;
|
|
|
|
|
$confirmed = is_array($result)
|
|
|
|
|
? ($result['public_ip_cidr'] ?? null) : null;
|
|
|
|
|
$confirmedPoolKey = is_array($result)
|
|
|
|
|
? ($result['public_pool_key'] ?? null) : null;
|
|
|
|
|
if (!is_string($requested) || !is_string($confirmed)
|
|
|
|
|
|| !is_string($requestedPoolKey)
|
|
|
|
|
|| !is_string($confirmedPoolKey)
|
|
|
|
|
|| $requested !== $confirmed
|
|
|
|
|
|| !hash_equals($requestedPoolKey, $confirmedPoolKey)) {
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
'引擎成功结果缺少已确认的公网 IP 或地址池', 502
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
$decoded = json_decode((string) $instance->spec_json, true);
|
|
|
|
|
if (!is_array($decoded) || array_is_list($decoded)) {
|
|
|
|
|
throw new CloudError('虚拟机配置记录损坏,无法保存新的公网 IP', 409);
|
|
|
|
|
}
|
|
|
|
|
CloudPlacementService::finishNetworkChange($instance,
|
|
|
|
|
(string) $job->operation_id, (int) $sourceIpId,
|
|
|
|
|
(int) $targetIpId, (int) $networkAttachmentId,
|
|
|
|
|
(int) $sourcePoolId, (int) $targetPoolId, 'succeeded');
|
|
|
|
|
$decoded['public_ip_cidr'] = $confirmed;
|
|
|
|
|
$instance->spec_json = json_encode($decoded,
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE);
|
|
|
|
|
self::recordRuntime($instance, is_array($result) ? $result : [],
|
|
|
|
|
(string) $job->kind, $payload);
|
|
|
|
|
$instance->updated_at = time();
|
|
|
|
|
$instance->save();
|
|
|
|
|
} else {
|
|
|
|
|
CloudPlacementService::finishNetworkChange($instance,
|
|
|
|
|
(string) $job->operation_id, (int) $sourceIpId,
|
|
|
|
|
(int) $targetIpId, (int) $networkAttachmentId,
|
|
|
|
|
(int) $sourcePoolId, (int) $targetPoolId, $status);
|
|
|
|
|
}
|
|
|
|
|
} elseif ($status === 'succeeded' && $job->kind === 'create') {
|
|
|
|
|
$vm = is_array($result) ? ($result['vm'] ?? $result) : [];
|
|
|
|
|
$uuid = $vm['uuid'] ?? '';
|
|
|
|
|
if ($order === null || !is_string($uuid)
|
|
|
|
|
|| preg_match('/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/iD', $uuid) !== 1) {
|
|
|
|
|
throw new CloudError('引擎成功结果缺少虚拟机标识', 502);
|
|
|
|
|
}
|
|
|
|
|
$requestedPublicIpCidr = is_array($payload['body'] ?? null)
|
|
|
|
|
? ($payload['body']['public_ip_cidr'] ?? null) : null;
|
|
|
|
|
$requestedPublicPoolKey = is_array($payload['body'] ?? null)
|
|
|
|
|
? ($payload['body']['public_pool_key'] ?? null) : null;
|
|
|
|
|
$confirmedPublicIpCidr = $vm['public_ip_cidr'] ?? null;
|
|
|
|
|
$confirmedPublicPoolKey = $vm['public_pool_key'] ?? null;
|
|
|
|
|
// Jobs queued before the multi-pool rollout have no key; the
|
|
|
|
|
// engine's unique-IP inference remains their compatibility
|
|
|
|
|
// path. Every newly queued placement includes a key and must
|
|
|
|
|
// receive the same identity back before PHP commits it.
|
|
|
|
|
$poolIdentityMatches = $requestedPublicPoolKey === null
|
|
|
|
|
|| (is_string($requestedPublicPoolKey)
|
|
|
|
|
&& is_string($confirmedPublicPoolKey)
|
|
|
|
|
&& hash_equals($requestedPublicPoolKey,
|
|
|
|
|
$confirmedPublicPoolKey));
|
|
|
|
|
if (!is_string($requestedPublicIpCidr)
|
|
|
|
|
|| !is_string($confirmedPublicIpCidr)
|
|
|
|
|
|| !hash_equals($requestedPublicIpCidr,
|
|
|
|
|
$confirmedPublicIpCidr)
|
|
|
|
|
|| !$poolIdentityMatches) {
|
|
|
|
|
// A recovered create result may describe a VM whose
|
|
|
|
|
// network was changed out of band after the original
|
|
|
|
|
// response was lost. Never commit the original address in
|
|
|
|
|
// that case: keep the order and reservation uncertain for
|
|
|
|
|
// an administrator to reconcile.
|
|
|
|
|
throw new CloudError(
|
|
|
|
|
'引擎成功结果与预留公网 IP 不一致,必须人工核查', 502
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
$instance = CloudInstance::where('order_id', $order->id)->first();
|
|
|
|
|
$newInstance = $instance === null;
|
|
|
|
|
$instance ??= new CloudInstance();
|
|
|
|
|
$instance->uuid = $uuid;
|
|
|
|
|
$instance->host_id = $order->host_id;
|
|
|
|
|
$instance->user_id = $order->user_id;
|
|
|
|
|
$instance->order_id = $order->id;
|
|
|
|
|
$instance->offer_id = $order->offer_id;
|
|
|
|
|
$instance->resource_pool_id = $order->resource_pool_id;
|
|
|
|
|
$instance->reservation_id = $order->reservation_id;
|
|
|
|
|
$spec = json_decode((string) $order->spec_json, true) ?: [];
|
|
|
|
|
$now = time();
|
|
|
|
|
$billingSnapshot = array_key_exists('validity_days', $spec)
|
|
|
|
|
&& array_key_exists('traffic_quota_bytes', $spec)
|
|
|
|
|
&& (int) $spec['validity_days'] >= 1;
|
|
|
|
|
$days = $billingSnapshot ? (int) $spec['validity_days'] : 0;
|
|
|
|
|
$traffic = $billingSnapshot
|
|
|
|
|
? max(0, (int) $spec['traffic_quota_bytes']) : 0;
|
|
|
|
|
$instance->plan_id = $order->plan_id;
|
|
|
|
|
// A late/replayed create result must not overwrite a display
|
|
|
|
|
// name the owner changed after the first success committed.
|
|
|
|
|
if ($newInstance) {
|
|
|
|
|
$instance->name = $payload['body']['name'];
|
|
|
|
|
}
|
|
|
|
|
$instance->status = 'active';
|
|
|
|
|
// Orders submitted by a pre-billing release do not contain an
|
|
|
|
|
// immutable term/quota snapshot. Never invent a 30-day expiry
|
|
|
|
|
// while such an order is in flight during upgrade; preserve
|
|
|
|
|
// its historical unlimited/exempt behavior instead.
|
|
|
|
|
$instance->service_status = $billingSnapshot ? 'active' : 'exempt';
|
|
|
|
|
$instance->network_status = 'active';
|
|
|
|
|
$instance->expires_at = $billingSnapshot ? $now + $days * 86400 : null;
|
|
|
|
|
$instance->renewal_price = $billingSnapshot ? $order->amount : null;
|
|
|
|
|
$instance->renewal_days = $days;
|
|
|
|
|
$instance->renewal_traffic_bytes = $traffic;
|
|
|
|
|
$instance->traffic_quota_bytes = $traffic;
|
|
|
|
|
$instance->traffic_download_bytes = 0;
|
|
|
|
|
$instance->traffic_upload_bytes = 0;
|
|
|
|
|
$instance->lifetime_download_bytes = 0;
|
|
|
|
|
$instance->lifetime_upload_bytes = 0;
|
|
|
|
|
$instance->traffic_synced_at = 0;
|
|
|
|
|
// Persist only the engine's slow-changing public metadata.
|
|
|
|
|
// SSR pages can then render IP, operating system, login name
|
|
|
|
|
// and actual capacity without issuing a full libvirt/XML read
|
|
|
|
|
// on every request. Billing/order fields remain authoritative
|
|
|
|
|
// because the allowlist cannot overwrite them.
|
|
|
|
|
$instance->spec_json = json_encode(
|
|
|
|
|
CloudInstanceMetadata::merge($spec, $vm),
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE
|
|
|
|
|
);
|
|
|
|
|
$instance->credential_cipher = isset($payload['credential']) ? CloudSecrets::encrypt($payload['credential']) : null;
|
|
|
|
|
// The paid service begins when provisioning actually succeeds,
|
|
|
|
|
// not while an opening invoice is waiting for payment. A
|
|
|
|
|
// reconciled/replayed success must retain the original value.
|
|
|
|
|
if (!$instance->exists) {
|
|
|
|
|
$instance->created_at = $now;
|
|
|
|
|
}
|
|
|
|
|
$instance->updated_at = time();
|
|
|
|
|
$instance->save();
|
|
|
|
|
CloudPlacementService::commit(
|
|
|
|
|
$order, (int) $instance->id, $confirmedPublicIpCidr
|
|
|
|
|
);
|
|
|
|
|
// Seed the independent runtime observation immediately after
|
|
|
|
|
// provisioning. If the engine returned a trusted power
|
|
|
|
|
// state, promote it now; otherwise the background host batch
|
|
|
|
|
// will provide the first observation without showing "未知".
|
|
|
|
|
self::recordRuntime($instance, $vm, 'create', []);
|
|
|
|
|
if ($billingSnapshot) {
|
|
|
|
|
$period = CloudServicePeriod::where('order_id', $order->id)->first()
|
|
|
|
|
?? new CloudServicePeriod();
|
|
|
|
|
$period->instance_id = $instance->id;
|
|
|
|
|
$period->user_id = $order->user_id;
|
|
|
|
|
$period->plan_id = $order->plan_id;
|
|
|
|
|
$period->order_id = $order->id;
|
|
|
|
|
$period->starts_at = $now;
|
|
|
|
|
$period->ends_at = $instance->expires_at;
|
|
|
|
|
$period->traffic_quota_bytes = $traffic;
|
|
|
|
|
$period->download_base_bytes = 0;
|
|
|
|
|
$period->upload_base_bytes = 0;
|
|
|
|
|
$period->download_used_bytes = 0;
|
|
|
|
|
$period->upload_used_bytes = 0;
|
|
|
|
|
$period->status = 'active';
|
|
|
|
|
$period->activated_at = $now;
|
|
|
|
|
$period->created_at = $period->created_at ?: $now;
|
|
|
|
|
$period->updated_at = $now;
|
|
|
|
|
$period->save();
|
|
|
|
|
$instance->current_period_id = $period->id;
|
|
|
|
|
$instance->save();
|
|
|
|
|
}
|
|
|
|
|
$job->instance_id = $order->instance_id = $instance->id;
|
|
|
|
|
$provisionInvoice = CloudInvoice::where('order_id', $order->id)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
if ($provisionInvoice !== null
|
|
|
|
|
&& (string) $provisionInvoice->purpose === 'provision') {
|
|
|
|
|
$provisionInvoice->instance_id = $instance->id;
|
|
|
|
|
$provisionInvoice->updated_at = time();
|
|
|
|
|
$provisionInvoice->save();
|
|
|
|
|
CloudInvoiceService::queueProvisionSucceeded(
|
|
|
|
|
$provisionInvoice, $now
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
$safe = $safe ?? [];
|
|
|
|
|
$safe['id'] = (string) $instance->id;
|
|
|
|
|
$safe['uuid'] = $uuid;
|
|
|
|
|
} elseif ($status === 'failed' && $order !== null && $order->refunded_at === null) {
|
|
|
|
|
if ($user === null) {
|
|
|
|
|
throw new CloudError('退款账户不存在', 409);
|
|
|
|
|
}
|
|
|
|
|
$user->money = CloudMoney::decimal(CloudMoney::cents((string) $user->money) + CloudMoney::cents($order->amount));
|
|
|
|
|
$user->save();
|
|
|
|
|
$invoice = CloudInvoice::where('order_id', $order->id)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
// Payment placement locks offer before host. Keep the same
|
|
|
|
|
// cross-resource order during refunds so a failed creation on
|
|
|
|
|
// one order cannot deadlock a concurrent payment for the same
|
|
|
|
|
// offer and host (offer -> host -> reservation -> IP).
|
|
|
|
|
CloudOrderService::releaseStock($order);
|
|
|
|
|
CloudPlacementService::release($order, 'create_failed');
|
|
|
|
|
if ($invoice !== null && (string) $invoice->purpose === 'provision') {
|
|
|
|
|
$invoice->status = 'refunded';
|
|
|
|
|
$invoice->updated_at = time();
|
|
|
|
|
$invoice->save();
|
|
|
|
|
}
|
|
|
|
|
$order->refunded_at = time();
|
|
|
|
|
} elseif ($status === 'failed' && $job->kind === 'expiry_delete'
|
|
|
|
|
&& $job->instance_id !== null) {
|
|
|
|
|
$instance = CloudInstance::where('id', $job->instance_id)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
if ($instance !== null && (string) $instance->status !== 'deleted') {
|
|
|
|
|
$instance->status = 'delete_failed';
|
|
|
|
|
$instance->service_status = 'purge_failed';
|
|
|
|
|
$instance->updated_at = time();
|
|
|
|
|
$instance->save();
|
|
|
|
|
}
|
|
|
|
|
} elseif ($status === 'failed' && $job->instance_id !== null
|
|
|
|
|
&& in_array($job->kind,
|
|
|
|
|
['service_suspend', 'service_resume', 'network_suspend', 'network_resume'], true)) {
|
|
|
|
|
// Record a visible policy error. The lifecycle reconciler
|
|
|
|
|
// retries this desired state with bounded exponential backoff
|
|
|
|
|
// after a temporary host/SSH failure is repaired.
|
|
|
|
|
$instance = CloudInstance::where('id', $job->instance_id)->lockForUpdate()->first();
|
|
|
|
|
if ($instance !== null) {
|
|
|
|
|
$expected = [
|
|
|
|
|
'service_suspend' => ['suspending_expired', 'purge_stopping'],
|
|
|
|
|
'service_resume' => ['resume_pending', 'resuming'],
|
|
|
|
|
'network_suspend' => ['blocking', 'blocking_expired'],
|
|
|
|
|
'network_resume' => ['restore_pending', 'restoring'],
|
|
|
|
|
][$job->kind];
|
|
|
|
|
if ($job->kind === 'service_suspend'
|
|
|
|
|
&& in_array((string) $instance->service_status, $expected, true)) {
|
|
|
|
|
if ((string) $instance->status === 'deleting'
|
|
|
|
|
&& (string) $instance->service_status === 'purge_stopping') {
|
|
|
|
|
// Stopping is a replayable target state. Leave the
|
|
|
|
|
// irreversible deletion latch in place while the
|
|
|
|
|
// lifecycle daemon applies its bounded retry.
|
|
|
|
|
$instance->service_status = 'purge_stop_failed';
|
|
|
|
|
} else {
|
|
|
|
|
$instance->service_status = 'suspend_failed';
|
|
|
|
|
}
|
|
|
|
|
} elseif ($job->kind === 'service_resume'
|
|
|
|
|
&& in_array((string) $instance->service_status, $expected, true)) {
|
|
|
|
|
$instance->service_status = 'resume_failed';
|
|
|
|
|
} elseif ($job->kind === 'network_suspend'
|
|
|
|
|
&& in_array((string) $instance->network_status, $expected, true)) {
|
|
|
|
|
$instance->network_status = 'block_failed';
|
|
|
|
|
} elseif ($job->kind === 'network_resume'
|
|
|
|
|
&& in_array((string) $instance->network_status, $expected, true)) {
|
|
|
|
|
$instance->network_status = 'restore_failed';
|
|
|
|
|
}
|
|
|
|
|
$instance->updated_at = time();
|
|
|
|
|
$instance->save();
|
|
|
|
|
}
|
|
|
|
|
} elseif ($job->instance_id !== null
|
|
|
|
|
&& ($status === 'succeeded'
|
|
|
|
|
|| ($status === 'uncertain' && $job->kind === 'reinstall'
|
|
|
|
|
&& self::reinstallNeedsReview($operation)))) {
|
|
|
|
|
$instance = CloudInstance::where('id', $job->instance_id)->lockForUpdate()->first();
|
|
|
|
|
if ($instance !== null) {
|
|
|
|
|
if (in_array((string) $job->kind, ['delete', 'expiry_delete'], true)) {
|
|
|
|
|
$instance->status = 'deleted';
|
|
|
|
|
$instance->credential_cipher = null;
|
|
|
|
|
if ($instance->order_id !== null) {
|
|
|
|
|
$provisionOrder = CloudOrder::where('id', $instance->order_id)
|
|
|
|
|
->lockForUpdate()->first();
|
|
|
|
|
if ($provisionOrder !== null) {
|
|
|
|
|
CloudPlacementService::release(
|
|
|
|
|
$provisionOrder, 'instance_deleted', true
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} elseif ($job->kind === 'reinstall') {
|
|
|
|
|
if (isset($payload['credential'])
|
|
|
|
|
&& is_array($payload['credential'])) {
|
|
|
|
|
$instance->credential_cipher = CloudSecrets::encrypt(
|
|
|
|
|
$payload['credential']
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
// The replacement template changes the static OS
|
|
|
|
|
// metadata even though IP and disk capacity remain
|
|
|
|
|
// stable. Invalidate only those cached fields so the
|
|
|
|
|
// next detail render performs one bounded engine
|
|
|
|
|
// metadata read and persists the new system label.
|
|
|
|
|
$decoded = json_decode((string) $instance->spec_json, true);
|
|
|
|
|
if (is_array($decoded) && !array_is_list($decoded)) {
|
|
|
|
|
$templateId = is_array($payload['body'] ?? null)
|
|
|
|
|
? ($payload['body']['template_id'] ?? null) : null;
|
|
|
|
|
if (is_string($templateId)
|
|
|
|
|
&& preg_match('/^[A-Za-z0-9][A-Za-z0-9_.-]{0,79}$/D',
|
|
|
|
|
$templateId) === 1) {
|
|
|
|
|
$decoded['template_id'] = $templateId;
|
|
|
|
|
}
|
|
|
|
|
unset($decoded['os_source'], $decoded['os_name'],
|
|
|
|
|
$decoded['install_iso'], $decoded['login_name'],
|
|
|
|
|
$decoded['engine_metadata_version']);
|
|
|
|
|
$instance->spec_json = json_encode($decoded,
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE);
|
|
|
|
|
}
|
|
|
|
|
} elseif ($job->kind === 'instance_settings') {
|
|
|
|
|
// The engine must acknowledge the controlled seed
|
|
|
|
|
// replacement before PHP changes any cached metadata.
|
|
|
|
|
// Passwords and public keys are intentionally never
|
|
|
|
|
// copied into spec_json or result_json.
|
|
|
|
|
if (!is_array($result) || ($result['settings_changed'] ?? false) !== true) {
|
|
|
|
|
throw new CloudError('引擎成功结果缺少已确认的实例设置', 502);
|
|
|
|
|
}
|
|
|
|
|
if (isset($payload['credential'])
|
|
|
|
|
&& is_array($payload['credential'])) {
|
|
|
|
|
$instance->credential_cipher = CloudSecrets::encrypt(
|
|
|
|
|
$payload['credential']
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
$requested = is_array($payload['body'] ?? null)
|
|
|
|
|
? $payload['body'] : [];
|
|
|
|
|
$decoded = json_decode((string) $instance->spec_json, true);
|
|
|
|
|
if (!is_array($decoded) || array_is_list($decoded)) {
|
|
|
|
|
throw new CloudError('虚拟机配置记录损坏,无法保存实例设置', 409);
|
|
|
|
|
}
|
|
|
|
|
if (array_key_exists('hostname', $requested)) {
|
|
|
|
|
$confirmed = $result['hostname'] ?? null;
|
|
|
|
|
if (!is_string($confirmed)
|
|
|
|
|
|| $confirmed !== $requested['hostname']
|
|
|
|
|
|| preg_match('/^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*$/D', $confirmed) !== 1
|
|
|
|
|
|| strlen($confirmed) > 253) {
|
|
|
|
|
throw new CloudError('引擎成功结果缺少已确认的主机名', 502);
|
|
|
|
|
}
|
|
|
|
|
$decoded['guest_hostname'] = $confirmed;
|
|
|
|
|
}
|
|
|
|
|
$instance->spec_json = json_encode($decoded,
|
|
|
|
|
JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES
|
|
|
|
|
| JSON_UNESCAPED_UNICODE);
|
|
|
|
|
} elseif ($job->kind === 'service_suspend'
|
|
|
|
|
&& in_array((string) $instance->service_status,
|
|
|
|
|
['suspending_expired', 'resume_pending', 'resuming',
|
|
|
|
|
'expired_grace', 'purge_stopping'], true)) {
|
|
|
|
|
// Preserve proof that the retired expiry policy
|
|
|
|
|
// touched a previously running guest. Network must be
|
|
|
|
|
// blocked before the lifecycle daemon safely restarts
|
|
|
|
|
// it; a tenant-stopped guest has no marker and stays
|
|
|
|
|
// stopped.
|
|
|
|
|
if (($result['service_was_running'] ?? false) === true
|
|
|
|
|
&& (string) $instance->service_status !== 'purge_stopping') {
|
|
|
|
|
$instance->suspended_at = time();
|
|
|
|
|
}
|
|
|
|
|
if ((string) $instance->status === 'deleting'
|
|
|
|
|
&& (string) $instance->service_status === 'purge_stopping') {
|
|
|
|
|
$instance->service_status = 'purge_deleting';
|
|
|
|
|
} elseif (($payload['suspend_reason'] ?? null) === 'expired') {
|
|
|
|
|
$expired = $instance->expires_at !== null
|
|
|
|
|
&& (int) $instance->expires_at <= time();
|
|
|
|
|
$instance->service_status = $instance->suspended_at !== null
|
|
|
|
|
? 'resume_pending' : ($expired ? 'expired_grace' : 'active');
|
|
|
|
|
} elseif ($instance->service_status === 'suspending_expired') {
|
|
|
|
|
$instance->service_status = 'suspended_expired';
|
|
|
|
|
}
|
|
|
|
|
} elseif ($job->kind === 'service_resume'
|
|
|
|
|
&& in_array((string) $instance->service_status, ['resuming', 'resume_pending'], true)) {
|
|
|
|
|
$instance->service_status = $instance->expires_at !== null
|
|
|
|
|
&& (int) $instance->expires_at <= time()
|
|
|
|
|
? 'expired_grace' : 'active';
|
|
|
|
|
$instance->suspended_at = null;
|
|
|
|
|
} elseif ($job->kind === 'network_suspend'
|
|
|
|
|
&& in_array((string) $instance->network_status,
|
|
|
|
|
['blocking', 'blocking_expired'], true)) {
|
|
|
|
|
$instance->network_status = ($payload['suspend_reason'] ?? '') === 'expired'
|
|
|
|
|
? 'blocked_expired' : 'blocked_traffic';
|
|
|
|
|
$instance->network_suspended_at = time();
|
|
|
|
|
} elseif ($job->kind === 'network_resume'
|
|
|
|
|
&& in_array((string) $instance->network_status, ['restoring', 'restore_pending'], true)) {
|
|
|
|
|
$instance->network_status = 'active';
|
|
|
|
|
$instance->network_suspended_at = null;
|
|
|
|
|
}
|
|
|
|
|
// Successful power/reinstall/policy operations often
|
|
|
|
|
// include the resulting VM state. Persist that bounded
|
|
|
|
|
// observation so the next page render is immediate;
|
|
|
|
|
// missing state is deliberately left for vm-states.
|
|
|
|
|
self::recordRuntime($instance, is_array($result) ? $result : [],
|
|
|
|
|
(string) $job->kind, $payload);
|
|
|
|
|
$instance->updated_at = time();
|
|
|
|
|
$instance->save();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
$job->status = $status;
|
|
|
|
|
if (in_array($status, ['succeeded', 'failed'], true)) {
|
|
|
|
|
// Terminal jobs never need credentials again. Preserve only
|
|
|
|
|
// a sanitized audit payload so an old create/reinstall row
|
|
|
|
|
// cannot remain a second recoverable copy of the root password.
|
|
|
|
|
$job->payload_cipher = CloudSecrets::encrypt(
|
|
|
|
|
CloudInput::safeResult($payload)
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
$job->result_json = $safe === null ? null : json_encode($safe, JSON_THROW_ON_ERROR);
|
|
|
|
|
$job->error_code = CloudOperationErrorCatalog::workerSummary(
|
|
|
|
|
(string) $job->kind, $status, $operation, $order !== null,
|
|
|
|
|
self::payloadAction($payload)
|
|
|
|
|
);
|
|
|
|
|
$job->lease_token = null;
|
|
|
|
|
$job->lease_until = 0;
|
|
|
|
|
$job->available_at = time() + 15;
|
|
|
|
|
$job->updated_at = time();
|
|
|
|
|
$job->save();
|
|
|
|
|
if ($order !== null) {
|
|
|
|
|
if ($job->kind === 'create' && $status === 'uncertain') {
|
|
|
|
|
CloudPlacementService::markUncertain($order);
|
|
|
|
|
}
|
|
|
|
|
$order->status = $status;
|
|
|
|
|
$order->updated_at = time();
|
|
|
|
|
$order->save();
|
|
|
|
|
}
|
|
|
|
|
}, 3);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function validSystemCleanup(CloudJob $job, array $payload): bool
|
|
|
|
|
{
|
|
|
|
|
if ($job->instance_id === null || ($payload['body'] ?? null) !== []) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
$instance = CloudInstance::find($job->instance_id);
|
|
|
|
|
if ($instance === null || (string) $instance->status !== 'deleting'
|
|
|
|
|
|| (string) $instance->service_status !== 'purge_deleting'
|
|
|
|
|
|| (int) $job->user_id !== (int) $instance->user_id
|
|
|
|
|
|| (int) $job->host_id !== (int) $instance->host_id
|
|
|
|
|
|| (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
return ($payload['method'] ?? null) === 'DELETE'
|
|
|
|
|
&& ($payload['resource'] ?? null) === 'vms/'
|
|
|
|
|
. rawurlencode((string) $instance->uuid) . '?delete_disks=true';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function validSystemPolicy(CloudJob $job, array $payload): bool
|
|
|
|
|
{
|
|
|
|
|
if ($job->instance_id === null || !is_array($payload['body'] ?? null)) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
$instance = CloudInstance::find($job->instance_id);
|
|
|
|
|
if ($instance === null || (int) $job->user_id !== (int) $instance->user_id
|
|
|
|
|
|| (int) $job->host_id !== (int) $instance->host_id
|
|
|
|
|
|| (int) ($payload['owner_id'] ?? 0) !== (int) $instance->user_id) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
$suffix = [
|
|
|
|
|
'service_suspend' => 'service-suspend',
|
|
|
|
|
'service_resume' => 'service-resume',
|
|
|
|
|
'network_suspend' => 'network-suspend',
|
|
|
|
|
'network_resume' => 'network-resume',
|
|
|
|
|
][$job->kind] ?? null;
|
|
|
|
|
return $suffix !== null && ($payload['method'] ?? null) === 'POST'
|
|
|
|
|
&& ($payload['resource'] ?? null) === 'vms/'
|
|
|
|
|
. rawurlencode((string) $instance->uuid) . '/' . $suffix;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function payloadAction(array $payload): ?string
|
|
|
|
|
{
|
|
|
|
|
$body = $payload['body'] ?? null;
|
|
|
|
|
$action = is_array($body) && is_string($body['action'] ?? null)
|
|
|
|
|
? $body['action'] : null;
|
|
|
|
|
return CloudOperationCatalog::isPowerAction($action ?? '') ? $action : null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Persist only a successful, allow-listed power observation. Runtime
|
|
|
|
|
* cache failures are isolated from the durable job outcome: the engine
|
|
|
|
|
* result and billing state remain authoritative even during a rolling
|
|
|
|
|
* schema upgrade where the cache table may not exist yet.
|
|
|
|
|
*/
|
|
|
|
|
private static function recordRuntime(CloudInstance $instance, array $result,
|
|
|
|
|
string $kind, array $payload): void
|
|
|
|
|
{
|
|
|
|
|
try {
|
|
|
|
|
if (in_array($kind, ['delete', 'expiry_delete'], true)) {
|
|
|
|
|
CloudInstanceRuntimeService::forget((int) $instance->id);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
$source = is_array($result['vm'] ?? null) ? $result['vm'] : $result;
|
|
|
|
|
$state = $source['state'] ?? ($source['power_state'] ?? null);
|
|
|
|
|
$action = $kind === 'action' ? self::payloadAction($payload) : null;
|
|
|
|
|
$normalized = CloudInstanceRuntimeService::normalizeState($state);
|
|
|
|
|
if ($kind === 'action') {
|
|
|
|
|
// A successful graceful-shutdown job only confirms that the
|
|
|
|
|
// request was accepted. libvirt may still report running (or
|
|
|
|
|
// omit the state) until the guest completes ACPI shutdown.
|
|
|
|
|
// Preserve that distinction instead of publishing either a
|
|
|
|
|
// new fresh-running observation or a fabricated stopped one.
|
|
|
|
|
$state = CloudInstanceRuntimeService::stateAfterSuccessfulAction(
|
|
|
|
|
$state, $action
|
|
|
|
|
);
|
|
|
|
|
} elseif ($normalized === 'unknown' && $kind === 'reinstall') {
|
|
|
|
|
// The engine starts the replacement definition before
|
|
|
|
|
// returning a successful reinstall result. Older engine
|
|
|
|
|
// responses may omit the nested vm.state, so preserve
|
|
|
|
|
// the same successful postcondition explicitly.
|
|
|
|
|
$state = 'running';
|
|
|
|
|
}
|
|
|
|
|
$instanceId = (int) $instance->id;
|
|
|
|
|
$hostId = (int) $instance->host_id;
|
|
|
|
|
$uuid = (string) $instance->uuid;
|
|
|
|
|
if ($kind === 'create') {
|
|
|
|
|
CloudInstanceRuntimeService::seed($instanceId, $hostId, $uuid);
|
|
|
|
|
}
|
|
|
|
|
if (CloudInstanceRuntimeService::normalizeState($state) !== 'unknown') {
|
|
|
|
|
CloudInstanceRuntimeService::markObserved(
|
|
|
|
|
$instanceId, $hostId, $uuid, $state
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
} catch (\Throwable $error) {
|
|
|
|
|
error_log('cloud runtime observation write unavailable: ' . get_class($error));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function exceptionDiagnostic(\Throwable $error, bool $dispatched,
|
|
|
|
|
?string $kind, bool $hasOrder, ?string $action): string
|
|
|
|
|
{
|
|
|
|
|
$operation = [];
|
|
|
|
|
if ($error instanceof CloudError) {
|
|
|
|
|
// CloudError text is engine-/host-controlled too. The catalog
|
|
|
|
|
// keeps only exact reviewed details and otherwise returns a
|
|
|
|
|
// stable generic code for the durable job row.
|
|
|
|
|
$operation['error'] = $error->getMessage();
|
|
|
|
|
} elseif ($error instanceof \GuzzleHttp\Exception\TransferException) {
|
|
|
|
|
$operation['error'] = '引擎连接中断或操作超时,正在核对结果';
|
|
|
|
|
}
|
|
|
|
|
$status = $dispatched ? 'uncertain' : 'failed';
|
|
|
|
|
return CloudOperationErrorCatalog::workerSummary($kind, $status,
|
|
|
|
|
$operation, $hasOrder, $action)
|
|
|
|
|
?? CloudOperationErrorCatalog::userMessage($kind, $status, null,
|
|
|
|
|
$hasOrder, $action)
|
|
|
|
|
?? '虚拟机操作失败,请稍后重试';
|
|
|
|
|
}
|
|
|
|
|
}
|