You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudLifecycleService.php

443 lines
23 KiB
PHTML

<?php
namespace App\Services\Cloud;
use App\Models\CloudInstance;
use App\Models\CloudInvoice;
use App\Models\CloudJob;
use App\Models\CloudServicePeriod;
final class CloudLifecycleService
{
private array $observedErrors = [];
public function __construct(private ?CloudRuntimeHeartbeat $heartbeat = null)
{
}
public function observationErrors(): array
{
return array_keys($this->observedErrors);
}
private function observeFailure(string $code): void
{
$this->observedErrors[$code] = true;
$this->heartbeat?->error($code);
}
public function tick(int $limit = 200, bool $skipEngineObservations = false): int
{
$this->observedErrors = [];
if ($skipEngineObservations) {
$this->observeFailure('engine_readiness_timeout');
}
$batchSize = max(1, min(1000, $limit));
$this->heartbeat?->busy('provision_expiry');
try {
CloudInvoiceService::expireProvision($batchSize);
} catch (\Throwable $error) {
$this->observeFailure('provision_expiry_failed');
// Releasing an abandoned provisioning reservation is database-only
// and must not stop instance expiry or traffic enforcement.
error_log('cloud provisioning invoice expiry unavailable: '
. get_class($error));
}
$this->heartbeat?->busy('invoice_generation');
try {
CloudInvoiceService::issueDue($batchSize);
foreach (CloudInvoiceService::observationErrors('issue_due') as $code) {
$this->observeFailure($code);
}
} catch (\Throwable $error) {
$this->observeFailure('invoice_generation_failed');
// Invoice creation is database-only and must not stop policy.
error_log('cloud invoice generation unavailable: ' . get_class($error));
}
$this->heartbeat?->busy('traffic_sync');
try {
CloudTrafficService::sync($batchSize);
} catch (\Throwable $error) {
$this->observeFailure('traffic_sync_failed');
// Unknown traffic is fail-open. Time expiry remains enforceable.
error_log('cloud traffic sync unavailable: ' . get_class($error));
}
// Power state is an observation cache, not a billing decision. Keep
// it on the lifecycle cadence so list pages can render immediately
// from the last successful host batch without waiting on libvirt.
if (!$skipEngineObservations) {
$this->heartbeat?->busy('runtime_status_sync');
try {
$runtime = (new CloudInstanceRuntimeSyncService())->sync($batchSize);
foreach (array_unique($runtime['errors'] ?? []) as $code) {
$this->observeFailure((string) $code);
}
} catch (\Throwable $error) {
$this->observeFailure('runtime_status_sync_failed');
// A status poll must never prevent invoice or expiry policy work.
error_log('cloud runtime status sync unavailable: ' . get_class($error));
}
}
$queued = 0;
$this->heartbeat?->busy('instance_reconcile');
CloudInstance::where('status', '<>', 'deleted')
->where('service_status', '<>', 'exempt')
->select('id')->chunkById($batchSize, function ($instances) use (&$queued) {
foreach ($instances as $instance) {
$this->heartbeat?->busy('instance_reconcile');
try {
$queued += $this->reconcile((int) $instance->id) ? 1 : 0;
} catch (\Throwable $error) {
$this->observeFailure('instance_reconcile_failed');
// One corrupt or temporarily contended record must not
// starve every higher instance ID on every daemon pass.
error_log('cloud lifecycle instance ' . (int) $instance->id
. ' unavailable: ' . get_class($error));
}
}
});
return $queued;
}
private function reconcile(int $instanceId): bool
{
return (new CloudInstance())->getConnection()->transaction(function () use ($instanceId) {
$now = time();
$instance = CloudInstance::where('id', $instanceId)->lockForUpdate()->first();
if ($instance === null || in_array((string) $instance->status,
['deleted', 'delete_failed'], true)
|| $instance->service_status === 'exempt') {
return false;
}
if ((string) $instance->service_status === 'purge_failed') {
return false;
}
$period = $instance->current_period_id === null ? null
: CloudServicePeriod::where('id', $instance->current_period_id)
->where('instance_id', $instance->id)->lockForUpdate()->first();
if ($period !== null && in_array((string) $period->status, ['active', 'exhausted'], true)) {
$this->updateUsage($instance, $period, $now);
if ((int) $period->ends_at <= $now) {
$period->status = 'finished';
$period->updated_at = $now;
$period->save();
$period = null;
$instance->current_period_id = null;
}
} else {
$period = null;
$instance->current_period_id = null;
}
if ($period === null) {
// The lifecycle daemon may be stopped for longer than one
// prepaid period. Never briefly activate an already expired
// historical period (which would also restore its network).
CloudServicePeriod::where('instance_id', $instance->id)
->where('status', 'scheduled')->where('ends_at', '<=', $now)
->update(['status' => 'finished', 'updated_at' => $now]);
$period = CloudServicePeriod::where('instance_id', $instance->id)
->where('status', 'scheduled')->where('starts_at', '<=', $now)
->where('ends_at', '>', $now)
->orderBy('starts_at')->orderBy('id')->lockForUpdate()->first();
if ($period !== null) {
$period->status = 'active';
$period->activated_at = $now;
$baseline = CloudTrafficService::periodBaseline(
$instance, (int) $period->starts_at);
if ($baseline === null
&& (int) $instance->traffic_synced_at >= (int) $period->starts_at) {
$baseline = [
'download' => max(0, (int) $instance->lifetime_download_bytes),
'upload' => max(0, (int) $instance->lifetime_upload_bytes),
];
}
$period->download_base_bytes = $baseline['download'] ?? null;
$period->upload_base_bytes = $baseline['upload'] ?? null;
$period->download_used_bytes = $baseline === null ? 0
: max(0, (int) $instance->lifetime_download_bytes
- (int) $baseline['download']);
$period->upload_used_bytes = $baseline === null ? 0
: max(0, (int) $instance->lifetime_upload_bytes
- (int) $baseline['upload']);
$period->updated_at = $now;
$period->save();
$instance->current_period_id = $period->id;
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
$instance->traffic_download_bytes = (int) $period->download_used_bytes;
$instance->traffic_upload_bytes = (int) $period->upload_used_bytes;
if (!in_array((string) $instance->service_status, ['active', 'exempt'], true)) {
$legacySuspend = $instance->suspended_at !== null
|| CloudJob::where('instance_id', $instance->id)
->where('kind', 'service_suspend')
->whereIn('status', ['queued', 'running', 'uncertain'])->exists();
$instance->service_status = $legacySuspend ? 'resume_pending' : 'active';
}
if ((string) $instance->network_status !== 'active') {
$instance->network_status = 'restore_pending';
$instance->network_suspended_at = null;
}
}
}
if ($period !== null) {
$usageKnown = $this->updateUsage($instance, $period, $now);
$used = (int) $period->download_used_bytes + (int) $period->upload_used_bytes;
$quota = max(0, (int) $period->traffic_quota_bytes);
if ($usageKnown && $quota > 0 && $used >= $quota) {
if ((string) $period->status !== 'exhausted') {
$period->status = 'exhausted';
$period->updated_at = $now;
$period->save();
}
if ((string) $instance->network_status === 'block_failed'
&& $this->policyRetryReady($instance, 'network_suspend', $now)) {
$instance->network_status = 'blocking';
} elseif (!in_array((string) $instance->network_status,
['blocking', 'blocked_traffic', 'block_failed'], true)) {
$instance->network_status = 'blocking';
}
} elseif ($usageKnown) {
if ((string) $instance->network_status === 'restore_failed'
&& $this->policyRetryReady($instance, 'network_resume', $now)) {
$instance->network_status = 'restore_pending';
$instance->network_suspended_at = null;
} elseif (!in_array((string) $instance->network_status,
['active', 'restore_failed'], true)) {
$instance->network_status = 'restore_pending';
$instance->network_suspended_at = null;
}
}
if ((string) $instance->service_status === 'resume_failed'
&& $this->policyRetryReady($instance, 'service_resume', $now)) {
$instance->service_status = 'resume_pending';
}
} elseif ($instance->expires_at !== null && (int) $instance->expires_at <= $now) {
$invoice = CloudInvoice::where('instance_id', $instance->id)
->where('renews_expires_at', $instance->expires_at)
->where('status', '<>', 'void')->orderByDesc('id')
->lockForUpdate()->first();
$pastGrace = $invoice !== null
&& in_array((string) $invoice->status,
['open', 'overdue', 'expired'], true)
&& CloudInvoiceService::paymentDeadlinePassed($invoice, $now);
if ($pastGrace) {
if (in_array((string) $invoice->status, ['open', 'overdue'], true)) {
$invoice->status = 'expired';
$invoice->updated_at = $now;
$invoice->save();
}
$instance->status = 'deleting';
if ((string) $instance->service_status === 'purge_stop_failed') {
if ($this->policyRetryReady($instance, 'service_suspend', $now)) {
$instance->service_status = 'purge_stopping';
}
} elseif (!in_array((string) $instance->service_status,
['purge_stopping', 'purge_deleting', 'purge_failed'], true)) {
$instance->service_status = 'purge_stopping';
}
} elseif (!in_array((string) $instance->service_status,
['purge_stopping', 'purge_stop_failed', 'purge_deleting', 'purge_failed'], true)) {
// Expiry keeps QEMU running throughout the three-day
// payment grace period. Only its NIC links are persisted
// down; a paid renewal restores those links. During an
// online upgrade, a durable suspended_at marker proves
// that the retired policy stopped a previously running
// guest, which must be restarted only after link-down.
$instance->service_status = $instance->suspended_at === null
? 'expired_grace' : 'resume_pending';
$network = (string) $instance->network_status;
if ($network === 'blocked_traffic') {
$instance->network_status = 'blocked_expired';
} elseif ($network === 'block_failed') {
if ($this->policyRetryReady($instance, 'network_suspend', $now)) {
$instance->network_status = 'blocking_expired';
}
} elseif (!in_array($network,
['blocking', 'blocking_expired', 'blocked_expired'], true)) {
$instance->network_status = 'blocking_expired';
}
}
}
$instance->updated_at = $now;
$instance->save();
if ($this->hasConflictingJob($instance)) {
return false;
}
$service = (string) $instance->service_status;
$network = (string) $instance->network_status;
if ($service === 'purge_stopping') {
return $this->enqueue($instance, 'service_suspend', 'service-suspend', 'purge');
}
if ($service === 'purge_deleting') {
$instance->status = 'deleting';
$instance->updated_at = $now;
$instance->save();
CloudJobService::enqueueExpiryDelete($instance);
return true;
}
if ($network === 'blocking_expired') {
return $this->enqueue($instance, 'network_suspend', 'network-suspend', 'expired');
}
if ($network === 'blocking') {
return $this->enqueue($instance, 'network_suspend', 'network-suspend', 'traffic');
}
if (in_array($service, ['resume_pending', 'resuming'], true)) {
$legacyExpiryResume = $instance->expires_at !== null
&& (int) $instance->expires_at <= $now;
if ($legacyExpiryResume && $network !== 'blocked_expired') {
return false;
}
$instance->service_status = 'resuming';
$instance->save();
return $this->enqueue($instance, 'service_resume', 'service-resume', null,
['start_if_suspended' => $instance->suspended_at !== null]);
}
if (in_array($network, ['restore_pending', 'restoring'], true)) {
$instance->network_status = 'restoring';
$instance->save();
return $this->enqueue($instance, 'network_resume', 'network-resume');
}
return false;
}, 3);
}
private function hasConflictingJob(CloudInstance $instance): bool
{
// Match reconcile()'s target ordering: purge stop/delete wins before
// network policy. Only a tick that will actually enqueue
// network_suspend may defer snapshot serialization to the engine's
// durable queued-versus-running decision.
$failSafeSuspend = !in_array((string) $instance->service_status,
['purge_stopping', 'purge_deleting'], true)
&& in_array((string) $instance->network_status,
['blocking', 'blocking_expired'], true);
$snapshotKinds = [
'snapshot_create', 'snapshot_restore', 'snapshot_delete',
];
// Firewall changes never alter power, link state or resource
// ownership. Let lifecycle policy enter its own durable queue even if
// a firewall operation is queued, running or awaiting exact evidence.
// The engine's per-VM target lock still serializes the actual libvirt
// mutations, while the worker's pre-dispatch policy check cancels a
// stale queued tenant change once expiry enforcement has started.
$nonBlockingActiveKinds = ['firewall_apply'];
$nonBlockingUncertainKinds = ['action', 'firewall_apply'];
if ($failSafeSuspend) {
// A lost snapshot response must not permanently defeat quota or
// expiry enforcement. The engine still rejects these target-state
// suspends while a snapshot executor or libvirt BACKUP is active;
// once neither is active, an old durable snapshot uncertainty is
// not allowed to keep networking up. Purge shutdown remains
// serialized because an uncertain restore reconciler may still
// need to recover the VM's original power state.
// Let the target-state request reach the engine even while PHP is
// still polling a snapshot job. The engine may cancel only a
// durably queued snapshot that has not touched libvirt; a running
// backup remains protected and returns 409 for lifecycle retry.
$nonBlockingActiveKinds = array_merge(
$nonBlockingActiveKinds, $snapshotKinds
);
$nonBlockingUncertainKinds = array_merge(
$nonBlockingUncertainKinds, $snapshotKinds
);
}
return CloudJob::where('instance_id', $instance->id)
->where(function ($query) use ($nonBlockingActiveKinds,
$nonBlockingUncertainKinds) {
$query->where(function ($active) use ($nonBlockingActiveKinds) {
$active->whereIn('status', ['queued', 'running']);
if ($nonBlockingActiveKinds !== []) {
$active->whereNotIn('kind', $nonBlockingActiveKinds);
}
})
->orWhere(function ($uncertain) use ($nonBlockingUncertainKinds) {
// A reset/shutdown action does not change resource
// ownership, so expiry policy may still converge around
// an old uncertain action. Delete is different: until
// the original destructive operation is reconciled it
// must serialize every later lifecycle mutation.
$uncertain->where('status', 'uncertain')
->whereNotIn('kind', $nonBlockingUncertainKinds);
});
})->exists();
}
private function policyRetryReady(CloudInstance $instance, string $kind, int $now): bool
{
$failed = CloudJob::where('instance_id', $instance->id)
->where('kind', $kind)->where('status', 'failed')
->orderByDesc('updated_at')->orderByDesc('id')->first();
if ($failed === null) {
return true;
}
$recentFailures = CloudJob::where('instance_id', $instance->id)
->where('kind', $kind)->where('status', 'failed')
->where('updated_at', '>=', $now - 3600)->count();
// Retry temporary host/SSH failures without creating a 15-second job
// storm. Enforcement always converges again after an administrator
// repairs the host; the delay is capped at five minutes.
$exponent = min(5, max(0, $recentFailures - 1));
$delay = min(300, 15 * (1 << $exponent));
return (int) $failed->updated_at + $delay <= $now;
}
private function updateUsage(CloudInstance $instance, CloudServicePeriod $period, int $now): bool
{
if ($period->download_base_bytes === null || $period->upload_base_bytes === null) {
// At a period boundary, an old cached total would charge bytes
// from the previous period to the new one. Wait for the first
// successful sample at/after the boundary and fail open until it
// establishes both baselines.
if ((int) $instance->traffic_synced_at < (int) $period->starts_at) {
$period->download_base_bytes = null;
$period->upload_base_bytes = null;
$period->download_used_bytes = 0;
$period->upload_used_bytes = 0;
$period->updated_at = $now;
$period->save();
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
$instance->traffic_download_bytes = 0;
$instance->traffic_upload_bytes = 0;
return false;
}
$period->download_base_bytes = max(0, (int) $instance->lifetime_download_bytes);
$period->upload_base_bytes = max(0, (int) $instance->lifetime_upload_bytes);
}
$download = max(0, (int) $instance->lifetime_download_bytes - (int) $period->download_base_bytes);
$upload = max(0, (int) $instance->lifetime_upload_bytes - (int) $period->upload_base_bytes);
$period->download_used_bytes = $download;
$period->upload_used_bytes = $upload;
$period->updated_at = $now;
$period->save();
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
$instance->traffic_download_bytes = $download;
$instance->traffic_upload_bytes = $upload;
return true;
}
private function enqueue(CloudInstance $instance, string $kind, string $suffix,
?string $reason = null, array $body = []): bool
{
$job = new CloudJob();
$job->operation_id = bin2hex(random_bytes(16));
$job->instance_id = $instance->id;
$job->user_id = $instance->user_id;
$job->host_id = $instance->host_id;
$job->kind = $kind;
$job->status = 'queued';
$job->payload_cipher = CloudSecrets::encrypt([
'method' => 'POST',
'resource' => 'vms/' . rawurlencode($instance->uuid) . '/' . $suffix,
'body' => $body, 'owner_id' => (int) $instance->user_id,
'system_policy' => true, 'suspend_reason' => $reason,
]);
$job->available_at = $job->created_at = $job->updated_at = time();
$job->save();
return true;
}
}