You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
443 lines
23 KiB
PHP
443 lines
23 KiB
PHP
<?php
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInvoice;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudServicePeriod;
|
|
|
|
final class CloudLifecycleService
|
|
{
|
|
private array $observedErrors = [];
|
|
|
|
public function __construct(private ?CloudRuntimeHeartbeat $heartbeat = null)
|
|
{
|
|
}
|
|
|
|
public function observationErrors(): array
|
|
{
|
|
return array_keys($this->observedErrors);
|
|
}
|
|
|
|
private function observeFailure(string $code): void
|
|
{
|
|
$this->observedErrors[$code] = true;
|
|
$this->heartbeat?->error($code);
|
|
}
|
|
|
|
public function tick(int $limit = 200, bool $skipEngineObservations = false): int
|
|
{
|
|
$this->observedErrors = [];
|
|
if ($skipEngineObservations) {
|
|
$this->observeFailure('engine_readiness_timeout');
|
|
}
|
|
$batchSize = max(1, min(1000, $limit));
|
|
$this->heartbeat?->busy('provision_expiry');
|
|
try {
|
|
CloudInvoiceService::expireProvision($batchSize);
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('provision_expiry_failed');
|
|
// Releasing an abandoned provisioning reservation is database-only
|
|
// and must not stop instance expiry or traffic enforcement.
|
|
error_log('cloud provisioning invoice expiry unavailable: '
|
|
. get_class($error));
|
|
}
|
|
$this->heartbeat?->busy('invoice_generation');
|
|
try {
|
|
CloudInvoiceService::issueDue($batchSize);
|
|
foreach (CloudInvoiceService::observationErrors('issue_due') as $code) {
|
|
$this->observeFailure($code);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('invoice_generation_failed');
|
|
// Invoice creation is database-only and must not stop policy.
|
|
error_log('cloud invoice generation unavailable: ' . get_class($error));
|
|
}
|
|
$this->heartbeat?->busy('traffic_sync');
|
|
try {
|
|
CloudTrafficService::sync($batchSize);
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('traffic_sync_failed');
|
|
// Unknown traffic is fail-open. Time expiry remains enforceable.
|
|
error_log('cloud traffic sync unavailable: ' . get_class($error));
|
|
}
|
|
// Power state is an observation cache, not a billing decision. Keep
|
|
// it on the lifecycle cadence so list pages can render immediately
|
|
// from the last successful host batch without waiting on libvirt.
|
|
if (!$skipEngineObservations) {
|
|
$this->heartbeat?->busy('runtime_status_sync');
|
|
try {
|
|
$runtime = (new CloudInstanceRuntimeSyncService())->sync($batchSize);
|
|
foreach (array_unique($runtime['errors'] ?? []) as $code) {
|
|
$this->observeFailure((string) $code);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('runtime_status_sync_failed');
|
|
// A status poll must never prevent invoice or expiry policy work.
|
|
error_log('cloud runtime status sync unavailable: ' . get_class($error));
|
|
}
|
|
}
|
|
$queued = 0;
|
|
$this->heartbeat?->busy('instance_reconcile');
|
|
CloudInstance::where('status', '<>', 'deleted')
|
|
->where('service_status', '<>', 'exempt')
|
|
->select('id')->chunkById($batchSize, function ($instances) use (&$queued) {
|
|
foreach ($instances as $instance) {
|
|
$this->heartbeat?->busy('instance_reconcile');
|
|
try {
|
|
$queued += $this->reconcile((int) $instance->id) ? 1 : 0;
|
|
} catch (\Throwable $error) {
|
|
$this->observeFailure('instance_reconcile_failed');
|
|
// One corrupt or temporarily contended record must not
|
|
// starve every higher instance ID on every daemon pass.
|
|
error_log('cloud lifecycle instance ' . (int) $instance->id
|
|
. ' unavailable: ' . get_class($error));
|
|
}
|
|
}
|
|
});
|
|
return $queued;
|
|
}
|
|
|
|
private function reconcile(int $instanceId): bool
|
|
{
|
|
return (new CloudInstance())->getConnection()->transaction(function () use ($instanceId) {
|
|
$now = time();
|
|
$instance = CloudInstance::where('id', $instanceId)->lockForUpdate()->first();
|
|
if ($instance === null || in_array((string) $instance->status,
|
|
['deleted', 'delete_failed'], true)
|
|
|| $instance->service_status === 'exempt') {
|
|
return false;
|
|
}
|
|
if ((string) $instance->service_status === 'purge_failed') {
|
|
return false;
|
|
}
|
|
$period = $instance->current_period_id === null ? null
|
|
: CloudServicePeriod::where('id', $instance->current_period_id)
|
|
->where('instance_id', $instance->id)->lockForUpdate()->first();
|
|
|
|
if ($period !== null && in_array((string) $period->status, ['active', 'exhausted'], true)) {
|
|
$this->updateUsage($instance, $period, $now);
|
|
if ((int) $period->ends_at <= $now) {
|
|
$period->status = 'finished';
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$period = null;
|
|
$instance->current_period_id = null;
|
|
}
|
|
} else {
|
|
$period = null;
|
|
$instance->current_period_id = null;
|
|
}
|
|
|
|
if ($period === null) {
|
|
// The lifecycle daemon may be stopped for longer than one
|
|
// prepaid period. Never briefly activate an already expired
|
|
// historical period (which would also restore its network).
|
|
CloudServicePeriod::where('instance_id', $instance->id)
|
|
->where('status', 'scheduled')->where('ends_at', '<=', $now)
|
|
->update(['status' => 'finished', 'updated_at' => $now]);
|
|
$period = CloudServicePeriod::where('instance_id', $instance->id)
|
|
->where('status', 'scheduled')->where('starts_at', '<=', $now)
|
|
->where('ends_at', '>', $now)
|
|
->orderBy('starts_at')->orderBy('id')->lockForUpdate()->first();
|
|
if ($period !== null) {
|
|
$period->status = 'active';
|
|
$period->activated_at = $now;
|
|
$baseline = CloudTrafficService::periodBaseline(
|
|
$instance, (int) $period->starts_at);
|
|
if ($baseline === null
|
|
&& (int) $instance->traffic_synced_at >= (int) $period->starts_at) {
|
|
$baseline = [
|
|
'download' => max(0, (int) $instance->lifetime_download_bytes),
|
|
'upload' => max(0, (int) $instance->lifetime_upload_bytes),
|
|
];
|
|
}
|
|
$period->download_base_bytes = $baseline['download'] ?? null;
|
|
$period->upload_base_bytes = $baseline['upload'] ?? null;
|
|
$period->download_used_bytes = $baseline === null ? 0
|
|
: max(0, (int) $instance->lifetime_download_bytes
|
|
- (int) $baseline['download']);
|
|
$period->upload_used_bytes = $baseline === null ? 0
|
|
: max(0, (int) $instance->lifetime_upload_bytes
|
|
- (int) $baseline['upload']);
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$instance->current_period_id = $period->id;
|
|
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
|
|
$instance->traffic_download_bytes = (int) $period->download_used_bytes;
|
|
$instance->traffic_upload_bytes = (int) $period->upload_used_bytes;
|
|
if (!in_array((string) $instance->service_status, ['active', 'exempt'], true)) {
|
|
$legacySuspend = $instance->suspended_at !== null
|
|
|| CloudJob::where('instance_id', $instance->id)
|
|
->where('kind', 'service_suspend')
|
|
->whereIn('status', ['queued', 'running', 'uncertain'])->exists();
|
|
$instance->service_status = $legacySuspend ? 'resume_pending' : 'active';
|
|
}
|
|
if ((string) $instance->network_status !== 'active') {
|
|
$instance->network_status = 'restore_pending';
|
|
$instance->network_suspended_at = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($period !== null) {
|
|
$usageKnown = $this->updateUsage($instance, $period, $now);
|
|
$used = (int) $period->download_used_bytes + (int) $period->upload_used_bytes;
|
|
$quota = max(0, (int) $period->traffic_quota_bytes);
|
|
if ($usageKnown && $quota > 0 && $used >= $quota) {
|
|
if ((string) $period->status !== 'exhausted') {
|
|
$period->status = 'exhausted';
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
}
|
|
if ((string) $instance->network_status === 'block_failed'
|
|
&& $this->policyRetryReady($instance, 'network_suspend', $now)) {
|
|
$instance->network_status = 'blocking';
|
|
} elseif (!in_array((string) $instance->network_status,
|
|
['blocking', 'blocked_traffic', 'block_failed'], true)) {
|
|
$instance->network_status = 'blocking';
|
|
}
|
|
} elseif ($usageKnown) {
|
|
if ((string) $instance->network_status === 'restore_failed'
|
|
&& $this->policyRetryReady($instance, 'network_resume', $now)) {
|
|
$instance->network_status = 'restore_pending';
|
|
$instance->network_suspended_at = null;
|
|
} elseif (!in_array((string) $instance->network_status,
|
|
['active', 'restore_failed'], true)) {
|
|
$instance->network_status = 'restore_pending';
|
|
$instance->network_suspended_at = null;
|
|
}
|
|
}
|
|
if ((string) $instance->service_status === 'resume_failed'
|
|
&& $this->policyRetryReady($instance, 'service_resume', $now)) {
|
|
$instance->service_status = 'resume_pending';
|
|
}
|
|
} elseif ($instance->expires_at !== null && (int) $instance->expires_at <= $now) {
|
|
$invoice = CloudInvoice::where('instance_id', $instance->id)
|
|
->where('renews_expires_at', $instance->expires_at)
|
|
->where('status', '<>', 'void')->orderByDesc('id')
|
|
->lockForUpdate()->first();
|
|
$pastGrace = $invoice !== null
|
|
&& in_array((string) $invoice->status,
|
|
['open', 'overdue', 'expired'], true)
|
|
&& CloudInvoiceService::paymentDeadlinePassed($invoice, $now);
|
|
if ($pastGrace) {
|
|
if (in_array((string) $invoice->status, ['open', 'overdue'], true)) {
|
|
$invoice->status = 'expired';
|
|
$invoice->updated_at = $now;
|
|
$invoice->save();
|
|
}
|
|
$instance->status = 'deleting';
|
|
if ((string) $instance->service_status === 'purge_stop_failed') {
|
|
if ($this->policyRetryReady($instance, 'service_suspend', $now)) {
|
|
$instance->service_status = 'purge_stopping';
|
|
}
|
|
} elseif (!in_array((string) $instance->service_status,
|
|
['purge_stopping', 'purge_deleting', 'purge_failed'], true)) {
|
|
$instance->service_status = 'purge_stopping';
|
|
}
|
|
} elseif (!in_array((string) $instance->service_status,
|
|
['purge_stopping', 'purge_stop_failed', 'purge_deleting', 'purge_failed'], true)) {
|
|
// Expiry keeps QEMU running throughout the three-day
|
|
// payment grace period. Only its NIC links are persisted
|
|
// down; a paid renewal restores those links. During an
|
|
// online upgrade, a durable suspended_at marker proves
|
|
// that the retired policy stopped a previously running
|
|
// guest, which must be restarted only after link-down.
|
|
$instance->service_status = $instance->suspended_at === null
|
|
? 'expired_grace' : 'resume_pending';
|
|
$network = (string) $instance->network_status;
|
|
if ($network === 'blocked_traffic') {
|
|
$instance->network_status = 'blocked_expired';
|
|
} elseif ($network === 'block_failed') {
|
|
if ($this->policyRetryReady($instance, 'network_suspend', $now)) {
|
|
$instance->network_status = 'blocking_expired';
|
|
}
|
|
} elseif (!in_array($network,
|
|
['blocking', 'blocking_expired', 'blocked_expired'], true)) {
|
|
$instance->network_status = 'blocking_expired';
|
|
}
|
|
}
|
|
}
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
|
|
if ($this->hasConflictingJob($instance)) {
|
|
return false;
|
|
}
|
|
|
|
$service = (string) $instance->service_status;
|
|
$network = (string) $instance->network_status;
|
|
if ($service === 'purge_stopping') {
|
|
return $this->enqueue($instance, 'service_suspend', 'service-suspend', 'purge');
|
|
}
|
|
if ($service === 'purge_deleting') {
|
|
$instance->status = 'deleting';
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
CloudJobService::enqueueExpiryDelete($instance);
|
|
return true;
|
|
}
|
|
if ($network === 'blocking_expired') {
|
|
return $this->enqueue($instance, 'network_suspend', 'network-suspend', 'expired');
|
|
}
|
|
if ($network === 'blocking') {
|
|
return $this->enqueue($instance, 'network_suspend', 'network-suspend', 'traffic');
|
|
}
|
|
if (in_array($service, ['resume_pending', 'resuming'], true)) {
|
|
$legacyExpiryResume = $instance->expires_at !== null
|
|
&& (int) $instance->expires_at <= $now;
|
|
if ($legacyExpiryResume && $network !== 'blocked_expired') {
|
|
return false;
|
|
}
|
|
$instance->service_status = 'resuming';
|
|
$instance->save();
|
|
return $this->enqueue($instance, 'service_resume', 'service-resume', null,
|
|
['start_if_suspended' => $instance->suspended_at !== null]);
|
|
}
|
|
if (in_array($network, ['restore_pending', 'restoring'], true)) {
|
|
$instance->network_status = 'restoring';
|
|
$instance->save();
|
|
return $this->enqueue($instance, 'network_resume', 'network-resume');
|
|
}
|
|
return false;
|
|
}, 3);
|
|
}
|
|
|
|
private function hasConflictingJob(CloudInstance $instance): bool
|
|
{
|
|
// Match reconcile()'s target ordering: purge stop/delete wins before
|
|
// network policy. Only a tick that will actually enqueue
|
|
// network_suspend may defer snapshot serialization to the engine's
|
|
// durable queued-versus-running decision.
|
|
$failSafeSuspend = !in_array((string) $instance->service_status,
|
|
['purge_stopping', 'purge_deleting'], true)
|
|
&& in_array((string) $instance->network_status,
|
|
['blocking', 'blocking_expired'], true);
|
|
$snapshotKinds = [
|
|
'snapshot_create', 'snapshot_restore', 'snapshot_delete',
|
|
];
|
|
// Firewall changes never alter power, link state or resource
|
|
// ownership. Let lifecycle policy enter its own durable queue even if
|
|
// a firewall operation is queued, running or awaiting exact evidence.
|
|
// The engine's per-VM target lock still serializes the actual libvirt
|
|
// mutations, while the worker's pre-dispatch policy check cancels a
|
|
// stale queued tenant change once expiry enforcement has started.
|
|
$nonBlockingActiveKinds = ['firewall_apply'];
|
|
$nonBlockingUncertainKinds = ['action', 'firewall_apply'];
|
|
if ($failSafeSuspend) {
|
|
// A lost snapshot response must not permanently defeat quota or
|
|
// expiry enforcement. The engine still rejects these target-state
|
|
// suspends while a snapshot executor or libvirt BACKUP is active;
|
|
// once neither is active, an old durable snapshot uncertainty is
|
|
// not allowed to keep networking up. Purge shutdown remains
|
|
// serialized because an uncertain restore reconciler may still
|
|
// need to recover the VM's original power state.
|
|
// Let the target-state request reach the engine even while PHP is
|
|
// still polling a snapshot job. The engine may cancel only a
|
|
// durably queued snapshot that has not touched libvirt; a running
|
|
// backup remains protected and returns 409 for lifecycle retry.
|
|
$nonBlockingActiveKinds = array_merge(
|
|
$nonBlockingActiveKinds, $snapshotKinds
|
|
);
|
|
$nonBlockingUncertainKinds = array_merge(
|
|
$nonBlockingUncertainKinds, $snapshotKinds
|
|
);
|
|
}
|
|
return CloudJob::where('instance_id', $instance->id)
|
|
->where(function ($query) use ($nonBlockingActiveKinds,
|
|
$nonBlockingUncertainKinds) {
|
|
$query->where(function ($active) use ($nonBlockingActiveKinds) {
|
|
$active->whereIn('status', ['queued', 'running']);
|
|
if ($nonBlockingActiveKinds !== []) {
|
|
$active->whereNotIn('kind', $nonBlockingActiveKinds);
|
|
}
|
|
})
|
|
->orWhere(function ($uncertain) use ($nonBlockingUncertainKinds) {
|
|
// A reset/shutdown action does not change resource
|
|
// ownership, so expiry policy may still converge around
|
|
// an old uncertain action. Delete is different: until
|
|
// the original destructive operation is reconciled it
|
|
// must serialize every later lifecycle mutation.
|
|
$uncertain->where('status', 'uncertain')
|
|
->whereNotIn('kind', $nonBlockingUncertainKinds);
|
|
});
|
|
})->exists();
|
|
}
|
|
|
|
private function policyRetryReady(CloudInstance $instance, string $kind, int $now): bool
|
|
{
|
|
$failed = CloudJob::where('instance_id', $instance->id)
|
|
->where('kind', $kind)->where('status', 'failed')
|
|
->orderByDesc('updated_at')->orderByDesc('id')->first();
|
|
if ($failed === null) {
|
|
return true;
|
|
}
|
|
$recentFailures = CloudJob::where('instance_id', $instance->id)
|
|
->where('kind', $kind)->where('status', 'failed')
|
|
->where('updated_at', '>=', $now - 3600)->count();
|
|
// Retry temporary host/SSH failures without creating a 15-second job
|
|
// storm. Enforcement always converges again after an administrator
|
|
// repairs the host; the delay is capped at five minutes.
|
|
$exponent = min(5, max(0, $recentFailures - 1));
|
|
$delay = min(300, 15 * (1 << $exponent));
|
|
return (int) $failed->updated_at + $delay <= $now;
|
|
}
|
|
|
|
private function updateUsage(CloudInstance $instance, CloudServicePeriod $period, int $now): bool
|
|
{
|
|
if ($period->download_base_bytes === null || $period->upload_base_bytes === null) {
|
|
// At a period boundary, an old cached total would charge bytes
|
|
// from the previous period to the new one. Wait for the first
|
|
// successful sample at/after the boundary and fail open until it
|
|
// establishes both baselines.
|
|
if ((int) $instance->traffic_synced_at < (int) $period->starts_at) {
|
|
$period->download_base_bytes = null;
|
|
$period->upload_base_bytes = null;
|
|
$period->download_used_bytes = 0;
|
|
$period->upload_used_bytes = 0;
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
|
|
$instance->traffic_download_bytes = 0;
|
|
$instance->traffic_upload_bytes = 0;
|
|
return false;
|
|
}
|
|
$period->download_base_bytes = max(0, (int) $instance->lifetime_download_bytes);
|
|
$period->upload_base_bytes = max(0, (int) $instance->lifetime_upload_bytes);
|
|
}
|
|
$download = max(0, (int) $instance->lifetime_download_bytes - (int) $period->download_base_bytes);
|
|
$upload = max(0, (int) $instance->lifetime_upload_bytes - (int) $period->upload_base_bytes);
|
|
$period->download_used_bytes = $download;
|
|
$period->upload_used_bytes = $upload;
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
|
|
$instance->traffic_download_bytes = $download;
|
|
$instance->traffic_upload_bytes = $upload;
|
|
return true;
|
|
}
|
|
|
|
private function enqueue(CloudInstance $instance, string $kind, string $suffix,
|
|
?string $reason = null, array $body = []): bool
|
|
{
|
|
$job = new CloudJob();
|
|
$job->operation_id = bin2hex(random_bytes(16));
|
|
$job->instance_id = $instance->id;
|
|
$job->user_id = $instance->user_id;
|
|
$job->host_id = $instance->host_id;
|
|
$job->kind = $kind;
|
|
$job->status = 'queued';
|
|
$job->payload_cipher = CloudSecrets::encrypt([
|
|
'method' => 'POST',
|
|
'resource' => 'vms/' . rawurlencode($instance->uuid) . '/' . $suffix,
|
|
'body' => $body, 'owner_id' => (int) $instance->user_id,
|
|
'system_policy' => true, 'suspend_reason' => $reason,
|
|
]);
|
|
$job->available_at = $job->created_at = $job->updated_at = time();
|
|
$job->save();
|
|
return true;
|
|
}
|
|
|
|
}
|