You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
351 lines
18 KiB
PHP
351 lines
18 KiB
PHP
<?php
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\CloudInstance;
|
|
use App\Models\CloudInvoice;
|
|
use App\Models\CloudJob;
|
|
use App\Models\CloudServicePeriod;
|
|
|
|
final class CloudLifecycleService
|
|
{
|
|
public function tick(int $limit = 200): int
|
|
{
|
|
$batchSize = max(1, min(1000, $limit));
|
|
try {
|
|
CloudInvoiceService::expireProvision($batchSize);
|
|
} catch (\Throwable $error) {
|
|
// Releasing an abandoned provisioning reservation is database-only
|
|
// and must not stop instance expiry or traffic enforcement.
|
|
error_log('cloud provisioning invoice expiry unavailable: '
|
|
. get_class($error));
|
|
}
|
|
try {
|
|
CloudInvoiceService::issueDue($batchSize);
|
|
} catch (\Throwable $error) {
|
|
// Invoice creation is database-only and must not stop policy.
|
|
error_log('cloud invoice generation unavailable: ' . get_class($error));
|
|
}
|
|
try {
|
|
CloudTrafficService::sync($batchSize);
|
|
} catch (\Throwable $error) {
|
|
// Unknown traffic is fail-open. Time expiry remains enforceable.
|
|
error_log('cloud traffic sync unavailable: ' . get_class($error));
|
|
}
|
|
$queued = 0;
|
|
CloudInstance::where('status', '<>', 'deleted')
|
|
->where('service_status', '<>', 'exempt')
|
|
->select('id')->chunkById($batchSize, function ($instances) use (&$queued) {
|
|
foreach ($instances as $instance) {
|
|
try {
|
|
$queued += $this->reconcile((int) $instance->id) ? 1 : 0;
|
|
} catch (\Throwable $error) {
|
|
// One corrupt or temporarily contended record must not
|
|
// starve every higher instance ID on every daemon pass.
|
|
error_log('cloud lifecycle instance ' . (int) $instance->id
|
|
. ' unavailable: ' . get_class($error));
|
|
}
|
|
}
|
|
});
|
|
return $queued;
|
|
}
|
|
|
|
private function reconcile(int $instanceId): bool
|
|
{
|
|
return (new CloudInstance())->getConnection()->transaction(function () use ($instanceId) {
|
|
$now = time();
|
|
$instance = CloudInstance::where('id', $instanceId)->lockForUpdate()->first();
|
|
if ($instance === null || in_array((string) $instance->status,
|
|
['deleted', 'delete_failed'], true)
|
|
|| $instance->service_status === 'exempt') {
|
|
return false;
|
|
}
|
|
if ((string) $instance->service_status === 'purge_failed') {
|
|
return false;
|
|
}
|
|
$period = $instance->current_period_id === null ? null
|
|
: CloudServicePeriod::where('id', $instance->current_period_id)
|
|
->where('instance_id', $instance->id)->lockForUpdate()->first();
|
|
|
|
if ($period !== null && in_array((string) $period->status, ['active', 'exhausted'], true)) {
|
|
$this->updateUsage($instance, $period, $now);
|
|
if ((int) $period->ends_at <= $now) {
|
|
$period->status = 'finished';
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$period = null;
|
|
$instance->current_period_id = null;
|
|
}
|
|
} else {
|
|
$period = null;
|
|
$instance->current_period_id = null;
|
|
}
|
|
|
|
if ($period === null) {
|
|
// The lifecycle daemon may be stopped for longer than one
|
|
// prepaid period. Never briefly activate an already expired
|
|
// historical period (which would also restore its network).
|
|
CloudServicePeriod::where('instance_id', $instance->id)
|
|
->where('status', 'scheduled')->where('ends_at', '<=', $now)
|
|
->update(['status' => 'finished', 'updated_at' => $now]);
|
|
$period = CloudServicePeriod::where('instance_id', $instance->id)
|
|
->where('status', 'scheduled')->where('starts_at', '<=', $now)
|
|
->where('ends_at', '>', $now)
|
|
->orderBy('starts_at')->orderBy('id')->lockForUpdate()->first();
|
|
if ($period !== null) {
|
|
$period->status = 'active';
|
|
$period->activated_at = $now;
|
|
$baseline = CloudTrafficService::periodBaseline(
|
|
$instance, (int) $period->starts_at);
|
|
if ($baseline === null
|
|
&& (int) $instance->traffic_synced_at >= (int) $period->starts_at) {
|
|
$baseline = [
|
|
'download' => max(0, (int) $instance->lifetime_download_bytes),
|
|
'upload' => max(0, (int) $instance->lifetime_upload_bytes),
|
|
];
|
|
}
|
|
$period->download_base_bytes = $baseline['download'] ?? null;
|
|
$period->upload_base_bytes = $baseline['upload'] ?? null;
|
|
$period->download_used_bytes = $baseline === null ? 0
|
|
: max(0, (int) $instance->lifetime_download_bytes
|
|
- (int) $baseline['download']);
|
|
$period->upload_used_bytes = $baseline === null ? 0
|
|
: max(0, (int) $instance->lifetime_upload_bytes
|
|
- (int) $baseline['upload']);
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$instance->current_period_id = $period->id;
|
|
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
|
|
$instance->traffic_download_bytes = (int) $period->download_used_bytes;
|
|
$instance->traffic_upload_bytes = (int) $period->upload_used_bytes;
|
|
if (!in_array((string) $instance->service_status, ['active', 'exempt'], true)) {
|
|
$legacySuspend = $instance->suspended_at !== null
|
|
|| CloudJob::where('instance_id', $instance->id)
|
|
->where('kind', 'service_suspend')
|
|
->whereIn('status', ['queued', 'running', 'uncertain'])->exists();
|
|
$instance->service_status = $legacySuspend ? 'resume_pending' : 'active';
|
|
}
|
|
if ((string) $instance->network_status !== 'active') {
|
|
$instance->network_status = 'restore_pending';
|
|
$instance->network_suspended_at = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($period !== null) {
|
|
$usageKnown = $this->updateUsage($instance, $period, $now);
|
|
$used = (int) $period->download_used_bytes + (int) $period->upload_used_bytes;
|
|
$quota = max(0, (int) $period->traffic_quota_bytes);
|
|
if ($usageKnown && $quota > 0 && $used >= $quota) {
|
|
if ((string) $period->status !== 'exhausted') {
|
|
$period->status = 'exhausted';
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
}
|
|
if ((string) $instance->network_status === 'block_failed'
|
|
&& $this->policyRetryReady($instance, 'network_suspend', $now)) {
|
|
$instance->network_status = 'blocking';
|
|
} elseif (!in_array((string) $instance->network_status,
|
|
['blocking', 'blocked_traffic', 'block_failed'], true)) {
|
|
$instance->network_status = 'blocking';
|
|
}
|
|
} elseif ($usageKnown) {
|
|
if ((string) $instance->network_status === 'restore_failed'
|
|
&& $this->policyRetryReady($instance, 'network_resume', $now)) {
|
|
$instance->network_status = 'restore_pending';
|
|
$instance->network_suspended_at = null;
|
|
} elseif (!in_array((string) $instance->network_status,
|
|
['active', 'restore_failed'], true)) {
|
|
$instance->network_status = 'restore_pending';
|
|
$instance->network_suspended_at = null;
|
|
}
|
|
}
|
|
if ((string) $instance->service_status === 'resume_failed'
|
|
&& $this->policyRetryReady($instance, 'service_resume', $now)) {
|
|
$instance->service_status = 'resume_pending';
|
|
}
|
|
} elseif ($instance->expires_at !== null && (int) $instance->expires_at <= $now) {
|
|
$invoice = CloudInvoice::where('instance_id', $instance->id)
|
|
->where('renews_expires_at', $instance->expires_at)
|
|
->where('status', '<>', 'void')->orderByDesc('id')
|
|
->lockForUpdate()->first();
|
|
$pastGrace = $invoice !== null
|
|
&& in_array((string) $invoice->status,
|
|
['open', 'overdue', 'expired'], true)
|
|
&& CloudInvoiceService::paymentDeadlinePassed($invoice, $now);
|
|
if ($pastGrace) {
|
|
if (in_array((string) $invoice->status, ['open', 'overdue'], true)) {
|
|
$invoice->status = 'expired';
|
|
$invoice->updated_at = $now;
|
|
$invoice->save();
|
|
}
|
|
$instance->status = 'deleting';
|
|
if ((string) $instance->service_status === 'purge_stop_failed') {
|
|
if ($this->policyRetryReady($instance, 'service_suspend', $now)) {
|
|
$instance->service_status = 'purge_stopping';
|
|
}
|
|
} elseif (!in_array((string) $instance->service_status,
|
|
['purge_stopping', 'purge_deleting', 'purge_failed'], true)) {
|
|
$instance->service_status = 'purge_stopping';
|
|
}
|
|
} elseif (!in_array((string) $instance->service_status,
|
|
['purge_stopping', 'purge_stop_failed', 'purge_deleting', 'purge_failed'], true)) {
|
|
// Expiry keeps QEMU running throughout the three-day
|
|
// payment grace period. Only its NIC links are persisted
|
|
// down; a paid renewal restores those links. During an
|
|
// online upgrade, a durable suspended_at marker proves
|
|
// that the retired policy stopped a previously running
|
|
// guest, which must be restarted only after link-down.
|
|
$instance->service_status = $instance->suspended_at === null
|
|
? 'expired_grace' : 'resume_pending';
|
|
$network = (string) $instance->network_status;
|
|
if ($network === 'blocked_traffic') {
|
|
$instance->network_status = 'blocked_expired';
|
|
} elseif ($network === 'block_failed') {
|
|
if ($this->policyRetryReady($instance, 'network_suspend', $now)) {
|
|
$instance->network_status = 'blocking_expired';
|
|
}
|
|
} elseif (!in_array($network,
|
|
['blocking', 'blocking_expired', 'blocked_expired'], true)) {
|
|
$instance->network_status = 'blocking_expired';
|
|
}
|
|
}
|
|
}
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
|
|
if ($this->hasConflictingJob($instance)) {
|
|
return false;
|
|
}
|
|
|
|
$service = (string) $instance->service_status;
|
|
$network = (string) $instance->network_status;
|
|
if ($service === 'purge_stopping') {
|
|
return $this->enqueue($instance, 'service_suspend', 'service-suspend', 'purge');
|
|
}
|
|
if ($service === 'purge_deleting') {
|
|
$instance->status = 'deleting';
|
|
$instance->updated_at = $now;
|
|
$instance->save();
|
|
CloudJobService::enqueueExpiryDelete($instance);
|
|
return true;
|
|
}
|
|
if ($network === 'blocking_expired') {
|
|
return $this->enqueue($instance, 'network_suspend', 'network-suspend', 'expired');
|
|
}
|
|
if ($network === 'blocking') {
|
|
return $this->enqueue($instance, 'network_suspend', 'network-suspend', 'traffic');
|
|
}
|
|
if (in_array($service, ['resume_pending', 'resuming'], true)) {
|
|
$legacyExpiryResume = $instance->expires_at !== null
|
|
&& (int) $instance->expires_at <= $now;
|
|
if ($legacyExpiryResume && $network !== 'blocked_expired') {
|
|
return false;
|
|
}
|
|
$instance->service_status = 'resuming';
|
|
$instance->save();
|
|
return $this->enqueue($instance, 'service_resume', 'service-resume', null,
|
|
['start_if_suspended' => $instance->suspended_at !== null]);
|
|
}
|
|
if (in_array($network, ['restore_pending', 'restoring'], true)) {
|
|
$instance->network_status = 'restoring';
|
|
$instance->save();
|
|
return $this->enqueue($instance, 'network_resume', 'network-resume');
|
|
}
|
|
return false;
|
|
}, 3);
|
|
}
|
|
|
|
private function hasConflictingJob(CloudInstance $instance): bool
|
|
{
|
|
return CloudJob::where('instance_id', $instance->id)
|
|
->where(function ($query) {
|
|
$query->whereIn('status', ['queued', 'running'])
|
|
->orWhere(function ($uncertain) {
|
|
// Reset/shutdown/delete style mutations cannot be
|
|
// replayed after a lost response and may legitimately
|
|
// remain uncertain forever. They must not permanently
|
|
// disable expiry and traffic policy. Reinstall,
|
|
// rescue and target-state policy operations can still
|
|
// converge, so their uncertainty remains a conflict.
|
|
$uncertain->where('status', 'uncertain')
|
|
->whereNotIn('kind', ['action', 'delete']);
|
|
});
|
|
})->exists();
|
|
}
|
|
|
|
private function policyRetryReady(CloudInstance $instance, string $kind, int $now): bool
|
|
{
|
|
$failed = CloudJob::where('instance_id', $instance->id)
|
|
->where('kind', $kind)->where('status', 'failed')
|
|
->orderByDesc('updated_at')->orderByDesc('id')->first();
|
|
if ($failed === null) {
|
|
return true;
|
|
}
|
|
$recentFailures = CloudJob::where('instance_id', $instance->id)
|
|
->where('kind', $kind)->where('status', 'failed')
|
|
->where('updated_at', '>=', $now - 3600)->count();
|
|
// Retry temporary host/SSH failures without creating a 15-second job
|
|
// storm. Enforcement always converges again after an administrator
|
|
// repairs the host; the delay is capped at five minutes.
|
|
$exponent = min(5, max(0, $recentFailures - 1));
|
|
$delay = min(300, 15 * (1 << $exponent));
|
|
return (int) $failed->updated_at + $delay <= $now;
|
|
}
|
|
|
|
private function updateUsage(CloudInstance $instance, CloudServicePeriod $period, int $now): bool
|
|
{
|
|
if ($period->download_base_bytes === null || $period->upload_base_bytes === null) {
|
|
// At a period boundary, an old cached total would charge bytes
|
|
// from the previous period to the new one. Wait for the first
|
|
// successful sample at/after the boundary and fail open until it
|
|
// establishes both baselines.
|
|
if ((int) $instance->traffic_synced_at < (int) $period->starts_at) {
|
|
$period->download_base_bytes = null;
|
|
$period->upload_base_bytes = null;
|
|
$period->download_used_bytes = 0;
|
|
$period->upload_used_bytes = 0;
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
|
|
$instance->traffic_download_bytes = 0;
|
|
$instance->traffic_upload_bytes = 0;
|
|
return false;
|
|
}
|
|
$period->download_base_bytes = max(0, (int) $instance->lifetime_download_bytes);
|
|
$period->upload_base_bytes = max(0, (int) $instance->lifetime_upload_bytes);
|
|
}
|
|
$download = max(0, (int) $instance->lifetime_download_bytes - (int) $period->download_base_bytes);
|
|
$upload = max(0, (int) $instance->lifetime_upload_bytes - (int) $period->upload_base_bytes);
|
|
$period->download_used_bytes = $download;
|
|
$period->upload_used_bytes = $upload;
|
|
$period->updated_at = $now;
|
|
$period->save();
|
|
$instance->traffic_quota_bytes = max(0, (int) $period->traffic_quota_bytes);
|
|
$instance->traffic_download_bytes = $download;
|
|
$instance->traffic_upload_bytes = $upload;
|
|
return true;
|
|
}
|
|
|
|
private function enqueue(CloudInstance $instance, string $kind, string $suffix,
|
|
?string $reason = null, array $body = []): bool
|
|
{
|
|
$job = new CloudJob();
|
|
$job->operation_id = bin2hex(random_bytes(16));
|
|
$job->instance_id = $instance->id;
|
|
$job->user_id = $instance->user_id;
|
|
$job->host_id = $instance->host_id;
|
|
$job->kind = $kind;
|
|
$job->status = 'queued';
|
|
$job->payload_cipher = CloudSecrets::encrypt([
|
|
'method' => 'POST',
|
|
'resource' => 'vms/' . rawurlencode($instance->uuid) . '/' . $suffix,
|
|
'body' => $body, 'owner_id' => (int) $instance->user_id,
|
|
'system_policy' => true, 'suspend_reason' => $reason,
|
|
]);
|
|
$job->available_at = $job->created_at = $job->updated_at = time();
|
|
$job->save();
|
|
return true;
|
|
}
|
|
|
|
}
|