You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudBillingLifecycleTest.php

1042 lines
47 KiB
PHP

<?php
// Billing lifecycle tests only. The in-memory keyring is not encryption and
// is never loaded by production.
namespace App\Services {
final class Config
{
public static array $values = [];
public static function get($name, $default = null)
{
return self::$values[$name] ?? $default;
}
}
}
namespace App\Services\Cloud {
final class CloudSecrets
{
private static array $memory = [];
public static function key(string $purpose): string
{
return str_repeat('k', 32);
}
public static function encrypt(array $payload, string $purpose): string
{
$id = bin2hex(random_bytes(16));
self::$memory[$id] = $payload;
return 'test-only:' . $id;
}
public static function decrypt(string $cipher, string $purpose): array
{
return self::$memory[substr($cipher, 10)];
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Models\CloudInvoice;
use App\Models\CloudJob;
use App\Models\CloudRenewal;
use App\Models\CloudServicePeriod;
use App\Models\User;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudInvoiceService;
use App\Services\Cloud\CloudLifecycleService;
use App\Services\Cloud\CloudRenewalService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function ($condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$terminalize = static function (CloudJob $job, string $outcome,
bool $effect = false, bool $safe = false): void {
$now = time();
$job->phase = 'terminal';
$job->outcome = $outcome;
$job->status = $outcome;
$job->attempts = 1;
$job->dispatch_attempts = 1;
$job->dispatch_started_at ??= $now - 1;
$job->dispatched_at ??= $now - 1;
$job->agent_accepted_at ??= $now - 1;
$job->receipt_deadline_at ??= $now + 900;
$job->terminal_at = $now;
$job->effect_may_have_applied = $effect ? 1 : 0;
$job->safe_to_resubmit = $safe ? 1 : 0;
$job->effect_resolution = $effect ? 'unknown' : 'not_applicable';
$job->lease_token = null;
$job->lease_until = 0;
$job->available_at = 0;
$job->updated_at = $now;
$job->save();
};
$capsule = new Capsule();
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'log');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->string('money');
$table->integer('enable')->default(1);
$table->integer('restrict_balance_purchase')->default(0);
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('user_id');
$table->integer('order_id')->nullable();
$table->integer('plan_id')->nullable();
$table->integer('current_period_id')->nullable();
$table->string('uuid');
$table->string('name');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->integer('expires_at')->nullable();
$table->string('renewal_price')->nullable();
$table->integer('renewal_days')->default(0);
$table->integer('renewal_traffic_bytes')->default(0);
$table->integer('traffic_quota_bytes')->default(0);
$table->integer('traffic_download_bytes')->default(0);
$table->integer('traffic_upload_bytes')->default(0);
$table->integer('lifetime_download_bytes')->default(0);
$table->integer('lifetime_upload_bytes')->default(0);
$table->integer('traffic_synced_at')->default(0);
$table->integer('suspended_at')->nullable();
$table->integer('network_suspended_at')->nullable();
$table->text('spec_json');
$table->text('sealed_credential')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['host_id', 'uuid']);
});
$schema->create('cloud_service_periods', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('plan_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('renewal_id')->nullable();
$table->integer('starts_at');
$table->integer('ends_at');
$table->integer('traffic_quota_bytes')->default(0);
$table->integer('download_base_bytes')->nullable();
$table->integer('upload_base_bytes')->nullable();
$table->integer('download_used_bytes')->default(0);
$table->integer('upload_used_bytes')->default(0);
$table->string('status');
$table->integer('activated_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
$table->unique('order_id');
$table->unique('renewal_id');
});
$schema->create('cloud_renewals', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('plan_id')->nullable();
$table->integer('period_id')->nullable();
$table->string('idempotency_key');
$table->string('request_hash');
$table->string('amount');
$table->integer('validity_days');
$table->integer('traffic_bytes_added');
$table->integer('previous_expires_at');
$table->integer('new_expires_at');
$table->integer('created_at');
$table->unique(['user_id', 'idempotency_key']);
});
$schema->create('cloud_invoices', static function (Blueprint $table): void {
$table->increments('id');
$table->string('invoice_no')->unique();
$table->string('purpose')->default('renewal');
$table->integer('instance_id')->nullable();
$table->integer('order_id')->nullable()->unique();
$table->integer('user_id');
$table->integer('plan_id')->nullable();
$table->integer('renewal_id')->nullable();
$table->integer('renews_expires_at');
$table->integer('due_at');
$table->string('amount');
$table->integer('validity_days');
$table->integer('traffic_quota_bytes');
$table->string('status');
$table->string('paid_via')->nullable();
$table->string('payment_reference')->nullable();
$table->integer('paid_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
$table->index(['instance_id', 'renews_expires_at', 'status']);
$table->unique('renewal_id');
});
$schema->create('cloud_mail_outbox', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('invoice_id');
$table->string('kind');
$table->string('execution_mode')->default('agent');
$table->string('status');
$table->integer('attempts');
$table->integer('available_at');
$table->string('lease_token')->nullable();
$table->integer('lease_until');
$table->integer('sent_at')->nullable();
$table->string('last_error');
$table->integer('created_at');
$table->integer('updated_at');
$table->unique(['invoice_id', 'kind']);
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->unique();
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->integer('parent_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase');
$table->string('outcome')->nullable();
$table->string('status');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at');
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->text('effect_resolution_reason')->nullable();
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at');
$table->string('error_code')->nullable();
$table->boolean('finalization_incident')->default(0);
$table->string('finalization_error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$logSchema = $capsule->schema('log');
$logSchema->create('cloud_vm_traffic_totals', static function (Blueprint $table): void {
$table->string('host_scope');
$table->string('vm_uuid');
$table->string('host_id')->nullable();
$table->integer('user_id')->nullable();
$table->integer('last_observed_at')->nullable();
$table->integer('download_rx_bytes')->default(0);
$table->integer('upload_tx_bytes')->default(0);
$table->integer('updated_at')->default(0);
$table->primary(['host_scope', 'vm_uuid']);
});
$logSchema->create('cloud_vm_traffic_checkpoints', static function (Blueprint $table): void {
$table->string('host_scope');
$table->string('vm_uuid');
$table->integer('checkpoint_hour');
$table->string('host_id')->nullable();
$table->integer('user_id')->nullable();
$table->integer('first_observed_at');
$table->integer('download_rx_bytes');
$table->integer('upload_tx_bytes');
$table->primary(['host_scope', 'vm_uuid', 'checkpoint_hour']);
});
$user = new User();
$user->id = 1;
$user->money = '100.00';
$user->enable = 1;
$user->restrict_balance_purchase = 0;
$user->save();
$now = time();
$instance = new CloudInstance();
$instance->host_id = 1;
$instance->user_id = 1;
$instance->plan_id = 7;
$instance->uuid = '11111111-1111-4111-8111-111111111111';
$instance->name = 'billed-vm';
$instance->status = 'active';
$instance->service_status = 'active';
$instance->network_status = 'active';
$instance->expires_at = $now + 3600;
$instance->renewal_price = '10.00';
$instance->renewal_days = 30;
$instance->renewal_traffic_bytes = 1000;
$instance->traffic_quota_bytes = 1000;
$instance->spec_json = '{}';
$instance->created_at = $instance->updated_at = $now - 100;
$instance->save();
$period = new CloudServicePeriod();
$period->instance_id = $instance->id;
$period->user_id = 1;
$period->plan_id = 7;
$period->starts_at = $now - 100;
$period->ends_at = $now + 3600;
$period->traffic_quota_bytes = 1000;
$period->download_base_bytes = 0;
$period->upload_base_bytes = 0;
$period->download_used_bytes = 0;
$period->upload_used_bytes = 0;
$period->status = 'active';
$period->activated_at = $now - 100;
$period->created_at = $period->updated_at = $now - 100;
$period->save();
$instance->current_period_id = $period->id;
$instance->save();
$scope = hash('sha256', 'virt-cloud/host/1');
$capsule->connection('log')->table('cloud_vm_traffic_totals')->insert([
'host_scope' => $scope,
'vm_uuid' => $instance->uuid,
'host_id' => '1',
'user_id' => 1,
'last_observed_at' => $now,
'download_rx_bytes' => 700,
'upload_tx_bytes' => 400,
'updated_at' => $now,
]);
$lifecycle = new CloudLifecycleService();
$lifecycle->tick(100);
$instance->refresh();
$period->refresh();
$networkSuspend = CloudJob::where('instance_id', $instance->id)
->where('kind', 'network_suspend')->first();
$assert($period->status === 'exhausted', 'Combined upload/download did not exhaust the period quota.');
$assert((int) $period->download_used_bytes === 700 && (int) $period->upload_used_bytes === 400,
'Current-period directional traffic was not synchronized exactly.');
$assert($instance->service_status === 'active' && $instance->network_status === 'blocking',
'Traffic exhaustion stopped service instead of requesting network-only suspension.');
$assert($networkSuspend !== null
&& !CloudJob::where('instance_id', $instance->id)->where('kind', 'service_suspend')->exists(),
'Traffic exhaustion queued the wrong enforcement operation.');
$actor = ['user_id' => 1, 'is_admin' => false, 'is_super_admin' => false,
'can_manage' => false, 'session_id' => ''];
$periodEndsAt = (int) $period->ends_at;
$renewal = CloudRenewalService::renew($instance, [
'idempotency_key' => 'renew-current-period-0001',
], $actor);
$sameRenewal = CloudRenewalService::renew($instance->fresh(), [
'idempotency_key' => 'renew-current-period-0001',
], $actor);
$scheduled = CloudServicePeriod::find($renewal->period_id);
$instance->refresh();
$assert($renewal->id === $sameRenewal->id && CloudRenewal::count() === 1
&& User::find(1)->money === '90.00', 'Renewal idempotency did not debit exactly once.');
$assert($scheduled->status === 'scheduled' && (int) $scheduled->starts_at === $periodEndsAt,
'Early renewal did not create a future independent period.');
$assert((int) $instance->current_period_id === (int) $period->id
&& (int) $instance->traffic_download_bytes === 700
&& (int) $instance->traffic_upload_bytes === 400
&& $instance->network_status === 'blocking',
'Early renewal refreshed current traffic or restored networking before the paid period began.');
// Model a successfully applied link-down operation, then move the clock
// boundary into the past. The next period must reset only its own usage
// and request link restoration without starting another VM service.
$terminalize($networkSuspend, 'succeeded');
$instance->network_status = 'blocked_traffic';
$instance->save();
$period->ends_at = $now - 1;
$period->save();
$scheduled->starts_at = $now - 1;
$scheduled->save();
$lifecycle->tick(100);
$instance->refresh();
$period->refresh();
$scheduled->refresh();
$networkResume = CloudJob::where('instance_id', $instance->id)
->where('kind', 'network_resume')->first();
$assert($period->status === 'finished' && $scheduled->status === 'active'
&& (int) $instance->current_period_id === (int) $scheduled->id,
'The next prepaid period was not activated at the period boundary.');
$assert((int) $scheduled->download_base_bytes === 700
&& (int) $scheduled->upload_base_bytes === 400
&& (int) $scheduled->download_used_bytes === 0
&& (int) $scheduled->upload_used_bytes === 0,
'New-period traffic did not start from an independent baseline.');
$assert($instance->service_status === 'active' && $instance->network_status === 'restoring'
&& $networkResume !== null,
'The new period did not request automatic network restoration.');
$terminalize($networkResume, 'succeeded');
$instance->network_status = 'active';
$instance->save();
$capsule->connection('log')->table('cloud_vm_traffic_totals')
->where('host_scope', $scope)->where('vm_uuid', $instance->uuid)->update([
'last_observed_at' => $now + 1,
'download_rx_bytes' => 800,
'upload_tx_bytes' => 450,
'updated_at' => $now + 1,
]);
$lifecycle->tick(100);
$scheduled->refresh();
$assert((int) $scheduled->download_used_bytes === 100
&& (int) $scheduled->upload_used_bytes === 50,
'New-period usage included traffic paid for in the previous period.');
$scheduled->ends_at = $now - 1;
$scheduled->save();
$instance->expires_at = $now - 1;
$instance->save();
$lifecycle->tick(100);
$instance->refresh();
$graceInvoice = CloudInvoice::where('instance_id', $instance->id)
->where('renews_expires_at', $now - 1)->where('status', 'open')->first();
$expiryNetworkSuspend = CloudJob::where('instance_id', $instance->id)
->where('kind', 'network_suspend')->where('status', 'queued')
->orderByDesc('id')->first();
$expiryNetworkPayload = $expiryNetworkSuspend === null ? []
: \App\Services\Cloud\CloudSecrets::decrypt(
$expiryNetworkSuspend->sealed_payload, 'job-payload');
$assert($graceInvoice !== null && $instance->service_status === 'expired_grace'
&& $instance->network_status === 'blocking_expired',
'Final paid expiry did not enter the network-only payment grace period.');
$assert($expiryNetworkSuspend !== null
&& ($expiryNetworkPayload['suspend_reason'] ?? null) === 'expired'
&& CloudJob::where('instance_id', $instance->id)
->where('kind', 'service_suspend')->count() === 0,
'Expiry must queue network_suspend(reason=expired) and never stop the VM during grace.');
// Model the expiry link-down completing before the tenant settles the
// renewal invoice. Paying inside grace must create a live paid period and
// restore networking without stopping or starting the VM service.
$terminalize($expiryNetworkSuspend, 'succeeded');
$instance->network_status = 'blocked_expired';
$instance->network_suspended_at = $now;
$instance->save();
CloudInvoiceService::payBalance($graceInvoice, $actor,
'invoice-grace-balance-0001');
$instance->refresh();
$assert($graceInvoice->fresh()->status === 'paid'
&& $instance->status === 'active'
&& $instance->service_status === 'active'
&& $instance->network_status === 'restore_pending',
'A renewal paid inside grace did not reactivate service and request network restoration.');
$lifecycle->tick(100);
$instance->refresh();
$graceNetworkResume = CloudJob::where('instance_id', $instance->id)
->where('kind', 'network_resume')->where('status', 'queued')
->orderByDesc('id')->first();
$assert($instance->status === 'active'
&& $instance->service_status === 'active'
&& $instance->network_status === 'restoring'
&& $graceNetworkResume !== null
&& CloudJob::where('instance_id', $instance->id)
->whereIn('kind', ['service_suspend', 'service_resume'])->count() === 0,
'Grace renewal changed VM power state instead of restoring only its network.');
// A matching unpaid invoice becomes terminal only after its three-day
// grace deadline. Cleanup is deliberately two-phase: first converge the
// guest to stopped, then enqueue one irreversible disk-deleting request.
$purgeDueAt = $now - CloudInvoiceService::PAYMENT_GRACE_SECONDS - 60;
$purge = new CloudInstance();
$purge->host_id = 1;
$purge->user_id = 1;
$purge->plan_id = 7;
$purge->uuid = '12121212-1212-4212-8212-121212121212';
$purge->name = 'expired-purge-vm';
$purge->status = 'active';
$purge->service_status = 'expired_grace';
$purge->network_status = 'blocked_expired';
$purge->expires_at = $purgeDueAt;
$purge->renewal_price = '10.00';
$purge->renewal_days = 30;
$purge->renewal_traffic_bytes = 1000;
$purge->traffic_quota_bytes = 1000;
$purge->network_suspended_at = $purgeDueAt;
$purge->spec_json = '{}';
$purge->created_at = $purge->updated_at = $purgeDueAt - 3600;
$purge->save();
$purgeInvoice = new CloudInvoice();
$purgeInvoice->invoice_no = 'VC-EXPIRED-PURGE';
$purgeInvoice->instance_id = $purge->id;
$purgeInvoice->user_id = 1;
$purgeInvoice->plan_id = 7;
$purgeInvoice->renews_expires_at = $purgeDueAt;
$purgeInvoice->due_at = $purgeDueAt;
$purgeInvoice->amount = '10.00';
$purgeInvoice->validity_days = 30;
$purgeInvoice->traffic_quota_bytes = 1000;
$purgeInvoice->status = 'open';
$purgeInvoice->created_at = $purgeInvoice->updated_at = $purgeDueAt;
$purgeInvoice->save();
$lifecycle->tick(100);
$purge->refresh();
$purgeInvoice->refresh();
$purgeStop = CloudJob::where('instance_id', $purge->id)
->where('kind', 'service_suspend')->where('status', 'queued')->first();
$purgeStopPayload = $purgeStop === null ? []
: \App\Services\Cloud\CloudSecrets::decrypt(
$purgeStop->sealed_payload, 'job-payload');
$assert($purgeInvoice->status === 'expired'
&& $purge->status === 'deleting'
&& $purge->service_status === 'purge_stopping',
'An invoice beyond grace did not latch the instance into purge stopping.');
$assert($purgeStop !== null
&& ($purgeStopPayload['suspend_reason'] ?? null) === 'purge',
'Expired cleanup did not queue service_suspend(reason=purge).');
// Simulate a successful worker result. The following lifecycle tick must
// enqueue exactly one DELETE request that explicitly removes disks.
$terminalize($purgeStop, 'succeeded');
$purge->service_status = 'purge_deleting';
$purge->save();
$lifecycle->tick(100);
$expiryDelete = CloudJob::where('instance_id', $purge->id)
->where('kind', 'expiry_delete')->first();
$expiryDeletePayload = $expiryDelete === null ? []
: \App\Services\Cloud\CloudSecrets::decrypt(
$expiryDelete->sealed_payload, 'job-payload');
$assert($expiryDelete !== null
&& CloudJob::where('instance_id', $purge->id)
->where('kind', 'expiry_delete')->count() === 1
&& ($expiryDeletePayload['method'] ?? null) === 'DELETE'
&& ($expiryDeletePayload['request_target'] ?? null)
=== '/v1/vms/' . $purge->uuid
&& ($expiryDeletePayload['body']['remove_managed_disks'] ?? null)
=== true,
'Purge did not enqueue one disk-deleting expiry DELETE operation.');
// A receipt deadline is a terminal failure. Its unresolved-effect fence
// keeps the instance latched and suppresses every duplicate deletion.
$terminalize($expiryDelete, 'failed', true, false);
$lifecycle->tick(100);
$purge->refresh();
$assert($purge->status === 'deleting'
&& $purge->service_status === 'purge_deleting'
&& CloudJob::where('instance_id', $purge->id)
->where('kind', 'expiry_delete')->count() === 1,
'A terminal unknown-effect expiry delete was replayed or released the deletion latch.');
// A missing sample at the billing boundary must fail open. Otherwise the
// first counter catch-up after an outage would charge previous-period
// bytes to the new period and could immediately disconnect the tenant.
$stale = new CloudInstance();
$stale->host_id = 1;
$stale->user_id = 1;
$stale->plan_id = 7;
$stale->uuid = '33333333-3333-4333-8333-333333333333';
$stale->name = 'stale-boundary-vm';
$stale->status = 'active';
$stale->service_status = 'active';
$stale->network_status = 'blocked_traffic';
$stale->expires_at = $now + 3600;
$stale->renewal_price = '10.00';
$stale->renewal_days = 30;
$stale->renewal_traffic_bytes = 100;
$stale->traffic_quota_bytes = 100;
$stale->lifetime_download_bytes = 5000;
$stale->lifetime_upload_bytes = 2000;
$stale->traffic_synced_at = $now - 300;
$stale->spec_json = '{}';
$stale->created_at = $stale->updated_at = $now - 7200;
$stale->save();
$staleOld = new CloudServicePeriod();
$staleOld->instance_id = $stale->id;
$staleOld->user_id = 1;
$staleOld->plan_id = 7;
$staleOld->starts_at = $now - 7200;
$staleOld->ends_at = $now - 1;
$staleOld->traffic_quota_bytes = 100;
$staleOld->download_base_bytes = 0;
$staleOld->upload_base_bytes = 0;
$staleOld->download_used_bytes = 5000;
$staleOld->upload_used_bytes = 2000;
$staleOld->status = 'exhausted';
$staleOld->activated_at = $now - 7200;
$staleOld->created_at = $staleOld->updated_at = $now - 7200;
$staleOld->save();
$staleNew = new CloudServicePeriod();
$staleNew->instance_id = $stale->id;
$staleNew->user_id = 1;
$staleNew->plan_id = 7;
$staleNew->starts_at = $now - 1;
$staleNew->ends_at = $now + 3600;
$staleNew->traffic_quota_bytes = 100;
$staleNew->download_used_bytes = 0;
$staleNew->upload_used_bytes = 0;
$staleNew->status = 'scheduled';
$staleNew->created_at = $staleNew->updated_at = $now - 60;
$staleNew->save();
$stale->current_period_id = $staleOld->id;
$stale->save();
$lifecycle->tick(100);
$stale->refresh();
$staleNew->refresh();
$staleResume = CloudJob::where('instance_id', $stale->id)
->where('kind', 'network_resume')->first();
$assert($staleNew->status === 'active'
&& $staleNew->download_base_bytes === null
&& $staleNew->upload_base_bytes === null
&& (int) $staleNew->download_used_bytes === 0
&& (int) $staleNew->upload_used_bytes === 0,
'A stale pre-boundary sample was used as the new period baseline.');
$assert($stale->network_status === 'restoring' && $staleResume !== null
&& !CloudJob::where('instance_id', $stale->id)->where('kind', 'network_suspend')->exists(),
'Unknown boundary traffic failed closed or prevented the paid-period network restore.');
$terminalize($staleResume, 'succeeded');
$stale->network_status = 'active';
$stale->save();
$capsule->connection('log')->table('cloud_vm_traffic_totals')->insert([
'host_scope' => $scope,
'vm_uuid' => $stale->uuid,
'host_id' => '1',
'user_id' => 1,
'last_observed_at' => $now + 2,
'download_rx_bytes' => 8000,
'upload_tx_bytes' => 4000,
'updated_at' => $now + 2,
]);
$lifecycle->tick(100);
$staleNew->refresh();
$assert((int) $staleNew->download_base_bytes === 8000
&& (int) $staleNew->upload_base_bytes === 4000
&& (int) $staleNew->download_used_bytes === 0
&& (int) $staleNew->upload_used_bytes === 0,
'The first post-boundary observation did not establish a fail-open baseline.');
$capsule->connection('log')->table('cloud_vm_traffic_totals')
->where('host_scope', $scope)->where('vm_uuid', $stale->uuid)->update([
'last_observed_at' => $now + 3,
'download_rx_bytes' => 8070,
'upload_tx_bytes' => 4040,
'updated_at' => $now + 3,
]);
$lifecycle->tick(100);
$stale->refresh();
$staleNew->refresh();
$assert((int) $staleNew->download_used_bytes === 70
&& (int) $staleNew->upload_used_bytes === 40
&& $stale->network_status === 'blocking'
&& CloudJob::where('instance_id', $stale->id)->where('kind', 'network_suspend')->exists(),
'Known post-baseline usage did not enforce the combined period quota.');
// A renewal submitted just after an expired period, before the lifecycle
// daemon has closed it, must not leave two periods marked active.
$late = new CloudInstance();
$late->host_id = 1;
$late->user_id = 1;
$late->plan_id = 7;
$late->uuid = '22222222-2222-4222-8222-222222222222';
$late->name = 'late-renewal-vm';
$late->status = 'active';
$late->service_status = 'active';
$late->network_status = 'blocked_traffic';
$late->expires_at = $now - 60;
$late->renewal_price = '10.00';
$late->renewal_days = 30;
$late->renewal_traffic_bytes = 1000;
$late->traffic_quota_bytes = 1000;
$late->lifetime_download_bytes = 2000;
$late->lifetime_upload_bytes = 1000;
$late->spec_json = '{}';
$late->created_at = $late->updated_at = $now - 7200;
$late->save();
$expiredPeriod = new CloudServicePeriod();
$expiredPeriod->instance_id = $late->id;
$expiredPeriod->user_id = 1;
$expiredPeriod->plan_id = 7;
$expiredPeriod->starts_at = $now - 7200;
$expiredPeriod->ends_at = $now - 60;
$expiredPeriod->traffic_quota_bytes = 1000;
$expiredPeriod->download_base_bytes = 0;
$expiredPeriod->upload_base_bytes = 0;
$expiredPeriod->download_used_bytes = 2000;
$expiredPeriod->upload_used_bytes = 1000;
$expiredPeriod->status = 'active';
$expiredPeriod->activated_at = $now - 7200;
$expiredPeriod->created_at = $expiredPeriod->updated_at = $now - 7200;
$expiredPeriod->save();
$late->current_period_id = $expiredPeriod->id;
$late->save();
$lateRenewal = CloudRenewalService::renew($late, [
'idempotency_key' => 'late-renewal-period-0001',
], $actor);
$late->refresh();
$expiredPeriod->refresh();
$latePeriod = CloudServicePeriod::find($lateRenewal->period_id);
$assert($expiredPeriod->status === 'finished' && $latePeriod->status === 'active'
&& (int) $late->current_period_id === (int) $latePeriod->id,
'Late renewal left an expired period active beside the new current period.');
// The tick argument is a database chunk size, not a permanent low-ID cap.
// Both instances must be reconciled even when one row is fetched per chunk.
$expiredIds = [];
foreach ([
'44444444-4444-4444-8444-444444444444',
'55555555-5555-4555-8555-555555555555',
] as $number => $uuid) {
$candidate = new CloudInstance();
$candidate->host_id = 1;
$candidate->user_id = 1;
$candidate->uuid = $uuid;
$candidate->name = 'expired-' . $number;
$candidate->status = 'active';
$candidate->service_status = 'active';
$candidate->network_status = 'active';
$candidate->expires_at = $now - 1;
$candidate->spec_json = '{}';
$candidate->created_at = $candidate->updated_at = $now - 100;
$candidate->save();
$expiredIds[] = $candidate->id;
}
$lifecycle->tick(1);
foreach ($expiredIds as $id) {
$expiredCandidate = CloudInstance::find($id);
$assert($expiredCandidate->service_status === 'expired_grace'
&& $expiredCandidate->network_status === 'blocking_expired'
&& CloudJob::where('instance_id', $id)->where('kind', 'network_suspend')->exists()
&& !CloudJob::where('instance_id', $id)->where('kind', 'service_suspend')->exists(),
'Lifecycle chunking permanently starved a higher instance ID.');
}
$makeInstance = static function (string $uuid, string $name, int $expiresAt,
string $serviceStatus = 'active', string $networkStatus = 'active') use ($now): CloudInstance {
$value = new CloudInstance();
$value->host_id = 1;
$value->user_id = 1;
$value->plan_id = 7;
$value->uuid = $uuid;
$value->name = $name;
$value->status = 'active';
$value->service_status = $serviceStatus;
$value->network_status = $networkStatus;
$value->expires_at = $expiresAt;
$value->spec_json = '{}';
$value->created_at = $value->updated_at = $now - 7200;
$value->save();
return $value;
};
$makePeriod = static function (CloudInstance $value, int $startsAt, int $endsAt,
string $status = 'scheduled', int $quota = 1000) use ($now): CloudServicePeriod {
$periodValue = new CloudServicePeriod();
$periodValue->instance_id = $value->id;
$periodValue->user_id = $value->user_id;
$periodValue->plan_id = $value->plan_id;
$periodValue->starts_at = $startsAt;
$periodValue->ends_at = $endsAt;
$periodValue->traffic_quota_bytes = $quota;
$periodValue->download_used_bytes = 0;
$periodValue->upload_used_bytes = 0;
$periodValue->status = $status;
$periodValue->created_at = $periodValue->updated_at = $now - 7200;
$periodValue->save();
return $periodValue;
};
$makeJob = static function (CloudInstance $value, string $kind,
string $state = 'failed_unknown') use ($now, $terminalize): CloudJob {
$jobValue = new CloudJob();
$jobValue->operation_id = bin2hex(random_bytes(16));
$jobValue->instance_id = $value->id;
$jobValue->user_id = $value->user_id;
$jobValue->host_id = $value->host_id;
$jobValue->kind = $kind;
$jobValue->execution_mode = 'agent';
$jobValue->sealed_payload = \App\Services\Cloud\CloudSecrets::encrypt([
'method' => 'POST', 'request_target' => '/v1/test', 'body' => [],
], 'job-payload');
$jobValue->lease_until = 0;
$jobValue->available_at = $now;
$jobValue->created_at = $jobValue->updated_at = $now - 60;
$jobValue->save();
if ($state === 'failed_unknown') {
$terminalize($jobValue, 'failed', true, false);
} elseif ($state === 'running') {
$jobValue->phase = 'awaiting_receipt';
$jobValue->status = 'running';
$jobValue->attempts = 1;
$jobValue->dispatch_attempts = 1;
$jobValue->dispatch_started_at = $now - 60;
$jobValue->dispatched_at = $now - 60;
$jobValue->agent_accepted_at = $now - 60;
$jobValue->receipt_deadline_at = $now + 900;
$jobValue->save();
} elseif ($state !== 'queued') {
throw new RuntimeException('Invalid test task state');
}
return $jobValue;
};
// If the lifecycle daemon misses multiple complete prepaid periods, it
// must finish those historical rows without ever activating them. Only
// the one period that contains "now" may restore networking.
$backlog = $makeInstance('66666666-6666-4666-8666-666666666666',
'period-backlog-vm', $now + 1800, 'active', 'blocked_traffic');
$backlog->lifetime_download_bytes = 10000;
$backlog->lifetime_upload_bytes = 5000;
$backlog->traffic_synced_at = $now;
$backlog->save();
$missedOne = $makePeriod($backlog, $now - 4000, $now - 3000);
$missedTwo = $makePeriod($backlog, $now - 3000, $now - 2000);
$currentBacklog = $makePeriod($backlog, $now - 100, $now + 900);
$futureBacklog = $makePeriod($backlog, $now + 900, $now + 1800);
$lifecycle->tick(100);
$backlog->refresh();
$missedOne->refresh();
$missedTwo->refresh();
$currentBacklog->refresh();
$futureBacklog->refresh();
$assert($missedOne->status === 'finished' && $missedOne->activated_at === null
&& $missedTwo->status === 'finished' && $missedTwo->activated_at === null,
'Historical scheduled periods were briefly activated after daemon downtime.');
$assert($currentBacklog->status === 'active'
&& (int) $backlog->current_period_id === (int) $currentBacklog->id
&& $futureBacklog->status === 'scheduled',
'Lifecycle did not select exactly the currently valid prepaid period.');
$assert($backlog->network_status === 'restoring'
&& CloudJob::where('instance_id', $backlog->id)->where('kind', 'network_resume')->count() === 1,
'Current-period activation did not produce exactly one network restore.');
// A failed action without a resource fence may not block expiry policy.
// A destructive terminal failure with unknown effect must block every
// later mutation until an administrator resolves that exact fence.
$nonReplayable = $makeInstance('77777777-7777-4777-8777-777777777777',
'terminal-action-failure', $now - 1);
$actionFailure = $makeJob($nonReplayable, 'action');
$actionFailure->effect_may_have_applied = 0;
$actionFailure->effect_resolution = 'not_applicable';
$actionFailure->save();
$uncertainDelete = $makeInstance('7a7a7a7a-7a7a-4a7a-8a7a-7a7a7a7a7a7a',
'unknown-effect-delete', $now - 1);
$makeJob($uncertainDelete, 'delete');
$replayable = $makeInstance('88888888-8888-4888-8888-888888888888',
'terminal-reinstall-failure', $now - 1);
$makeJob($replayable, 'reinstall');
$snapshotUncertain = $makeInstance(
'8a8a8a8a-8a8a-4a8a-8a8a-8a8a8a8a8a8a',
'snapshot-failed-expiry', $now - 1);
$makeJob($snapshotUncertain, 'snapshot_restore');
$snapshotQueued = $makeInstance(
'8c8c8c8c-8c8c-4c8c-8c8c-8c8c8c8c8c8c',
'snapshot-queued-expiry', $now - 1);
$makeJob($snapshotQueued, 'snapshot_create', 'queued');
$snapshotRunning = $makeInstance(
'8d8d8d8d-8d8d-4d8d-8d8d-8d8d8d8d8d8d',
'snapshot-running-expiry', $now - 1);
$makeJob($snapshotRunning, 'snapshot_delete', 'running');
$snapshotPurge = $makeInstance(
'8b8b8b8b-8b8b-4b8b-8b8b-8b8b8b8b8b8b',
'snapshot-failed-purge', $now + 3600, 'purge_stopping',
'blocked_expired');
$snapshotPurge->status = 'deleting';
$snapshotPurge->save();
$makeJob($snapshotPurge, 'snapshot_restore');
$policyPending = $makeInstance('99999999-9999-4999-8999-999999999999',
'policy-failed', $now - 1, 'suspending_expired');
$makeJob($policyPending, 'service_suspend');
$lifecycle->tick(100);
$nonReplayable->refresh();
$uncertainDelete->refresh();
$replayable->refresh();
$snapshotUncertain->refresh();
$snapshotQueued->refresh();
$snapshotRunning->refresh();
$snapshotPurge->refresh();
$policyPending->refresh();
$assert($nonReplayable->service_status === 'expired_grace'
&& $nonReplayable->network_status === 'blocking_expired'
&& CloudJob::where('instance_id', $nonReplayable->id)
->where('kind', 'network_suspend')->count() === 1
&& CloudJob::where('instance_id', $nonReplayable->id)
->where('kind', 'service_suspend')->count() === 0,
'Terminal action failure without a fence blocked expiry network suspension.');
$assert($uncertainDelete->service_status === 'active'
&& $uncertainDelete->network_status === 'active'
&& CloudJob::where('instance_id', $uncertainDelete->id)->count() === 1,
'An unknown-effect destructive delete did not serialize lifecycle policy.');
$assert($replayable->service_status === 'expired_grace'
&& $replayable->network_status === 'blocking_expired'
&& CloudJob::where('instance_id', $replayable->id)
->where('kind', 'network_suspend')->count() === 0,
'Terminal reinstall failure did not serialize expiry policy.');
$assert($snapshotUncertain->service_status === 'expired_grace'
&& $snapshotUncertain->network_status === 'blocking_expired'
&& CloudJob::where('instance_id', $snapshotUncertain->id)
->where('kind', 'network_suspend')->count() === 0,
'A terminal unknown-effect snapshot failure did not serialize the actual expiry mutation.');
$assert($snapshotQueued->service_status === 'expired_grace'
&& $snapshotQueued->network_status === 'blocking_expired'
&& CloudJob::where('instance_id', $snapshotQueued->id)
->where('kind', 'network_suspend')->count() === 0,
'A queued snapshot did not serialize the later expiry mutation.');
$assert($snapshotRunning->service_status === 'expired_grace'
&& $snapshotRunning->network_status === 'blocking_expired'
&& CloudJob::where('instance_id', $snapshotRunning->id)
->where('kind', 'network_suspend')->count() === 0,
'An awaiting-receipt snapshot did not serialize the later expiry mutation.');
$assert($snapshotPurge->status === 'deleting'
&& $snapshotPurge->service_status === 'purge_stopping'
&& !CloudJob::where('instance_id', $snapshotPurge->id)
->where('kind', 'service_suspend')->exists(),
'Purge shutdown bypassed an unknown-effect restore that may still recover power.');
$assert($policyPending->service_status === 'expired_grace'
&& $policyPending->network_status === 'blocking_expired'
&& CloudJob::where('instance_id', $policyPending->id)
->where('kind', 'service_suspend')->count() === 1
&& !CloudJob::where('instance_id', $policyPending->id)
->where('kind', 'network_suspend')->exists(),
'A terminal failed expiry shutdown was retained or duplicated.');
// Renewal prepayment is intentionally bounded. Rejection must happen in
// the wallet transaction before either money or ledger rows are changed.
\App\Services\Config::$values['cloud_max_renewal_periods_ahead'] = 2;
$renewalLimited = $makeInstance('aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
'renewal-limit-vm', $now + 3000);
$renewalLimited->renewal_price = '7.25';
$renewalLimited->renewal_days = 30;
$renewalLimited->renewal_traffic_bytes = 2048;
$renewalLimited->traffic_quota_bytes = 2048;
$renewalLimited->save();
$limitedCurrent = $makePeriod($renewalLimited, $now - 100, $now + 1000, 'active', 2048);
$limitedCurrent->download_base_bytes = 0;
$limitedCurrent->upload_base_bytes = 0;
$limitedCurrent->activated_at = $now - 100;
$limitedCurrent->save();
$makePeriod($renewalLimited, $now + 1000, $now + 2000, 'scheduled', 2048);
$makePeriod($renewalLimited, $now + 2000, $now + 3000, 'scheduled', 2048);
$renewalLimited->current_period_id = $limitedCurrent->id;
$renewalLimited->save();
$balanceBeforeLimit = (string) User::find(1)->money;
$periodCountBeforeLimit = CloudServicePeriod::where('instance_id', $renewalLimited->id)->count();
$renewalCountBeforeLimit = CloudRenewal::where('instance_id', $renewalLimited->id)->count();
$expiryBeforeLimit = (int) $renewalLimited->expires_at;
try {
CloudRenewalService::renew($renewalLimited, [
'idempotency_key' => 'renewal-ahead-limit-0001',
], $actor);
throw new RuntimeException('Renewal ahead limit was not enforced.');
} catch (CloudError $error) {
$assert($error->httpStatus === 409, 'Renewal ahead limit returned the wrong status.');
}
$renewalLimited->refresh();
$assert((string) User::find(1)->money === $balanceBeforeLimit
&& CloudServicePeriod::where('instance_id', $renewalLimited->id)->count() === $periodCountBeforeLimit
&& CloudRenewal::where('instance_id', $renewalLimited->id)->count() === $renewalCountBeforeLimit
&& (int) $renewalLimited->expires_at === $expiryBeforeLimit,
'Rejected over-limit renewal changed money, expiry or ledger rows.');
unset(\App\Services\Config::$values['cloud_max_renewal_periods_ahead']);
// A newer log timestamp with lower cumulative counters indicates a log
// reset or corrupt observation. Billing totals and blocked networking are
// monotonic and must not roll back with it.
$rollback = $makeInstance('bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb',
'counter-rollback-vm', $now + 3600, 'active', 'blocked_traffic');
$rollback->traffic_quota_bytes = 1000;
$rollback->lifetime_download_bytes = 1000;
$rollback->lifetime_upload_bytes = 500;
$rollback->traffic_download_bytes = 1000;
$rollback->traffic_upload_bytes = 500;
$rollback->traffic_synced_at = $now - 10;
$rollback->save();
$rollbackPeriod = $makePeriod($rollback, $now - 100, $now + 3600, 'exhausted', 1000);
$rollbackPeriod->download_base_bytes = 0;
$rollbackPeriod->upload_base_bytes = 0;
$rollbackPeriod->download_used_bytes = 1000;
$rollbackPeriod->upload_used_bytes = 500;
$rollbackPeriod->activated_at = $now - 100;
$rollbackPeriod->save();
$rollback->current_period_id = $rollbackPeriod->id;
$rollback->save();
$capsule->connection('log')->table('cloud_vm_traffic_totals')->insert([
'host_scope' => $scope,
'vm_uuid' => $rollback->uuid,
'host_id' => '1',
'user_id' => 1,
'last_observed_at' => $now + 100,
'download_rx_bytes' => 100,
'upload_tx_bytes' => 50,
'updated_at' => $now + 100,
]);
$lifecycle->tick(100);
$rollback->refresh();
$rollbackPeriod->refresh();
$assert((int) $rollback->lifetime_download_bytes === 1000
&& (int) $rollback->lifetime_upload_bytes === 500
&& (int) $rollback->traffic_synced_at === $now - 10
&& (int) $rollbackPeriod->download_used_bytes === 1000
&& (int) $rollbackPeriod->upload_used_bytes === 500,
'A newer but lower log total rolled billing counters backward.');
$assert($rollback->network_status === 'blocked_traffic'
&& !CloudJob::where('instance_id', $rollback->id)->where('kind', 'network_resume')->exists(),
'Counter rollback incorrectly restored an exhausted VM network.');
// Hourly cumulative checkpoints survive the 30-day chart retention and
// keep an independent billing boundary reconstructable after a prolonged
// lifecycle-daemon outage.
$checkpointed = $makeInstance('cccccccc-cccc-4ccc-8ccc-cccccccccccc',
'checkpoint-boundary-vm', $now + 3600);
$checkpointed->lifetime_download_bytes = 1000;
$checkpointed->lifetime_upload_bytes = 500;
$checkpointed->traffic_synced_at = $now;
$checkpointed->save();
$checkpointPeriod = $makePeriod($checkpointed, $now - 3600, $now + 3600,
'scheduled', 10000);
$capsule->connection('log')->table('cloud_vm_traffic_checkpoints')->insert([
'host_scope' => $scope,
'vm_uuid' => $checkpointed->uuid,
'checkpoint_hour' => intdiv($now - 3540, 3600),
'host_id' => '1',
'user_id' => 1,
'first_observed_at' => $now - 3540,
'download_rx_bytes' => 400,
'upload_tx_bytes' => 200,
]);
$lifecycle->tick(100);
$checkpointed->refresh();
$checkpointPeriod->refresh();
$assert($checkpointPeriod->status === 'active'
&& (int) $checkpointPeriod->download_base_bytes === 400
&& (int) $checkpointPeriod->upload_base_bytes === 200
&& (int) $checkpointPeriod->download_used_bytes === 600
&& (int) $checkpointPeriod->upload_used_bytes === 300,
'Durable checkpoint did not reconstruct the independent period boundary.');
echo "Cloud independent-period renewal, traffic quota, expiry grace and purge lifecycle checks passed.\n";
}