You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudServiceCompensationExe...

579 lines
26 KiB
PHP

<?php
namespace App\Services\Cloud {
/** Keep this focused transaction test independent from live key files. */
final class CloudSecrets
{
public static function encrypt(array $payload, string $purpose): string
{
return json_encode([
'purpose' => $purpose,
'payload' => $payload,
], JSON_THROW_ON_ERROR);
}
}
}
namespace {
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstance;
use App\Models\CloudJob;
use App\Models\CloudServiceCompensation;
use App\Models\CloudServiceCompensationItem;
use App\Models\CloudServicePeriod;
use App\Services\Cloud\CloudServiceCompensationExecutionService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$capsule = new Capsule();
$capsule->addConnection([
'driver' => 'sqlite', 'database' => ':memory:',
], 'default');
$capsule->setAsGlobal();
$capsule->bootEloquent();
$schema = $capsule->schema('default');
$schema->create('user', static function (Blueprint $table): void {
$table->increments('id');
$table->string('money');
$table->string('email')->default('');
$table->integer('enable')->default(1);
$table->boolean('is_admin')->default(false);
});
$schema->create('cloud_instances', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('host_id');
$table->integer('user_id');
$table->integer('order_id')->nullable();
$table->integer('plan_id')->nullable();
$table->integer('offer_id')->nullable();
$table->integer('current_period_id')->nullable();
$table->string('uuid');
$table->string('name');
$table->string('status');
$table->string('service_status');
$table->string('network_status');
$table->integer('expires_at')->nullable();
$table->string('renewal_price')->nullable();
$table->integer('renewal_days')->default(0);
$table->integer('renewal_traffic_bytes')->default(0);
$table->integer('traffic_quota_bytes')->default(0);
$table->text('spec_json');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_service_periods', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('plan_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('renewal_id')->nullable();
$table->integer('traffic_pool_group_id')->nullable();
$table->string('traffic_quota_mode')->default('finite');
$table->integer('starts_at');
$table->integer('ends_at');
$table->integer('traffic_quota_bytes')->default(0);
$table->integer('download_base_bytes')->nullable();
$table->integer('upload_base_bytes')->nullable();
$table->integer('download_used_bytes')->default(0);
$table->integer('upload_used_bytes')->default(0);
$table->string('status');
$table->integer('activated_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_invoices', static function (Blueprint $table): void {
$table->increments('id');
$table->string('invoice_no')->unique();
$table->string('purpose');
$table->integer('instance_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('user_id');
$table->integer('plan_id')->nullable();
$table->integer('renewal_id')->nullable();
$table->integer('renews_expires_at')->nullable();
$table->integer('due_at');
$table->string('amount');
$table->string('currency')->default('USD');
$table->integer('validity_days');
$table->integer('traffic_quota_bytes');
$table->string('status');
$table->string('paid_via')->nullable();
$table->string('payment_reference')->nullable();
$table->integer('paid_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_invoice_payments', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('invoice_id');
$table->integer('user_id');
$table->string('provider');
$table->string('provider_order_no');
$table->string('idempotency_key');
$table->string('amount');
$table->string('status');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_invoice_items', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('invoice_id');
$table->integer('order_item_id')->nullable();
$table->string('code');
$table->string('label');
$table->string('amount');
$table->string('currency')->default('USD');
$table->text('snapshot_json');
$table->integer('created_at');
});
$schema->create('cloud_mail_outbox', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('invoice_id');
$table->string('kind');
$table->string('status');
$table->integer('attempts')->default(0);
$table->integer('available_at');
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('sent_at')->nullable();
$table->string('last_error')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_jobs', static function (Blueprint $table): void {
$table->increments('id');
$table->string('operation_id')->nullable()->unique();
$table->string('request_idempotency_key')->nullable();
$table->string('request_fingerprint')->nullable();
$table->integer('parent_job_id')->nullable();
$table->integer('supersedes_job_id')->nullable();
$table->integer('vpc_repair_root_job_id')->nullable();
$table->integer('order_id')->nullable();
$table->integer('invoice_id')->nullable();
$table->integer('instance_id')->nullable();
$table->integer('snapshot_id')->nullable();
$table->integer('backup_id')->nullable();
$table->integer('backup_restore_id')->nullable();
$table->integer('backup_recovery_id')->nullable();
$table->integer('public_network_transition_id')->nullable();
$table->integer('vpc_id')->nullable();
$table->integer('vpc_port_id')->nullable();
$table->integer('vpc_batch_id')->nullable();
$table->integer('user_id');
$table->integer('host_id')->nullable();
$table->string('kind');
$table->string('execution_mode');
$table->string('phase')->default('accepted');
$table->string('outcome')->nullable();
$table->string('status')->default('queued');
$table->text('sealed_payload');
$table->integer('attempts')->default(0);
$table->integer('dispatch_attempts')->default(0);
$table->integer('dispatch_deadline_at');
$table->integer('dispatch_started_at')->nullable();
$table->integer('agent_accepted_at')->nullable();
$table->integer('receipt_deadline_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->boolean('effect_may_have_applied')->nullable();
$table->boolean('safe_to_resubmit')->nullable();
$table->string('effect_resolution')->default('not_applicable');
$table->integer('effect_resolved_at')->nullable();
$table->integer('effect_resolved_by')->nullable();
$table->text('effect_resolution_reason')->default('');
$table->boolean('finalization_incident')->default(false);
$table->string('finalization_error_code')->nullable();
$table->string('completion_event_id')->nullable();
$table->integer('read_at')->nullable();
$table->integer('dispatched_at')->nullable();
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('available_at');
$table->string('error_code')->nullable();
$table->text('result_json')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_traffic_entitlements', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('service_period_id');
});
$schema->create('cloud_backup_subscriptions', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('instance_id')->unique();
$table->integer('user_id');
});
$schema->create('cloud_backup_period_entitlements', static function (Blueprint $table): void {
$table->increments('id');
$table->integer('subscription_id');
$table->integer('instance_id');
$table->integer('service_period_id')->unique();
$table->integer('user_id');
});
$schema->create('cloud_service_compensations', static function (Blueprint $table): void {
$table->increments('id');
$table->string('compensation_key')->unique();
$table->string('incident_key');
$table->integer('requested_by_user_id');
$table->string('request_idempotency_key');
$table->string('request_fingerprint');
$table->string('scope_type');
$table->integer('scope_id');
$table->text('scope_snapshot_json');
$table->string('reason_text');
$table->integer('incident_starts_at')->nullable();
$table->integer('incident_ends_at')->nullable();
$table->integer('compensation_seconds');
$table->boolean('notify_users');
$table->string('audience_digest');
$table->integer('instance_count');
$table->integer('recipient_count')->default(0);
$table->integer('applied_count')->default(0);
$table->integer('deferred_count')->default(0);
$table->integer('rejected_count')->default(0);
$table->integer('attention_count')->default(0);
$table->string('state');
$table->string('last_error_code')->default('');
$table->integer('terminal_at')->nullable();
$table->integer('created_at');
$table->integer('updated_at');
});
$schema->create('cloud_service_compensation_items', static function (Blueprint $table): void {
$table->increments('id');
$table->string('item_key')->unique();
$table->integer('batch_id');
$table->string('incident_key');
$table->integer('instance_id');
$table->integer('user_id');
$table->integer('host_id');
$table->string('instance_name');
$table->integer('requested_seconds');
$table->integer('service_period_id')->nullable();
$table->integer('traffic_entitlement_id')->nullable();
$table->integer('backup_entitlement_id')->nullable();
$table->integer('source_invoice_id')->nullable();
$table->integer('replacement_invoice_id')->nullable();
$table->integer('audit_job_id')->nullable();
$table->integer('before_expires_at')->nullable();
$table->integer('after_expires_at')->nullable();
$table->integer('before_period_ends_at')->nullable();
$table->integer('after_period_ends_at')->nullable();
$table->integer('before_traffic_ends_at')->nullable();
$table->integer('after_traffic_ends_at')->nullable();
$table->integer('before_backup_ends_at')->nullable();
$table->integer('after_backup_ends_at')->nullable();
$table->text('before_snapshot_json');
$table->text('after_snapshot_json')->nullable();
$table->string('state');
$table->integer('attempts')->default(0);
$table->integer('available_at');
$table->string('lease_token')->nullable();
$table->integer('lease_until')->default(0);
$table->integer('applied_at')->nullable();
$table->integer('terminal_at')->nullable();
$table->string('last_error_code')->default('');
$table->string('last_error')->default('');
$table->integer('created_at');
$table->integer('updated_at');
});
$now = 2_100_000_000;
Capsule::table('user')->insert([
'id' => 1, 'money' => '50.00', 'email' => 'owner@example.test',
'enable' => 1, 'is_admin' => 0,
]);
$makeInstance = static function (int $id, int $expiry, string $name) use (
$now
): void {
Capsule::table('cloud_instances')->insert([
'id' => $id, 'host_id' => 7, 'user_id' => 1,
'order_id' => null, 'plan_id' => 5, 'offer_id' => 9,
'current_period_id' => $id, 'uuid' => sprintf(
'00000000-0000-4000-8000-%012d', $id
), 'name' => $name, 'status' => 'active',
'service_status' => 'active', 'network_status' => 'active',
'expires_at' => $expiry, 'renewal_price' => '12.00',
'renewal_days' => 30, 'renewal_traffic_bytes' => 2048,
'traffic_quota_bytes' => 2048,
'spec_json' => json_encode(['vcpu' => 2], JSON_THROW_ON_ERROR),
'created_at' => $now - 1000, 'updated_at' => $now - 100,
]);
Capsule::table('cloud_service_periods')->insert([
'id' => $id, 'instance_id' => $id, 'user_id' => 1,
'plan_id' => 5, 'order_id' => null, 'renewal_id' => null,
'traffic_pool_group_id' => null,
'traffic_quota_mode' => 'finite', 'starts_at' => $now - 1000,
'ends_at' => $expiry, 'traffic_quota_bytes' => 2048,
'download_base_bytes' => 0, 'upload_base_bytes' => 0,
'download_used_bytes' => 0, 'upload_used_bytes' => 0,
'status' => 'active', 'activated_at' => $now - 1000,
'created_at' => $now - 1000, 'updated_at' => $now - 100,
]);
};
$makeBatch = static function (int $id, int $instanceId, string $suffix) use (
$now
): void {
Capsule::table('cloud_service_compensations')->insert([
'id' => $id, 'compensation_key' => str_repeat($suffix, 32),
'incident_key' => 'incident-' . $suffix,
'requested_by_user_id' => 99,
'request_idempotency_key' => 'request-key-' . str_repeat($suffix, 16),
'request_fingerprint' => str_repeat($suffix, 64),
'scope_type' => 'instance', 'scope_id' => $instanceId,
'scope_snapshot_json' => '{}', 'reason_text' => 'test outage',
'incident_starts_at' => null, 'incident_ends_at' => null,
'compensation_seconds' => 3600, 'notify_users' => 0,
'audience_digest' => str_repeat($suffix, 64),
'instance_count' => 1, 'recipient_count' => 0,
'applied_count' => 0, 'deferred_count' => 0,
'rejected_count' => 0, 'attention_count' => 0,
'state' => 'pending', 'last_error_code' => '',
'terminal_at' => null, 'created_at' => $now,
'updated_at' => $now,
]);
Capsule::table('cloud_service_compensation_items')->insert([
'id' => $id, 'item_key' => str_repeat($suffix, 32),
'batch_id' => $id, 'incident_key' => 'incident-' . $suffix,
'instance_id' => $instanceId, 'user_id' => 1, 'host_id' => 7,
'instance_name' => 'vm-' . $suffix, 'requested_seconds' => 3600,
'before_snapshot_json' => '{}', 'state' => 'pending',
'attempts' => 0, 'available_at' => $now,
'lease_token' => null, 'lease_until' => 0,
'last_error_code' => '', 'last_error' => '',
'created_at' => $now, 'updated_at' => $now,
]);
};
$oldExpiry = $now + 10 * 86400;
$makeInstance(1, $oldExpiry, 'vm-success');
$makeBatch(1, 1, 'a');
$clockNow = $now;
$execution = new CloudServiceCompensationExecutionService(
static function () use (&$clockNow): int {
return $clockNow;
}
);
$claim = $execution->claimOne(1);
$assert(is_array($claim), 'Success item was not claimed.');
$result = $execution->applyClaim(
(int) $claim['item_id'], (string) $claim['token']
);
$assert(($result['state'] ?? '') === 'applied',
'Success item did not reach applied.');
$instance = CloudInstance::findOrFail(1);
$period = CloudServicePeriod::findOrFail(1);
$item = CloudServiceCompensationItem::findOrFail(1);
$batch = CloudServiceCompensation::findOrFail(1);
$assert((int) $instance->expires_at === $oldExpiry + 3600,
'Instance expiry was not extended exactly once.');
$assert((int) $period->ends_at === $oldExpiry + 3600,
'Tail period was not extended exactly once.');
$assert((string) Capsule::table('user')->where('id', 1)->value('money')
=== '50.00', 'Compensation changed wallet balance.');
$assert(Capsule::table('cloud_traffic_entitlements')->count() === 0
&& Capsule::table('cloud_backup_period_entitlements')->count() === 0,
'Compensation created a new paid entitlement.');
$assert((string) $item->state === 'applied'
&& (int) $item->audit_job_id > 0
&& (int) $item->available_at === $now
&& (string) $batch->state === 'completed'
&& (int) $batch->applied_count === 1,
'Applied item or batch projection is incomplete.');
$afterEvidence = json_decode(
(string) $item->after_snapshot_json,
true,
64,
JSON_THROW_ON_ERROR
);
$assert((string) $item->before_snapshot_json === '{}'
&& (int) ($afterEvidence['applied_from']['instance']['expires_at'] ?? 0)
=== $oldExpiry
&& (int) ($afterEvidence['applied_to']['instance']['expires_at'] ?? 0)
=== $oldExpiry + 3600,
'Reviewed evidence changed or application evidence is incomplete.');
$job = CloudJob::findOrFail((int) $item->audit_job_id);
$assert((string) $job->kind === 'service_compensation'
&& (string) $job->execution_mode === 'local'
&& (string) $job->phase === 'terminal'
&& (string) $job->outcome === 'succeeded'
&& (string) $job->effect_resolution === 'applied',
'Local compensation audit job is not terminal success.');
$execution->applyClaim((int) $claim['item_id'], (string) $claim['token']);
$assert((int) CloudInstance::findOrFail(1)->expires_at
=== $oldExpiry + 3600
&& CloudJob::where('instance_id', 1)->count() === 1,
'Idempotent replay extended time or duplicated the audit job.');
$pendingExpiry = $oldExpiry + 100;
$makeInstance(2, $pendingExpiry, 'vm-payment-open');
$makeBatch(2, 2, 'b');
Capsule::table('cloud_invoices')->insert([
'id' => 2, 'invoice_no' => 'INV-PENDING', 'purpose' => 'renewal',
'instance_id' => 2, 'order_id' => null, 'user_id' => 1,
'plan_id' => 5, 'renewal_id' => null,
'renews_expires_at' => $pendingExpiry, 'due_at' => $pendingExpiry,
'amount' => '12.00', 'currency' => 'USD', 'validity_days' => 30,
'traffic_quota_bytes' => 2048, 'status' => 'open',
'paid_via' => null, 'payment_reference' => null, 'paid_at' => null,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_invoice_payments')->insert([
'invoice_id' => 2, 'user_id' => 1, 'provider' => 'stripe_card',
'provider_order_no' => 'pending-order',
'idempotency_key' => 'pending-payment-key', 'amount' => '12.00',
'status' => 'pending', 'created_at' => $now, 'updated_at' => $now,
]);
$claim = $execution->claimOne(2);
$assert(is_array($claim), 'Payment-fenced item was not claimed.');
$deferred = $execution->applyClaim(
(int) $claim['item_id'], (string) $claim['token']
);
$assert(($deferred['state'] ?? '') === 'deferred'
&& ($deferred['last_error_code'] ?? '') === 'renewal_payment_open',
'Pending payment did not defer compensation.');
$firstWait = CloudServiceCompensationItem::findOrFail(2);
$assert((int) $firstWait->attempts === 0
&& (int) $firstWait->available_at === $clockNow + 300,
'External payment wait consumed retry budget or used unbounded polling.');
$assert((int) CloudInstance::findOrFail(2)->expires_at === $pendingExpiry
&& (int) CloudServicePeriod::findOrFail(2)->ends_at === $pendingExpiry
&& CloudJob::where('instance_id', 2)->count() === 0
&& (string) Capsule::table('cloud_invoices')
->where('id', 2)->value('status') === 'open',
'Deferred payment fence changed business state.');
// An external payment can legitimately remain pending longer than the
// worker's technical retry budget. Rechecking that unchanged fact must
// not turn a still-recoverable compensation into terminal attention.
for ($wait = 1; $wait < 10; $wait++) {
$waiting = CloudServiceCompensationItem::findOrFail(2);
if ((string) $waiting->state !== 'deferred') {
break;
}
$clockNow = (int) $waiting->available_at;
$waitClaim = $execution->claimOne(2);
if ($waitClaim === null) {
break;
}
$execution->applyClaim(
(int) $waitClaim['item_id'],
(string) $waitClaim['token']
);
}
$waiting = CloudServiceCompensationItem::findOrFail(2);
$assert((string) $waiting->state === 'deferred'
&& (int) $waiting->attempts === 0
&& (string) $waiting->last_error_code === 'renewal_payment_open',
'External payment wait exhausted the technical retry budget.');
Capsule::table('cloud_invoice_payments')->where('invoice_id', 2)->delete();
$clockNow = (int) $waiting->available_at;
$resumedClaim = $execution->claimOne(2);
$assert(is_array($resumedClaim),
'Resolved payment wait did not make compensation claimable.');
$resumed = $execution->applyClaim(
(int) $resumedClaim['item_id'],
(string) $resumedClaim['token']
);
$assert(($resumed['state'] ?? '') === 'applied'
&& (int) CloudInstance::findOrFail(2)->expires_at
=== $pendingExpiry + 3600,
'Resolved payment wait did not resume compensation exactly once.');
$mailExpiry = $clockNow + 3600;
$makeInstance(3, $mailExpiry, 'vm-renewal-mail-sending');
$makeBatch(3, 3, 'c');
Capsule::table('cloud_invoices')->insert([
'id' => 3, 'invoice_no' => 'INV-MAIL-SENDING',
'purpose' => 'renewal', 'instance_id' => 3, 'order_id' => null,
'user_id' => 1, 'plan_id' => 5, 'renewal_id' => null,
'renews_expires_at' => $mailExpiry, 'due_at' => $mailExpiry,
'amount' => '12.00', 'currency' => 'USD', 'validity_days' => 30,
'traffic_quota_bytes' => 2048, 'status' => 'open',
'paid_via' => null, 'payment_reference' => null, 'paid_at' => null,
'created_at' => $now, 'updated_at' => $now,
]);
Capsule::table('cloud_mail_outbox')->insert([
'invoice_id' => 3, 'kind' => 'renewal_due', 'status' => 'sending',
'attempts' => 1, 'available_at' => $clockNow,
'lease_token' => str_repeat('f', 32),
'lease_until' => $clockNow + 300, 'sent_at' => null,
'last_error' => '', 'created_at' => $now, 'updated_at' => $now,
]);
$mailClaim = $execution->claimOne(3);
$assert(is_array($mailClaim),
'Mail-fenced compensation item was not claimed.');
$mailDeferred = $execution->applyClaim(
(int) $mailClaim['item_id'],
(string) $mailClaim['token']
);
$mailWaiting = CloudServiceCompensationItem::findOrFail(3);
$assert(($mailDeferred['state'] ?? '') === 'deferred'
&& ($mailDeferred['last_error_code'] ?? '')
=== 'renewal_mail_sending'
&& (int) $mailWaiting->attempts === 0
&& (int) $mailWaiting->available_at === $clockNow + 300
&& (int) CloudInstance::findOrFail(3)->expires_at === $mailExpiry
&& (string) Capsule::table('cloud_invoices')
->where('id', 3)->value('status') === 'open'
&& (string) Capsule::table('cloud_mail_outbox')
->where('invoice_id', 3)->value('status') === 'sending',
'Sending renewal reminder was not a recoverable compensation fence.');
for ($wait = 1; $wait < 10; $wait++) {
$mailWaiting = CloudServiceCompensationItem::findOrFail(3);
$clockNow = (int) $mailWaiting->available_at;
$repeatMailClaim = $execution->claimOne(3);
$assert(is_array($repeatMailClaim),
'Long-running renewal mail stopped being claimable for recheck.');
$execution->applyClaim(
(int) $repeatMailClaim['item_id'],
(string) $repeatMailClaim['token']
);
}
$mailWaiting = CloudServiceCompensationItem::findOrFail(3);
$assert((string) $mailWaiting->state === 'deferred'
&& (int) $mailWaiting->attempts === 0
&& (string) $mailWaiting->last_error_code
=== 'renewal_mail_sending',
'Long-running renewal mail exhausted compensation retry budget.');
Capsule::table('cloud_mail_outbox')->where('invoice_id', 3)->update([
'status' => 'failed', 'lease_token' => null, 'lease_until' => 0,
'available_at' => 0,
'last_error' => '邮件投递结果已核对', 'updated_at' => $clockNow,
]);
$clockNow = (int) $mailWaiting->available_at;
$mailResumedClaim = $execution->claimOne(3);
$assert(is_array($mailResumedClaim),
'Resolved renewal-mail fence did not make compensation claimable.');
$mailApplied = $execution->applyClaim(
(int) $mailResumedClaim['item_id'],
(string) $mailResumedClaim['token']
);
$replacementInvoiceId = (int) CloudServiceCompensationItem::findOrFail(3)
->replacement_invoice_id;
$assert(($mailApplied['state'] ?? '') === 'applied'
&& (int) CloudInstance::findOrFail(3)->expires_at
=== $mailExpiry + 3600
&& (string) Capsule::table('cloud_invoices')
->where('id', 3)->value('status') === 'void'
&& $replacementInvoiceId > 0
&& Capsule::table('cloud_mail_outbox')
->where('invoice_id', $replacementInvoiceId)
->where('kind', 'renewal_due')->where('status', 'pending')
->exists(),
'Resolved renewal-mail fence did not replace the invoice exactly once.');
echo "Service compensation execution tests passed.\n";
}