You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
317 lines
12 KiB
PHP
317 lines
12 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudServiceCompensation;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudServiceCompensationNotificationService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->string('email');
|
|
});
|
|
$schema->create('cloud_service_compensations', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->string('compensation_key');
|
|
$table->string('incident_key');
|
|
$table->text('scope_snapshot_json');
|
|
$table->string('reason_text');
|
|
$table->integer('incident_starts_at')->nullable();
|
|
$table->integer('incident_ends_at')->nullable();
|
|
$table->integer('compensation_seconds');
|
|
$table->boolean('notify_users');
|
|
$table->integer('recipient_count')->default(0);
|
|
$table->integer('instance_count')->default(1);
|
|
$table->integer('applied_count')->default(0);
|
|
$table->integer('deferred_count')->default(0);
|
|
$table->integer('rejected_count')->default(0);
|
|
$table->integer('attention_count')->default(0);
|
|
$table->string('state');
|
|
$table->string('last_error_code')->default('');
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_service_compensation_items', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->integer('batch_id');
|
|
$table->integer('user_id');
|
|
$table->integer('instance_id');
|
|
$table->string('instance_name');
|
|
$table->integer('requested_seconds');
|
|
$table->string('state');
|
|
$table->integer('before_expires_at')->nullable();
|
|
$table->integer('after_expires_at')->nullable();
|
|
});
|
|
$schema->create('cloud_service_compensation_recipients', static function (
|
|
Blueprint $table
|
|
): void {
|
|
$table->increments('id');
|
|
$table->integer('batch_id');
|
|
$table->integer('user_id');
|
|
$table->string('email_snapshot');
|
|
$table->integer('instance_count');
|
|
$table->text('payload_json');
|
|
$table->string('payload_sha256');
|
|
$table->string('status');
|
|
$table->integer('attempts')->default(0);
|
|
$table->integer('available_at');
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('sent_at')->nullable();
|
|
$table->string('last_error_code')->default('');
|
|
$table->string('last_error')->default('');
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
$table->unique(['batch_id', 'user_id']);
|
|
});
|
|
|
|
$now = 2_100_100_000;
|
|
Capsule::table('user')->insert([
|
|
['id' => 1, 'email' => 'owner@example.test'],
|
|
['id' => 2, 'email' => 'second@example.test'],
|
|
]);
|
|
$makeBatch = static function (
|
|
int $id,
|
|
bool $notify,
|
|
string $state = 'completed'
|
|
) use ($now): void {
|
|
$terminal = in_array($state, ['completed', 'partial', 'attention'], true);
|
|
Capsule::table('cloud_service_compensations')->insert([
|
|
'id' => $id,
|
|
'compensation_key' => str_pad((string) $id, 32, 'a'),
|
|
'incident_key' => 'incident-' . $id,
|
|
'scope_snapshot_json' => json_encode([
|
|
'label' => 'HKG-A',
|
|
], JSON_THROW_ON_ERROR),
|
|
'reason_text' => "网络故障 <{$id}>",
|
|
'incident_starts_at' => $now - 600,
|
|
'incident_ends_at' => $now - 300,
|
|
'compensation_seconds' => 3600,
|
|
'notify_users' => $notify ? 1 : 0,
|
|
'recipient_count' => 0,
|
|
'instance_count' => 1,
|
|
'applied_count' => $terminal ? 1 : 0,
|
|
'deferred_count' => 0,
|
|
'rejected_count' => 0,
|
|
'attention_count' => 0,
|
|
'state' => $state,
|
|
'last_error_code' => '',
|
|
'terminal_at' => $terminal ? $now : null,
|
|
'updated_at' => $now,
|
|
]);
|
|
};
|
|
$addItem = static function (
|
|
int $batchId,
|
|
int $userId,
|
|
int $instanceId,
|
|
string $state = 'applied'
|
|
) use ($now): void {
|
|
Capsule::table('cloud_service_compensation_items')->insert([
|
|
'batch_id' => $batchId,
|
|
'user_id' => $userId,
|
|
'instance_id' => $instanceId,
|
|
'instance_name' => 'vm-<' . $instanceId . '>',
|
|
'requested_seconds' => 3600,
|
|
'state' => $state,
|
|
'before_expires_at' => $state === 'applied' ? $now + 1000 : null,
|
|
'after_expires_at' => $state === 'applied' ? $now + 4600 : null,
|
|
]);
|
|
};
|
|
|
|
$sent = [];
|
|
$leaseObserved = false;
|
|
$service = new CloudServiceCompensationNotificationService(
|
|
static function (string $to, string $subject, array $variables) use (
|
|
&$sent,
|
|
&$leaseObserved,
|
|
&$now
|
|
): bool {
|
|
$sending = Capsule::table('cloud_service_compensation_recipients')
|
|
->where('email_snapshot', $to)->where('status', 'sending')
|
|
->orderByDesc('id')->first();
|
|
$leaseObserved = $sending !== null
|
|
&& (int) $sending->attempts === 1
|
|
&& (int) $sending->lease_until === $now + 300;
|
|
$sent[] = compact('to', 'subject', 'variables');
|
|
return true;
|
|
},
|
|
static fn (string $email): bool => false,
|
|
static function () use (&$now): int { return $now; }
|
|
);
|
|
|
|
$makeBatch(1, true, 'pending');
|
|
$addItem(1, 1, 101, 'pending');
|
|
try {
|
|
$service->freezeLocked(CloudServiceCompensation::findOrFail(1), $now);
|
|
throw new RuntimeException('Open batch notification was frozen.');
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === 409,
|
|
'Open batch freeze returned the wrong error.');
|
|
}
|
|
|
|
$makeBatch(2, true);
|
|
$addItem(2, 1, 201);
|
|
$addItem(2, 1, 202);
|
|
$batch = CloudServiceCompensation::findOrFail(2);
|
|
$batch->instance_count = 2;
|
|
$batch->applied_count = 2;
|
|
$batch->save();
|
|
$service->freezeLocked($batch, $now);
|
|
$service->freezeLocked(CloudServiceCompensation::findOrFail(2), $now);
|
|
$recipient = Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 2)->first();
|
|
$payload = json_decode((string) $recipient->payload_json, true,
|
|
64, JSON_THROW_ON_ERROR);
|
|
$assert((int) $recipient->instance_count === 2
|
|
&& (int) $recipient->available_at === $now
|
|
&& (int) CloudServiceCompensation::findOrFail(2)->recipient_count === 1
|
|
&& count($payload['instances']) === 2,
|
|
'Terminal applied instances were not frozen by owner.');
|
|
$assert(hash_equals((string) $recipient->payload_sha256,
|
|
hash('sha256', (string) $recipient->payload_json)),
|
|
'Frozen notification payload digest is invalid.');
|
|
$assert($service->tick() === true && count($sent) === 1 && $leaseObserved,
|
|
'Frozen compensation mail was not delivered.');
|
|
$delivered = Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 2)->first();
|
|
$assert((string) $delivered->status === 'sent'
|
|
&& (int) $delivered->sent_at === $now
|
|
&& str_contains((string) $sent[0]['variables']['reason_text'], '<2>'),
|
|
'Successful compensation delivery did not converge.');
|
|
$assert($service->tick() === false && count($sent) === 1,
|
|
'Sent mail was replayed.');
|
|
|
|
$makeBatch(3, false);
|
|
$addItem(3, 1, 301);
|
|
$service->freezeLocked(CloudServiceCompensation::findOrFail(3), $now);
|
|
$assert(!Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 3)->exists(),
|
|
'Notification-disabled batch created a recipient.');
|
|
|
|
$makeBatch(4, true);
|
|
$addItem(4, 2, 401);
|
|
$service->freezeLocked(CloudServiceCompensation::findOrFail(4), $now);
|
|
$unknown = new CloudServiceCompensationNotificationService(
|
|
static fn (): bool => false,
|
|
static fn (): bool => false,
|
|
static function () use (&$now): int { return $now; }
|
|
);
|
|
$assert($unknown->tick() === true,
|
|
'Rejected SMTP handoff was not processed.');
|
|
$assert((string) Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 4)->value('status') === 'unknown'
|
|
&& $unknown->tick() === false,
|
|
'Unknown SMTP effect was replayed.');
|
|
|
|
$makeBatch(5, true);
|
|
$addItem(5, 2, 501);
|
|
$service->freezeLocked(CloudServiceCompensation::findOrFail(5), $now);
|
|
Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 5)->update([
|
|
'status' => 'sending', 'attempts' => 1,
|
|
'lease_token' => str_repeat('f', 32), 'lease_until' => $now,
|
|
]);
|
|
$assert($service->tick() === true
|
|
&& (string) Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 5)->value('status') === 'unknown'
|
|
&& count($sent) === 1,
|
|
'Expired delivery lease was not fenced without replay.');
|
|
|
|
$makeBatch(6, true);
|
|
$addItem(6, 2, 601);
|
|
$service->freezeLocked(CloudServiceCompensation::findOrFail(6), $now);
|
|
Capsule::table('user')->where('id', 2)->update([
|
|
'email' => 'changed@example.test',
|
|
]);
|
|
$service->freezeLocked(CloudServiceCompensation::findOrFail(6), $now);
|
|
$assert($service->tick() === true
|
|
&& (string) Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 6)->value('status') === 'skipped'
|
|
&& count($sent) === 1,
|
|
'Changed recipient email was not skipped before SMTP.');
|
|
|
|
Capsule::table('user')->where('id', 2)->update([
|
|
'email' => 'second@example.test',
|
|
]);
|
|
$makeBatch(7, true);
|
|
$addItem(7, 1, 701);
|
|
$service->freezeLocked(CloudServiceCompensation::findOrFail(7), $now);
|
|
$throws = new CloudServiceCompensationNotificationService(
|
|
static function (): bool {
|
|
throw new RuntimeException('SMTP effect cannot be confirmed.');
|
|
},
|
|
static fn (): bool => false,
|
|
static function () use (&$now): int { return $now; }
|
|
);
|
|
$assert($throws->tick() === true
|
|
&& (string) Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 7)->value('status') === 'unknown'
|
|
&& $throws->tick() === false,
|
|
'Thrown SMTP handoff was not fenced as non-replayable unknown.');
|
|
|
|
$makeBatch(8, true);
|
|
$addItem(8, 1, 801);
|
|
$service->freezeLocked(CloudServiceCompensation::findOrFail(8), $now);
|
|
$blacklisted = new CloudServiceCompensationNotificationService(
|
|
static function (): bool {
|
|
throw new RuntimeException('Blacklisted mail reached SMTP.');
|
|
},
|
|
static fn (string $email): bool => $email === 'owner@example.test',
|
|
static function () use (&$now): int { return $now; }
|
|
);
|
|
$assert($blacklisted->tick() === true
|
|
&& (string) Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 8)->value('status') === 'skipped'
|
|
&& (string) Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 8)->value('last_error_code')
|
|
=== 'recipient_blacklisted',
|
|
'Blacklisted recipient was not skipped before SMTP.');
|
|
|
|
$makeBatch(9, true);
|
|
$addItem(9, 2, 901);
|
|
$service->freezeLocked(CloudServiceCompensation::findOrFail(9), $now);
|
|
Capsule::table('user')->where('id', 2)->delete();
|
|
$assert($service->tick() === true
|
|
&& (string) Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 9)->value('status') === 'skipped'
|
|
&& (string) Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 9)->value('last_error_code') === 'user_deleted'
|
|
&& count($sent) === 1,
|
|
'Deleted recipient was not skipped before SMTP.');
|
|
|
|
$makeBatch(10, true, 'partial');
|
|
$addItem(10, 1, 1001, 'applied');
|
|
$addItem(10, 2, 1002, 'rejected');
|
|
$partial = CloudServiceCompensation::findOrFail(10);
|
|
$partial->instance_count = 2;
|
|
$partial->applied_count = 1;
|
|
$partial->rejected_count = 1;
|
|
$partial->save();
|
|
$service->freezeLocked($partial, $now);
|
|
$assert(Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 10)->count() === 1
|
|
&& (int) Capsule::table('cloud_service_compensation_recipients')
|
|
->where('batch_id', 10)->value('user_id') === 1,
|
|
'Non-applied compensation item leaked into the recipient projection.');
|
|
|
|
echo "Service compensation notification tests passed.\n";
|