You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
719 lines
28 KiB
PHP
719 lines
28 KiB
PHP
<?php
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Services\Cloud {
|
|
/** Isolated encrypted envelope for Agent mutation identity tests. */
|
|
final class CloudSecrets
|
|
{
|
|
private static array $values = [];
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
{
|
|
$key = bin2hex(random_bytes(16));
|
|
self::$values[$key] = $value;
|
|
return 'agent-mutation-identity-test:' . $key;
|
|
}
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
{
|
|
$key = substr(
|
|
$value,
|
|
strlen('agent-mutation-identity-test:')
|
|
);
|
|
if (!array_key_exists($key, self::$values)) {
|
|
throw new \RuntimeException('Missing identity test payload.');
|
|
}
|
|
return self::$values[$key];
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
use App\Models\CloudJob;
|
|
use App\Services\Cloud\CloudError;
|
|
use App\Services\Cloud\CloudOperationCompletionService;
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
use App\Services\Cloud\CloudOperationRecoveryPolicy;
|
|
use App\Services\Cloud\CloudRecoveryOperationIdentityConflict;
|
|
use App\Services\Cloud\CloudSecrets;
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
if (!$condition) {
|
|
throw new RuntimeException($message);
|
|
}
|
|
};
|
|
$expectCloudError = static function (
|
|
callable $callback,
|
|
int $status
|
|
) use ($assert): void {
|
|
try {
|
|
$callback();
|
|
} catch (CloudError $error) {
|
|
$assert($error->httpStatus === $status,
|
|
'Unexpected CloudError status.');
|
|
return;
|
|
}
|
|
throw new RuntimeException('Expected CloudError was not thrown.');
|
|
};
|
|
|
|
$capsule = new Capsule();
|
|
$capsule->addConnection([
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
], 'default');
|
|
$capsule->setAsGlobal();
|
|
$capsule->bootEloquent();
|
|
$schema = $capsule->schema('default');
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('instance_id')->nullable();
|
|
$table->integer('snapshot_id')->nullable();
|
|
$table->string('operation_id')->unique();
|
|
$table->string('execution_mode');
|
|
$table->string('kind');
|
|
$table->string('phase');
|
|
$table->string('outcome')->nullable();
|
|
$table->string('status');
|
|
$table->integer('dispatch_attempts')->default(1);
|
|
$table->integer('terminal_at')->nullable();
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
$table->string('effect_resolution')->default('not_applicable');
|
|
$table->text('sealed_payload');
|
|
$table->string('completion_event_id')->nullable();
|
|
$table->boolean('finalization_incident')->default(0);
|
|
$table->string('finalization_error_code')->nullable();
|
|
$table->integer('read_at')->nullable();
|
|
$table->string('lease_token')->nullable();
|
|
$table->integer('lease_until')->default(0);
|
|
$table->integer('available_at')->default(0);
|
|
$table->string('error_code')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_agent_completion_events',
|
|
static function (Blueprint $table): void {
|
|
$table->string('event_id')->primary();
|
|
$table->integer('host_id');
|
|
$table->string('operation_id');
|
|
$table->string('payload_sha256');
|
|
$table->string('outcome');
|
|
$table->boolean('effect_may_have_applied');
|
|
$table->boolean('safe_to_resubmit');
|
|
$table->string('error_code')->default('');
|
|
$table->text('result_json')->nullable();
|
|
$table->string('disposition')->default('received');
|
|
$table->integer('received_at');
|
|
$table->integer('applied_at')->nullable();
|
|
$table->unique(['host_id', 'operation_id']);
|
|
});
|
|
$schema->create('cloud_host_storage_configurations',
|
|
static function (Blueprint $table): void {
|
|
$table->integer('host_id')->primary();
|
|
$table->integer('revision');
|
|
foreach (['managed', 'template', 'seed', 'user_iso', 'rescue']
|
|
as $role) {
|
|
$table->text($role . '_directory');
|
|
}
|
|
$table->string('state');
|
|
$table->string('operation_id')->nullable();
|
|
$table->integer('job_id')->nullable();
|
|
$table->integer('created_at');
|
|
$table->integer('updated_at');
|
|
});
|
|
$schema->create('cloud_orders',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('user_id');
|
|
$table->integer('offer_id');
|
|
$table->integer('resource_pool_id');
|
|
$table->integer('host_id');
|
|
$table->integer('reservation_id');
|
|
$table->text('spec_json');
|
|
});
|
|
$schema->create('cloud_resource_reservations',
|
|
static function (Blueprint $table): void {
|
|
$table->increments('id');
|
|
$table->integer('order_id')->nullable();
|
|
$table->integer('backup_restore_id')->nullable();
|
|
$table->integer('offer_id')->nullable();
|
|
$table->integer('resource_pool_id');
|
|
$table->integer('host_id');
|
|
$table->integer('user_id');
|
|
$table->integer('instance_id')->nullable();
|
|
$table->string('operation_id');
|
|
$table->string('state');
|
|
});
|
|
|
|
$now = time();
|
|
$createBody = static function (string $operationId, int $suffix): array {
|
|
return [
|
|
'operation_id' => $operationId,
|
|
'source' => 'cloud',
|
|
'name' => 'identity-vm-' . $suffix,
|
|
'vcpu' => 2,
|
|
'memory_mib' => 2048,
|
|
'disk_gib' => 40,
|
|
'bandwidth_mbps' => 100,
|
|
'public_network' => [
|
|
'contract' => 1,
|
|
'network_selector' => 'bridge:br0',
|
|
'ipv4' => [
|
|
'pool_key' => str_repeat('2', 32),
|
|
'pool_cidr' => '8.8.8.0/24',
|
|
'address_cidr' => '8.8.8.' . $suffix . '/24',
|
|
'gateway' => '8.8.8.1',
|
|
'gateway_on_link' => false,
|
|
],
|
|
'ipv6' => [
|
|
'pool_key' => str_repeat('3', 32),
|
|
'pool_cidr' => '2404:c140:156:3a::/64',
|
|
'address_cidr' => '2404:c140:156:3a::'
|
|
. $suffix . '/64',
|
|
'gateway' => '2404:c140:156::1',
|
|
'gateway_on_link' => true,
|
|
],
|
|
'dns_servers' => [
|
|
'1.1.1.1',
|
|
'2606:4700:4700::1111',
|
|
],
|
|
],
|
|
'template_id' => 'debian-13',
|
|
'template_revision' => 1,
|
|
'template_generation_token' => str_repeat('1', 32),
|
|
'root_password' => 'Identity-test9!Pass',
|
|
'platform_network_policy_snapshot' => ['schema_version' => 1],
|
|
];
|
|
};
|
|
$insertCreate = static function (
|
|
int $id,
|
|
string $operationId
|
|
) use ($createBody, $now): void {
|
|
$orderId = 1000 + $id;
|
|
$reservationId = 2000 + $id;
|
|
$body = $createBody($operationId, 20 + $id);
|
|
Capsule::table('cloud_orders')->insert([
|
|
'id' => $orderId,
|
|
'user_id' => 11,
|
|
'offer_id' => 1,
|
|
'resource_pool_id' => 1,
|
|
'host_id' => 7,
|
|
'reservation_id' => $reservationId,
|
|
'spec_json' => json_encode([
|
|
'source' => 'cloud',
|
|
'public_network' => $body['public_network'],
|
|
], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES),
|
|
]);
|
|
Capsule::table('cloud_resource_reservations')->insert([
|
|
'id' => $reservationId,
|
|
'order_id' => $orderId,
|
|
'backup_restore_id' => null,
|
|
'offer_id' => 1,
|
|
'resource_pool_id' => 1,
|
|
'host_id' => 7,
|
|
'user_id' => 11,
|
|
'instance_id' => null,
|
|
'operation_id' => $operationId,
|
|
'state' => 'held',
|
|
]);
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => $id,
|
|
'host_id' => 7,
|
|
'user_id' => 11,
|
|
'order_id' => $orderId,
|
|
'operation_id' => $operationId,
|
|
'execution_mode' => 'agent',
|
|
'kind' => 'create',
|
|
'phase' => 'awaiting_receipt',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'dispatch_attempts' => 1,
|
|
'effect_resolution' => 'not_applicable',
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/vms',
|
|
'body' => $body,
|
|
'credential' => [
|
|
'password' => 'Identity-test9!Pass',
|
|
'login_name' => 'root',
|
|
'auth_mode' => 'password',
|
|
],
|
|
'template_metadata' => [
|
|
'template_id' => 'debian-13',
|
|
'os_source' => 'cloud_template',
|
|
'os_name' => 'Debian 13',
|
|
'login_name' => 'root',
|
|
'debian_12_network_fix' => false,
|
|
],
|
|
], 'job-payload'),
|
|
'created_at' => $now - 30,
|
|
'updated_at' => $now - 10,
|
|
]);
|
|
};
|
|
|
|
$createOperation = str_repeat('1a', 16);
|
|
$storageOperation = str_repeat('2b', 16);
|
|
$invalidCreateOperation = str_repeat('3c', 16);
|
|
$hashCreateOperation = str_repeat('4d', 16);
|
|
$secretCreateOperation = str_repeat('5e', 16);
|
|
$activeLedgerConflictOperation = str_repeat('6f', 16);
|
|
$terminalLedgerConflictOperation = str_repeat('70', 16);
|
|
$repairedDuringDeferOperation = str_repeat('81', 16);
|
|
$insertCreate(1, $createOperation);
|
|
$insertCreate(3, $invalidCreateOperation);
|
|
$insertCreate(4, $hashCreateOperation);
|
|
$insertCreate(5, $secretCreateOperation);
|
|
$insertCreate(6, $activeLedgerConflictOperation);
|
|
$insertCreate(7, $terminalLedgerConflictOperation);
|
|
$insertCreate(8, $repairedDuringDeferOperation);
|
|
$storageBody = [
|
|
'operation_id' => $storageOperation,
|
|
'managed_directory' => '/srv/virt/managed',
|
|
'template_directory' => '/srv/virt/templates',
|
|
'seed_directory' => '/srv/virt/seeds',
|
|
'user_iso_directory' => '/srv/virt/user-isos',
|
|
'rescue_directory' => '/srv/virt/rescue',
|
|
];
|
|
Capsule::table('cloud_jobs')->insert([
|
|
'id' => 2,
|
|
'host_id' => 8,
|
|
'user_id' => 11,
|
|
'operation_id' => $storageOperation,
|
|
'execution_mode' => 'agent',
|
|
'kind' => 'host_storage_configure',
|
|
'phase' => 'dispatching',
|
|
'outcome' => null,
|
|
'status' => 'running',
|
|
'dispatch_attempts' => 1,
|
|
'effect_resolution' => 'not_applicable',
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
'method' => 'POST',
|
|
'request_target' => '/v1/storage/configuration',
|
|
'body' => $storageBody,
|
|
'owner_id' => 11,
|
|
'can_manage' => true,
|
|
'storage_revision' => 4,
|
|
], 'job-payload'),
|
|
'created_at' => $now - 30,
|
|
'updated_at' => $now - 10,
|
|
]);
|
|
Capsule::table('cloud_host_storage_configurations')->insert([
|
|
'host_id' => 8,
|
|
'revision' => 4,
|
|
'managed_directory' => $storageBody['managed_directory'],
|
|
'template_directory' => $storageBody['template_directory'],
|
|
'seed_directory' => $storageBody['seed_directory'],
|
|
'user_iso_directory' => $storageBody['user_iso_directory'],
|
|
'rescue_directory' => $storageBody['rescue_directory'],
|
|
'state' => 'applying',
|
|
'operation_id' => $storageOperation,
|
|
'job_id' => 2,
|
|
'created_at' => $now - 30,
|
|
'updated_at' => $now - 10,
|
|
]);
|
|
|
|
$createJob = CloudJob::findOrFail(1);
|
|
$storageJob = CloudJob::findOrFail(2);
|
|
$createIdentity = CloudOperationRecoveryIdentity::expected($createJob);
|
|
$storageIdentity = CloudOperationRecoveryIdentity::expected($storageJob);
|
|
$assert(CloudOperationRecoveryIdentity::supports('create')
|
|
&& CloudOperationRecoveryIdentity::supports(
|
|
'host_storage_configure'
|
|
)
|
|
&& !CloudOperationRecoveryPolicy::supports('create')
|
|
&& !CloudOperationRecoveryPolicy::supports(
|
|
'host_storage_configure'
|
|
)
|
|
&& $createIdentity['action'] === 'create-vm'
|
|
&& preg_match(
|
|
'/^[a-f0-9]{8}-[a-f0-9]{4}-5[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/D',
|
|
$createIdentity['resource_id']
|
|
) === 1
|
|
&& $storageIdentity['action'] === 'configure-storage'
|
|
&& $storageIdentity['resource_id'] === 'host-storage',
|
|
'Agent identity support was incorrectly coupled to recovery policy.');
|
|
|
|
$operationForJob = new ReflectionMethod(
|
|
CloudWorkerService::class,
|
|
'operationForJob'
|
|
);
|
|
$validSync = [
|
|
'operation_id' => $createOperation,
|
|
'status' => 'succeeded',
|
|
] + $createIdentity;
|
|
$assert($operationForJob->invoke(null, $createJob, $validSync)
|
|
=== $validSync,
|
|
'A correctly bound synchronous create response was rejected.');
|
|
$wrongSync = $validSync;
|
|
$wrongSync['resource_id'] = 'different-create-resource';
|
|
try {
|
|
$operationForJob->invoke(null, $createJob, $wrongSync);
|
|
throw new RuntimeException(
|
|
'A cross-operation synchronous create response was accepted.'
|
|
);
|
|
} catch (CloudRecoveryOperationIdentityConflict $expected) {
|
|
// Expected: no result from the mismatched response can reach ledgers.
|
|
}
|
|
$validStorageSync = [
|
|
'operation_id' => $storageOperation,
|
|
'status' => 'succeeded',
|
|
] + $storageIdentity;
|
|
$wrongStorageSync = $validStorageSync;
|
|
$wrongStorageSync['request_fingerprint'] = str_repeat('f', 64);
|
|
try {
|
|
$operationForJob->invoke(null, $storageJob, $wrongStorageSync);
|
|
throw new RuntimeException(
|
|
'A cross-operation synchronous storage response was accepted.'
|
|
);
|
|
} catch (CloudRecoveryOperationIdentityConflict $expected) {
|
|
// Expected.
|
|
}
|
|
|
|
$receipt = static function (
|
|
string $eventId,
|
|
int $hostId,
|
|
string $operationId,
|
|
array $identity
|
|
): array {
|
|
return [
|
|
'type' => 'completion',
|
|
'id' => $eventId,
|
|
'agent_id' => (string) $hostId,
|
|
'operation_id' => $operationId,
|
|
'action' => $identity['action'],
|
|
'resource_id' => $identity['resource_id'],
|
|
'request_fingerprint' => $identity['request_fingerprint'],
|
|
'status' => 'succeeded',
|
|
'effect_may_have_applied' => false,
|
|
'safe_to_resubmit' => false,
|
|
'error_code' => '',
|
|
'result' => [],
|
|
];
|
|
};
|
|
|
|
// Completion ingestion may persist evidence for a corrupt dispatch
|
|
// ledger, but it must not ACK/consume that evidence. Both an active task
|
|
// and an apparently settled terminal task stay visibly repairable until
|
|
// dispatch_attempts is restored to one of its only valid states (0/1).
|
|
Capsule::table('cloud_jobs')->where('id', 6)->update([
|
|
'dispatch_attempts' => 2,
|
|
]);
|
|
$activeLedgerConflictJob = CloudJob::findOrFail(6);
|
|
$activeLedgerConflictReceipt = $receipt(
|
|
'active-dispatch-ledger-conflict',
|
|
7,
|
|
$activeLedgerConflictOperation,
|
|
CloudOperationRecoveryIdentity::expected(
|
|
$activeLedgerConflictJob,
|
|
true
|
|
)
|
|
);
|
|
$activeLedgerConflictAccepted =
|
|
CloudOperationCompletionService::accept(
|
|
$activeLedgerConflictReceipt
|
|
);
|
|
$activeLedgerConflictJob->refresh();
|
|
$activeLedgerConflictEvent = Capsule::table(
|
|
'cloud_agent_completion_events'
|
|
)->where(
|
|
'event_id',
|
|
$activeLedgerConflictReceipt['id']
|
|
)->first();
|
|
$assert($activeLedgerConflictAccepted === [
|
|
'accepted' => true,
|
|
'disposition' => 'received',
|
|
]
|
|
&& (string) $activeLedgerConflictJob->phase === 'terminal'
|
|
&& (string) $activeLedgerConflictJob->outcome === 'failed'
|
|
&& (bool) $activeLedgerConflictJob->effect_may_have_applied
|
|
&& !(bool) $activeLedgerConflictJob->safe_to_resubmit
|
|
&& (string) $activeLedgerConflictJob->effect_resolution === 'unknown'
|
|
&& (bool) $activeLedgerConflictJob->finalization_incident
|
|
&& (string) $activeLedgerConflictJob->finalization_error_code
|
|
=== 'recovery_dispatch_ledger_conflict'
|
|
&& (string) $activeLedgerConflictEvent->disposition === 'received'
|
|
&& $activeLedgerConflictEvent->applied_at === null,
|
|
'An active invalid dispatch ledger consumed its immutable completion: '
|
|
. json_encode([
|
|
'accepted' => $activeLedgerConflictAccepted,
|
|
'job' => $activeLedgerConflictJob->toArray(),
|
|
'event' => (array) $activeLedgerConflictEvent,
|
|
], JSON_THROW_ON_ERROR));
|
|
|
|
$terminalLedgerConflictEventId =
|
|
'terminal-dispatch-ledger-conflict';
|
|
Capsule::table('cloud_jobs')->where('id', 7)->update([
|
|
'phase' => 'terminal',
|
|
'outcome' => 'succeeded',
|
|
'status' => 'succeeded',
|
|
'dispatch_attempts' => 2,
|
|
'terminal_at' => time(),
|
|
'effect_may_have_applied' => 0,
|
|
'safe_to_resubmit' => 0,
|
|
'effect_resolution' => 'applied',
|
|
'completion_event_id' => $terminalLedgerConflictEventId,
|
|
]);
|
|
$terminalLedgerConflictJob = CloudJob::findOrFail(7);
|
|
$terminalLedgerConflictTerminalAt =
|
|
(int) $terminalLedgerConflictJob->terminal_at;
|
|
$terminalLedgerConflictReceipt = $receipt(
|
|
$terminalLedgerConflictEventId,
|
|
7,
|
|
$terminalLedgerConflictOperation,
|
|
CloudOperationRecoveryIdentity::expected(
|
|
$terminalLedgerConflictJob,
|
|
true
|
|
)
|
|
);
|
|
$terminalLedgerConflictAccepted =
|
|
CloudOperationCompletionService::accept(
|
|
$terminalLedgerConflictReceipt
|
|
);
|
|
$terminalLedgerConflictJob->refresh();
|
|
$terminalLedgerConflictEvent =
|
|
\App\Models\CloudAgentCompletionEvent::where(
|
|
'event_id',
|
|
$terminalLedgerConflictEventId
|
|
)->firstOrFail();
|
|
$assert($terminalLedgerConflictAccepted === [
|
|
'accepted' => true,
|
|
'disposition' => 'received',
|
|
]
|
|
&& (string) $terminalLedgerConflictJob->phase === 'terminal'
|
|
&& (string) $terminalLedgerConflictJob->outcome === 'succeeded'
|
|
&& (string) $terminalLedgerConflictJob->status === 'succeeded'
|
|
&& !(bool) $terminalLedgerConflictJob->effect_may_have_applied
|
|
&& !(bool) $terminalLedgerConflictJob->safe_to_resubmit
|
|
&& (string) $terminalLedgerConflictJob->effect_resolution === 'applied'
|
|
&& $terminalLedgerConflictJob->error_code === null
|
|
&& (int) $terminalLedgerConflictJob->terminal_at
|
|
=== $terminalLedgerConflictTerminalAt
|
|
&& (string) $terminalLedgerConflictJob->completion_event_id
|
|
=== $terminalLedgerConflictEventId
|
|
&& (bool) $terminalLedgerConflictJob->finalization_incident
|
|
&& (string) $terminalLedgerConflictJob->finalization_error_code
|
|
=== 'recovery_dispatch_ledger_conflict'
|
|
&& (string) $terminalLedgerConflictEvent->disposition === 'received'
|
|
&& $terminalLedgerConflictEvent->applied_at === null
|
|
&& !CloudOperationCompletionService::settledTaskWithEvent(
|
|
$terminalLedgerConflictJob,
|
|
$terminalLedgerConflictEvent
|
|
),
|
|
'A terminal invalid dispatch ledger ACKed a received completion.');
|
|
|
|
// Simulate an operator repairing the counter after a worker observed the
|
|
// invalid value but before deferDispatchLedgerConflict acquired its lock.
|
|
// The repaired value must remain sticky until background validation.
|
|
Capsule::table('cloud_jobs')->where('id', 8)->update([
|
|
'dispatch_attempts' => 2,
|
|
]);
|
|
$repairedDuringDefer = CloudJob::findOrFail(8);
|
|
Capsule::table('cloud_jobs')->where('id', 8)->update([
|
|
'dispatch_attempts' => 1,
|
|
]);
|
|
CloudOperationCompletionService::deferDispatchLedgerConflict(
|
|
$repairedDuringDefer
|
|
);
|
|
$repairedDuringDefer->refresh();
|
|
$assert((int) $repairedDuringDefer->dispatch_attempts === 1
|
|
&& (string) $repairedDuringDefer->phase === 'terminal'
|
|
&& (string) $repairedDuringDefer->outcome === 'failed'
|
|
&& (bool) $repairedDuringDefer->effect_may_have_applied
|
|
&& (string) $repairedDuringDefer->effect_resolution === 'unknown'
|
|
&& (bool) $repairedDuringDefer->finalization_incident
|
|
&& (string) $repairedDuringDefer->finalization_error_code
|
|
=== 'recovery_dispatch_ledger_conflict',
|
|
'A lock-window dispatch repair bypassed the sticky safety incident.');
|
|
|
|
$wrongCreate = $receipt(
|
|
'create-wrong-identity-event',
|
|
7,
|
|
$createOperation,
|
|
$createIdentity
|
|
);
|
|
$wrongCreate['resource_id'] = 'different-create-resource';
|
|
$createAccepted = CloudOperationCompletionService::accept($wrongCreate);
|
|
$wrongStorage = $receipt(
|
|
'storage-wrong-identity-event',
|
|
8,
|
|
$storageOperation,
|
|
$storageIdentity
|
|
);
|
|
$wrongStorage['request_fingerprint'] = str_repeat('f', 64);
|
|
$storageAccepted = CloudOperationCompletionService::accept($wrongStorage);
|
|
$createAfterWrong = CloudJob::findOrFail(1);
|
|
$storageAfterWrong = CloudJob::findOrFail(2);
|
|
$assert($createAccepted === [
|
|
'accepted' => true, 'disposition' => 'rejected',
|
|
]
|
|
&& $storageAccepted === [
|
|
'accepted' => true, 'disposition' => 'rejected',
|
|
]
|
|
&& (string) $createAfterWrong->phase === 'awaiting_receipt'
|
|
&& $createAfterWrong->outcome === null
|
|
&& (string) $createAfterWrong->effect_resolution === 'not_applicable'
|
|
&& (string) $storageAfterWrong->phase === 'dispatching'
|
|
&& $storageAfterWrong->outcome === null
|
|
&& (string) $storageAfterWrong->effect_resolution
|
|
=== 'not_applicable'
|
|
&& (string) Capsule::table('cloud_host_storage_configurations')
|
|
->where('host_id', 8)->value('state') === 'applying',
|
|
'A wrong identity bypassed create or storage deadline finalization.');
|
|
|
|
$invalidCreate = $receipt(
|
|
'create-invalid-receipt-event',
|
|
7,
|
|
$invalidCreateOperation,
|
|
CloudOperationRecoveryIdentity::expected(CloudJob::findOrFail(3))
|
|
);
|
|
unset($invalidCreate['action']);
|
|
$expectCloudError(
|
|
static fn () => CloudOperationCompletionService::accept(
|
|
$invalidCreate
|
|
),
|
|
400
|
|
);
|
|
$invalidStorage = $receipt(
|
|
'storage-invalid-receipt-event',
|
|
8,
|
|
$storageOperation,
|
|
$storageIdentity
|
|
);
|
|
unset($invalidStorage['action']);
|
|
$expectCloudError(
|
|
static fn () => CloudOperationCompletionService::accept(
|
|
$invalidStorage
|
|
),
|
|
400
|
|
);
|
|
$invalidCreateJob = CloudJob::findOrFail(3);
|
|
$invalidStorageJob = CloudJob::findOrFail(2);
|
|
$assert((string) $invalidCreateJob->phase === 'awaiting_receipt'
|
|
&& $invalidCreateJob->outcome === null
|
|
&& (bool) $invalidCreateJob->finalization_incident
|
|
&& (string) $invalidCreateJob->finalization_error_code
|
|
=== 'recovery_receipt_invalid'
|
|
&& (string) $invalidStorageJob->phase === 'dispatching'
|
|
&& $invalidStorageJob->outcome === null
|
|
&& (bool) $invalidStorageJob->finalization_incident
|
|
&& (string) $invalidStorageJob->finalization_error_code
|
|
=== 'recovery_receipt_invalid'
|
|
&& (string) Capsule::table('cloud_host_storage_configurations')
|
|
->where('host_id', 8)->value('state') === 'applying',
|
|
'An attributable invalid receipt bypassed create/storage finalizers.');
|
|
|
|
$hashIdentity = CloudOperationRecoveryIdentity::expected(
|
|
CloudJob::findOrFail(4)
|
|
);
|
|
$hashReceipt = $receipt(
|
|
'create-hash-conflict-event',
|
|
7,
|
|
$hashCreateOperation,
|
|
$hashIdentity
|
|
);
|
|
Capsule::table('cloud_agent_completion_events')->insert([
|
|
'event_id' => $hashReceipt['id'],
|
|
'host_id' => 7,
|
|
'operation_id' => $hashCreateOperation,
|
|
'payload_sha256' => str_repeat('0', 64),
|
|
'outcome' => 'failed',
|
|
'effect_may_have_applied' => 1,
|
|
'safe_to_resubmit' => 0,
|
|
'error_code' => 'old_conflicting_receipt',
|
|
'result_json' => null,
|
|
'disposition' => 'rejected',
|
|
'received_at' => $now - 5,
|
|
'applied_at' => null,
|
|
]);
|
|
$expectCloudError(
|
|
static fn () => CloudOperationCompletionService::accept($hashReceipt),
|
|
409
|
|
);
|
|
$hashJob = CloudJob::findOrFail(4);
|
|
$assert((string) $hashJob->phase === 'awaiting_receipt'
|
|
&& $hashJob->outcome === null
|
|
&& (bool) $hashJob->finalization_incident
|
|
&& (string) $hashJob->finalization_error_code
|
|
=== 'recovery_operation_identity_conflict',
|
|
'A payload-hash conflict was invisible or bypassed create finalization.');
|
|
|
|
$secretReceipt = $receipt(
|
|
'create-secret-result-event',
|
|
7,
|
|
$secretCreateOperation,
|
|
CloudOperationRecoveryIdentity::expected(
|
|
CloudJob::findOrFail(5),
|
|
true
|
|
)
|
|
);
|
|
$secretReceipt['result'] = [
|
|
'vm' => ['ssh_private_key' => 'must-never-be-persisted'],
|
|
];
|
|
$expectCloudError(
|
|
static fn () => CloudOperationCompletionService::accept(
|
|
$secretReceipt
|
|
),
|
|
400
|
|
);
|
|
$secretJob = CloudJob::findOrFail(5);
|
|
$assert(!Capsule::table('cloud_agent_completion_events')
|
|
->where('event_id', $secretReceipt['id'])->exists()
|
|
&& (string) $secretJob->phase === 'awaiting_receipt'
|
|
&& $secretJob->outcome === null
|
|
&& (bool) $secretJob->finalization_incident
|
|
&& (string) $secretJob->finalization_error_code
|
|
=== 'recovery_receipt_invalid',
|
|
'A create completion secret was persisted or bypassed its finalizer.');
|
|
|
|
$unknownSecretCases = [
|
|
'unknown-generated-password' => [
|
|
'generatedPassword' => 'must-never-persist',
|
|
],
|
|
'unknown-root-password' => [
|
|
'nested' => ['root-password' => 'must-never-persist'],
|
|
],
|
|
'unknown-access-token' => [
|
|
'access_token' => 'must-never-persist',
|
|
],
|
|
'unknown-passphrase' => [
|
|
'diskPassphrase' => 'must-never-persist',
|
|
],
|
|
'unknown-plain-token' => [
|
|
'token' => 'must-never-persist',
|
|
],
|
|
];
|
|
$unknownIndex = 0;
|
|
foreach ($unknownSecretCases as $eventId => $secretResult) {
|
|
$unknownIndex++;
|
|
$unknownOperation = substr(hash(
|
|
'sha256',
|
|
'unknown-secret-operation:' . $unknownIndex
|
|
), 0, 32);
|
|
$unknownReceipt = $receipt(
|
|
$eventId . '-event',
|
|
77,
|
|
$unknownOperation,
|
|
$createIdentity
|
|
);
|
|
$unknownReceipt['result'] = $secretResult;
|
|
$expectCloudError(
|
|
static fn () => CloudOperationCompletionService::accept(
|
|
$unknownReceipt
|
|
),
|
|
400
|
|
);
|
|
$assert(!Capsule::table('cloud_agent_completion_events')
|
|
->where('event_id', $unknownReceipt['id'])->exists(),
|
|
'An unknown-operation secret completion was persisted.');
|
|
}
|
|
|
|
echo "Agent mutation identity binding tests passed.\n";
|
|
}
|