|
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
namespace App\Services {
|
|
|
|
|
final class Config
|
|
|
|
|
{
|
|
|
|
|
public static function get($name, $default = null) { return $default; }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
namespace App\Services\Cloud {
|
|
|
|
|
/** Test-only keyring; never used by the application. */
|
|
|
|
|
final class CloudSecrets
|
|
|
|
|
{
|
|
|
|
|
private static array $values = [];
|
|
|
|
|
public static function encrypt(array $value): string
|
|
|
|
|
{
|
|
|
|
|
$key = bin2hex(random_bytes(16));
|
|
|
|
|
self::$values[$key] = $value;
|
|
|
|
|
return $key;
|
|
|
|
|
}
|
|
|
|
|
public static function decrypt(string $key): array { return self::$values[$key]; }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Simulate a healthy power API and unavailable guest/private readback. */
|
|
|
|
|
final class CloudEngineClient
|
|
|
|
|
{
|
|
|
|
|
public array $calls = [];
|
|
|
|
|
public array $response = [];
|
|
|
|
|
public $afterCommand = null;
|
|
|
|
|
public function host(int $host, string $method, string $resource,
|
|
|
|
|
array $actor, ?array $body, string $operation): array
|
|
|
|
|
{
|
|
|
|
|
$this->calls[] = [$host, $method, $resource, $body];
|
|
|
|
|
if (!str_ends_with($resource, '/actions')
|
|
|
|
|
&& !str_ends_with($resource, '/settings') && $method !== 'DELETE') {
|
|
|
|
|
throw new CloudError('Private network/QGA unavailable', 503);
|
|
|
|
|
}
|
|
|
|
|
if ($this->afterCommand !== null) { ($this->afterCommand)(); }
|
|
|
|
|
return $this->response + ['operation_id' => $operation];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudInstance;
|
|
|
|
|
use App\Models\CloudInstanceRuntime;
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
use App\Models\CloudVpcMember;
|
|
|
|
|
use App\Models\CloudVpcOperation;
|
|
|
|
|
use App\Services\Cloud\CloudEngineClient;
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
use App\Services\Cloud\CloudInstanceRuntimeService;
|
|
|
|
|
use App\Services\Cloud\CloudJobService;
|
|
|
|
|
use App\Services\Cloud\CloudOperationErrorCatalog;
|
|
|
|
|
use App\Services\Cloud\CloudSecrets;
|
|
|
|
|
use App\Services\Cloud\CloudVpcService;
|
|
|
|
|
use App\Services\Cloud\CloudVpcPowerWorkflow;
|
|
|
|
|
use App\Services\Cloud\CloudWorkerService;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) { throw new RuntimeException($message); }
|
|
|
|
|
};
|
|
|
|
|
$capsule = new Capsule();
|
|
|
|
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
|
|
|
|
|
$capsule->setAsGlobal();
|
|
|
|
|
$capsule->bootEloquent();
|
|
|
|
|
$schema = $capsule->schema();
|
|
|
|
|
$schema->create('user', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->integer('enable')->default(1);
|
|
|
|
|
$t->boolean('is_admin')->default(false);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instances', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->integer('host_id'); $t->integer('user_id');
|
|
|
|
|
$t->integer('order_id')->nullable(); $t->string('uuid'); $t->string('name');
|
|
|
|
|
$t->string('status')->default('active');
|
|
|
|
|
$t->string('service_status')->default('exempt');
|
|
|
|
|
$t->string('network_status')->default('active');
|
|
|
|
|
$t->integer('expires_at')->nullable(); $t->text('spec_json')->default('{}');
|
|
|
|
|
$t->integer('updated_at')->default(0);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_jobs', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('operation_id')->unique();
|
|
|
|
|
$t->integer('order_id')->nullable(); $t->integer('instance_id')->nullable();
|
|
|
|
|
$t->integer('snapshot_id')->nullable(); $t->integer('user_id'); $t->integer('host_id');
|
|
|
|
|
$t->string('kind'); $t->string('status'); $t->text('payload_cipher');
|
|
|
|
|
$t->integer('attempts')->default(0); $t->integer('dispatched_at')->nullable();
|
|
|
|
|
$t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0);
|
|
|
|
|
$t->integer('available_at'); $t->integer('created_at'); $t->integer('updated_at');
|
|
|
|
|
$t->string('error_code')->nullable(); $t->text('result_json')->nullable();
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instance_runtime', static function (Blueprint $t): void {
|
|
|
|
|
$t->integer('instance_id')->primary(); $t->integer('host_id'); $t->string('uuid');
|
|
|
|
|
$t->string('power_state'); $t->string('sync_state'); $t->string('last_error_code');
|
|
|
|
|
foreach (['last_success_at', 'last_attempt_at', 'revision', 'created_at', 'updated_at'] as $key) {
|
|
|
|
|
$t->integer($key);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instance_snapshots', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->integer('instance_id'); $t->string('state');
|
|
|
|
|
$t->string('last_operation_id')->nullable(); $t->integer('deleted_at')->nullable();
|
|
|
|
|
$t->integer('updated_at')->default(0);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_vpcs', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('vpc_key'); $t->integer('desired_revision');
|
|
|
|
|
$t->string('desired_digest'); $t->string('state'); $t->string('sync_state');
|
|
|
|
|
$t->string('last_operation_id')->nullable();
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_vpc_operations', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->integer('vpc_id'); $t->integer('user_id');
|
|
|
|
|
$t->string('operation_id'); $t->string('kind'); $t->string('status');
|
|
|
|
|
$t->integer('desired_revision'); $t->text('power_workflow_json')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('power_instance_id')->nullable()->index();
|
|
|
|
|
$t->string('lease_token')->nullable(); $t->integer('lease_until')->default(0);
|
|
|
|
|
$t->string('error_code')->nullable(); $t->integer('updated_at')->default(0);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_vpc_host_deployments', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->integer('vpc_id'); $t->integer('host_id'); $t->string('state');
|
|
|
|
|
});
|
|
|
|
|
foreach (['cloud_vpc_members', 'cloud_vpc_private_ips'] as $table) {
|
|
|
|
|
$schema->create($table, static function (Blueprint $t) use ($table): void {
|
|
|
|
|
$t->increments('id'); $t->integer('vpc_id'); $t->integer('subnet_id');
|
|
|
|
|
$t->integer('instance_id'); $t->string('private_ip'); $t->string('state');
|
|
|
|
|
$t->integer('desired_revision'); $t->string('desired_digest');
|
|
|
|
|
$t->integer('observed_revision')->default(0); $t->string('observed_digest')->nullable();
|
|
|
|
|
$t->string('last_error_code')->nullable(); $t->integer('updated_at')->default(0);
|
|
|
|
|
if ($table === 'cloud_vpc_members') {
|
|
|
|
|
$t->string('member_key'); $t->integer('host_id'); $t->integer('private_ip_id');
|
|
|
|
|
$t->string('last_operation_id')->nullable();
|
|
|
|
|
} else {
|
|
|
|
|
$t->integer('member_id');
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
$uuid = '11111111-1111-4111-8111-111111111110';
|
|
|
|
|
Capsule::table('user')->insert(['id' => 1]);
|
|
|
|
|
Capsule::table('cloud_instances')->insert([
|
|
|
|
|
'id' => 10, 'host_id' => 7, 'user_id' => 1, 'uuid' => $uuid, 'name' => 'Power VM',
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_vpcs')->insert([
|
|
|
|
|
'id' => 1, 'vpc_key' => str_repeat('a', 32), 'desired_revision' => 1,
|
|
|
|
|
'desired_digest' => str_repeat('d', 64), 'state' => 'active', 'sync_state' => 'failed',
|
|
|
|
|
]);
|
|
|
|
|
$member = ['id' => 1, 'vpc_id' => 1, 'subnet_id' => 1, 'instance_id' => 10,
|
|
|
|
|
'private_ip' => '10.0.0.1', 'state' => 'uncertain',
|
|
|
|
|
'desired_revision' => 1, 'desired_digest' => str_repeat('d', 64)];
|
|
|
|
|
Capsule::table('cloud_vpc_members')->insert($member + [
|
|
|
|
|
'member_key' => str_repeat('b', 32), 'host_id' => 7, 'private_ip_id' => 1,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_vpc_private_ips')->insert($member + ['member_id' => 1]);
|
|
|
|
|
$actor = ['user_id' => 1, 'is_admin' => false, 'is_super_admin' => false,
|
|
|
|
|
'can_manage' => false, 'session_id' => ''];
|
|
|
|
|
$engine = new CloudEngineClient();
|
|
|
|
|
$worker = new CloudWorkerService($engine);
|
|
|
|
|
$enqueue = static fn (string $action): CloudJob => CloudJobService::enqueue(
|
|
|
|
|
CloudInstance::find(10), $actor, 'action', 'POST', '/actions', ['action' => $action]
|
|
|
|
|
);
|
|
|
|
|
$expectation = CloudVpcService::instancePrivateExpectation(10);
|
|
|
|
|
$assert($expectation['required'] === true && $expectation['reason_code'] === null,
|
|
|
|
|
'Fixture must have a valid attached private target despite failed readiness.');
|
|
|
|
|
|
|
|
|
|
// Each command can be followed immediately by another command even when
|
|
|
|
|
// the guest is broken and every private-readback endpoint would fail.
|
|
|
|
|
foreach (['shutdown', 'poweroff', 'start', 'reboot', 'reset'] as $action) {
|
|
|
|
|
$instance = CloudInstance::find(10);
|
|
|
|
|
$instance->network_status = $action === 'start' ? 'blocked_traffic' : 'active';
|
|
|
|
|
$instance->save();
|
|
|
|
|
$beforeState = $action === 'start' ? 'stopped' : 'running';
|
|
|
|
|
CloudInstanceRuntimeService::markObserved(10, 7, $uuid, $beforeState);
|
|
|
|
|
$job = $enqueue($action);
|
|
|
|
|
$engine->response = ['status' => 'succeeded', 'result' => [
|
|
|
|
|
'uuid' => $uuid, 'action' => $action, 'command_accepted' => true,
|
|
|
|
|
]];
|
|
|
|
|
$count = count($engine->calls);
|
|
|
|
|
$assert($worker->tick(), 'Worker did not claim the power request.');
|
|
|
|
|
$done = CloudJob::find($job->id);
|
|
|
|
|
$receipt = json_decode($done->result_json, true);
|
|
|
|
|
$assert($done->status === 'succeeded' && $done->lease_token === null
|
|
|
|
|
&& (int) $done->lease_until === 0 && count($engine->calls) === $count + 1,
|
|
|
|
|
'Confirmed ' . $action . ' waited for or dispatched private reconciliation.');
|
|
|
|
|
$assert(($receipt['command_accepted'] ?? null) === true
|
|
|
|
|
&& ($receipt['action'] ?? null) === $action && !isset($receipt['state']),
|
|
|
|
|
'Power receipt was discarded or fabricated an observed power state.');
|
|
|
|
|
$assert(CloudInstanceRuntime::find(10)->power_state
|
|
|
|
|
=== ($action === 'shutdown' ? 'shutdown' : $beforeState),
|
|
|
|
|
'Receipt-only command fabricated a running/stopped observation.');
|
|
|
|
|
$assert(CloudInstance::find(10)->network_status === $instance->network_status
|
|
|
|
|
&& Capsule::table('cloud_vpc_members')->value('state') === 'uncertain'
|
|
|
|
|
&& Capsule::table('cloud_vpcs')->value('sync_state') === 'failed',
|
|
|
|
|
'Ordinary power command changed independent network policy/readiness.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Exercise the real admission path: a VPC graceful-shutdown timeout must
|
|
|
|
|
// permit force-off and durably revoke the old automatic restart intent.
|
|
|
|
|
$vpcOperation = new CloudVpcOperation();
|
|
|
|
|
$vpcOperation->operation_id = str_repeat('c', 32);
|
|
|
|
|
$vpcOperation->vpc_id = 1;
|
|
|
|
|
$vpcOperation->user_id = 1;
|
|
|
|
|
$vpcOperation->kind = 'member_attach';
|
|
|
|
|
$vpcOperation->status = 'paused';
|
|
|
|
|
$vpcOperation->desired_revision = 1;
|
|
|
|
|
$vpcOperation->save();
|
|
|
|
|
CloudVpcPowerWorkflow::initialize($vpcOperation,
|
|
|
|
|
CloudInstance::find(10), CloudVpcMember::find(1));
|
|
|
|
|
$powerWorkflow = CloudVpcPowerWorkflow::state($vpcOperation);
|
|
|
|
|
$powerWorkflow['phase'] = 'paused';
|
|
|
|
|
$powerWorkflow['paused_from'] = 'waiting_stopped';
|
|
|
|
|
$powerWorkflow['original_running'] = true;
|
|
|
|
|
$powerWorkflow['error_code'] = 'vpc_power_shutdown_timeout';
|
|
|
|
|
$vpcOperation->power_workflow_json = json_encode($powerWorkflow);
|
|
|
|
|
$vpcOperation->save();
|
|
|
|
|
Capsule::table('cloud_vpcs')->where('id', 1)->update([
|
|
|
|
|
'last_operation_id' => $vpcOperation->operation_id,
|
|
|
|
|
]);
|
|
|
|
|
try {
|
|
|
|
|
CloudJobService::enqueue(CloudInstance::find(10), $actor,
|
|
|
|
|
'instance_settings', 'POST', '/settings', [
|
|
|
|
|
'hostname' => 'during-vpc', 'confirm_restart' => true,
|
|
|
|
|
]);
|
|
|
|
|
throw new RuntimeException('Settings mutation bypassed unfinished VPC maintenance.');
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === 409, 'Unexpected VPC admission failure.');
|
|
|
|
|
}
|
|
|
|
|
$manualPower = $enqueue('poweroff');
|
|
|
|
|
$assert($manualPower->status === 'queued'
|
|
|
|
|
&& CloudVpcPowerWorkflow::state($vpcOperation->fresh())['resume_cancelled'] === true,
|
|
|
|
|
'VPC timeout blocked force-off or did not revoke automatic restart.');
|
|
|
|
|
$engine->response = ['status' => 'succeeded', 'result' => [
|
|
|
|
|
'uuid' => $uuid, 'action' => 'poweroff', 'command_accepted' => true,
|
|
|
|
|
]];
|
|
|
|
|
$worker->tick();
|
|
|
|
|
$assert($manualPower->fresh()->status === 'succeeded',
|
|
|
|
|
'Force-off after VPC timeout acquired a private-readiness dependency.');
|
|
|
|
|
// End this isolated fixture; production completion requires host proof.
|
|
|
|
|
$powerWorkflow = CloudVpcPowerWorkflow::state($vpcOperation->fresh());
|
|
|
|
|
$powerWorkflow['phase'] = 'completed';
|
|
|
|
|
$vpcOperation->power_workflow_json = json_encode($powerWorkflow);
|
|
|
|
|
$vpcOperation->save();
|
|
|
|
|
|
|
|
|
|
// A changing private target does not invalidate an already accepted power
|
|
|
|
|
// command in finish()'s transaction either.
|
|
|
|
|
$job = $enqueue('shutdown');
|
|
|
|
|
$engine->response = ['status' => 'succeeded', 'result' => [
|
|
|
|
|
'uuid' => $uuid, 'action' => 'shutdown', 'command_accepted' => true,
|
|
|
|
|
]];
|
|
|
|
|
$engine->afterCommand = static function (): void {
|
|
|
|
|
foreach (['cloud_vpcs', 'cloud_vpc_members', 'cloud_vpc_private_ips'] as $table) {
|
|
|
|
|
Capsule::table($table)->update(['desired_revision' => 2]);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
$worker->tick();
|
|
|
|
|
$assert(CloudJob::find($job->id)->status === 'succeeded',
|
|
|
|
|
'Private revision change retroactively blocked confirmed power receipt.');
|
|
|
|
|
$engine->afterCommand = null;
|
|
|
|
|
$next = $enqueue('poweroff');
|
|
|
|
|
$assert($next->status === 'queued', 'Accepted shutdown still blocks force poweroff.');
|
|
|
|
|
// Isolate the following rejection cases; no production cancellation code is used.
|
|
|
|
|
$next->status = 'cancelled'; $next->save();
|
|
|
|
|
|
|
|
|
|
// An engine-side preflight/admission failure is terminal even though PHP
|
|
|
|
|
// sent HTTP and the VM has an unavailable private network. No guest write
|
|
|
|
|
// happened, so it must not dispatch VPC reconciliation or block force-off.
|
|
|
|
|
foreach (['action', 'instance_settings'] as $kind) {
|
|
|
|
|
$job = $kind === 'action' ? $enqueue('reboot')
|
|
|
|
|
: CloudJobService::enqueue(CloudInstance::find(10), $actor,
|
|
|
|
|
'instance_settings', 'POST', '/settings', [
|
|
|
|
|
'hostname' => 'preflight-test', 'confirm_restart' => true,
|
|
|
|
|
]);
|
|
|
|
|
$engine->response = ['status' => 'failed',
|
|
|
|
|
'execution_outcome' => 'not_dispatched', 'mutation_started' => false,
|
|
|
|
|
'safe_to_refund' => false, 'error_code' => 'execution_result_uncertain'];
|
|
|
|
|
$count = count($engine->calls);
|
|
|
|
|
$worker->tick();
|
|
|
|
|
$failed = CloudJob::find($job->id);
|
|
|
|
|
$assert($failed->status === 'failed' && $failed->lease_token === null
|
|
|
|
|
&& (int) $failed->lease_until === 0
|
|
|
|
|
&& count($engine->calls) === $count + 1,
|
|
|
|
|
'Proven preflight failure acquired a private-state dependency for ' . $kind . '.');
|
|
|
|
|
$followup = $enqueue('poweroff');
|
|
|
|
|
$engine->response = ['status' => 'succeeded', 'result' => [
|
|
|
|
|
'uuid' => $uuid, 'action' => 'poweroff', 'command_accepted' => true,
|
|
|
|
|
]];
|
|
|
|
|
$worker->tick();
|
|
|
|
|
$assert(CloudJob::find($followup->id)->status === 'succeeded',
|
|
|
|
|
'Preflight failure blocked the next force-off request for ' . $kind . '.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// User deletion has a separate instance admission latch. Undo it only
|
|
|
|
|
// for this operation's explicit no-write receipt, never a legacy or
|
|
|
|
|
// contradictory outcome; preserve the snapshot catalog in all failures.
|
|
|
|
|
foreach ([[], ['execution_outcome' => null], ['mutation_started' => true],
|
|
|
|
|
['operation_id' => str_repeat('f', 32)], ['executor_active' => true],
|
|
|
|
|
['vm_delete_committed' => true], ['result' => ['cleanup_pending' => true]]]
|
|
|
|
|
as $index => $change) {
|
|
|
|
|
$id = 11 + $index;
|
|
|
|
|
$targetUuid = sprintf('22222222-2222-4222-8222-%012d', $id);
|
|
|
|
|
Capsule::table('cloud_instances')->insert(['id' => $id, 'host_id' => 7,
|
|
|
|
|
'user_id' => 1, 'uuid' => $targetUuid, 'name' => 'Delete preflight']);
|
|
|
|
|
Capsule::table('cloud_instance_snapshots')->insert([
|
|
|
|
|
'instance_id' => $id, 'state' => 'ready']);
|
|
|
|
|
$delete = CloudJobService::enqueue(CloudInstance::find($id), $actor,
|
|
|
|
|
'delete', 'DELETE', '?delete_disks=true');
|
|
|
|
|
$assert(CloudInstance::find($id)->status === 'deleting',
|
|
|
|
|
'Delete request did not establish its admission latch.');
|
|
|
|
|
$engine->response = array_replace(['status' => 'failed',
|
|
|
|
|
'execution_outcome' => 'not_dispatched', 'mutation_started' => false,
|
|
|
|
|
'safe_to_refund' => false, 'error_code' => 'execution_result_uncertain'], $change);
|
|
|
|
|
$worker->tick();
|
|
|
|
|
$assert(CloudJob::find($delete->id)->status === 'failed'
|
|
|
|
|
&& CloudInstance::find($id)->status === ($index === 0 ? 'active' : 'delete_failed')
|
|
|
|
|
&& Capsule::table('cloud_instance_snapshots')->where('instance_id', $id)
|
|
|
|
|
->value('state') === 'ready',
|
|
|
|
|
'User-delete latch ignored missing/conflicting execution proof.');
|
|
|
|
|
if ($index === 0) {
|
|
|
|
|
$next = CloudJobService::enqueue(CloudInstance::find($id), $actor,
|
|
|
|
|
'action', 'POST', '/actions', ['action' => 'poweroff']);
|
|
|
|
|
$next->status = 'cancelled'; $next->save();
|
|
|
|
|
} else {
|
|
|
|
|
try {
|
|
|
|
|
CloudJobService::enqueue(CloudInstance::find($id), $actor,
|
|
|
|
|
'action', 'POST', '/actions', ['action' => 'poweroff']);
|
|
|
|
|
throw new RuntimeException('Unproven failed delete admitted another write.');
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === 409, 'Unproven delete had unexpected admission.');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Expiry cleanup owns a separate service-policy latch. A no-write failure
|
|
|
|
|
// must not restore an expired instance to an active service.
|
|
|
|
|
Capsule::table('cloud_instances')->insert(['id' => 30, 'host_id' => 7,
|
|
|
|
|
'user_id' => 1, 'uuid' => '22222222-2222-4222-8222-000000000030',
|
|
|
|
|
'name' => 'Expiry preflight', 'status' => 'deleting',
|
|
|
|
|
'service_status' => 'purge_deleting']);
|
|
|
|
|
$expiry = CloudJobService::enqueueExpiryDelete(CloudInstance::find(30));
|
|
|
|
|
$engine->response = ['status' => 'failed', 'execution_outcome' => 'not_dispatched',
|
|
|
|
|
'mutation_started' => false, 'safe_to_refund' => false];
|
|
|
|
|
$worker->tick();
|
|
|
|
|
$assert(CloudJob::find($expiry->id)->status === 'failed'
|
|
|
|
|
&& CloudInstance::find(30)->status === 'delete_failed'
|
|
|
|
|
&& CloudInstance::find(30)->service_status === 'purge_failed',
|
|
|
|
|
'Preflight failure reopened an expired service instead of preserving its policy latch.');
|
|
|
|
|
|
|
|
|
|
$gate = new ReflectionMethod(CloudWorkerService::class, 'applyPrivateLifecycleGate');
|
|
|
|
|
$confirmed = new ReflectionMethod(CloudWorkerService::class, 'confirmedPowerReceipt');
|
|
|
|
|
$job = $enqueue('shutdown');
|
|
|
|
|
$payload = CloudSecrets::decrypt($job->payload_cipher);
|
|
|
|
|
$success = ['operation_id' => $job->operation_id, 'status' => 'succeeded', 'result' => [
|
|
|
|
|
'uuid' => $uuid, 'action' => 'shutdown', 'command_accepted' => true,
|
|
|
|
|
]];
|
|
|
|
|
$assert($confirmed->invoke(null, $job, $success, $payload), 'Valid receipt not recognized.');
|
|
|
|
|
$invalid = [
|
|
|
|
|
array_replace_recursive($success, ['result' => ['uuid' => '22222222-2222-4222-8222-222222222222']]),
|
|
|
|
|
array_replace_recursive($success, ['result' => ['action' => 'poweroff']]),
|
|
|
|
|
array_replace_recursive($success, ['result' => ['command_accepted' => 'true']]),
|
|
|
|
|
array_replace_recursive($success, ['result' => ['command_accepted' => false]]),
|
|
|
|
|
array_replace($success, ['operation_id' => str_repeat('f', 32)]),
|
|
|
|
|
array_replace($success, ['status' => 'uncertain']),
|
|
|
|
|
array_replace($success, ['status' => 'running']),
|
|
|
|
|
array_replace($success, ['status' => 'failed']),
|
|
|
|
|
];
|
|
|
|
|
foreach ($invalid as $operation) {
|
|
|
|
|
$assert(!$confirmed->invoke(null, $job, $operation, $payload),
|
|
|
|
|
'Mismatched, unconfirmed or nonterminal receipt bypassed the private gate.');
|
|
|
|
|
}
|
|
|
|
|
foreach ([['method' => 'GET'], ['resource' => 'vms/' . $uuid . '/reinstall'],
|
|
|
|
|
['body' => ['action' => 'delete']]] as $change) {
|
|
|
|
|
$assert(!$confirmed->invoke(null, $job, $success, array_replace($payload, $change)),
|
|
|
|
|
'Receipt bypass was not bound to the immutable power request.');
|
|
|
|
|
}
|
|
|
|
|
$safe = CloudOperationErrorCatalog::workerResult([
|
|
|
|
|
'command_accepted' => 'true', 'action' => 'arbitrary-host-message',
|
|
|
|
|
]);
|
|
|
|
|
$assert($safe === [], 'Unvalidated power receipt fields reached stored/user results.');
|
|
|
|
|
|
|
|
|
|
// A lost power response is still uncertain, and the next write is rejected.
|
|
|
|
|
$engine->response = ['status' => 'uncertain', 'mutation_started' => true,
|
|
|
|
|
'execution_outcome' => 'unknown',
|
|
|
|
|
'error_code' => 'libvirt_execution_uncertain'];
|
|
|
|
|
$worker->tick();
|
|
|
|
|
$assert(CloudJob::find($job->id)->status === 'uncertain',
|
|
|
|
|
'A truly uncertain write was upgraded to success.');
|
|
|
|
|
try {
|
|
|
|
|
$enqueue('poweroff');
|
|
|
|
|
throw new RuntimeException('Unknown write allowed a new power mutation.');
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === 409, 'Unknown write had an unexpected admission result.');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Unavailable private proof still gates destructive and policy mutations.
|
|
|
|
|
// Invoke the production gate directly to avoid manufacturing their finance
|
|
|
|
|
// ledgers; these are not ordinary action jobs even if a result carries a receipt.
|
|
|
|
|
foreach (['delete', 'expiry_delete', 'reinstall', 'network_change',
|
|
|
|
|
'network_suspend', 'network_resume', 'service_suspend', 'service_resume'] as $kind) {
|
|
|
|
|
$other = clone $job;
|
|
|
|
|
$other->kind = $kind;
|
|
|
|
|
$count = count($engine->calls);
|
|
|
|
|
$result = $gate->invoke($worker, $other, $success, $payload, $actor);
|
|
|
|
|
$assert($result['status'] === 'uncertain' && count($engine->calls) > $count,
|
|
|
|
|
'Power receipt weakened private readback for ' . $kind . '.');
|
|
|
|
|
}
|
|
|
|
|
echo "Power command receipt workflow tests passed.\n";
|
|
|
|
|
}
|