|
|
|
|
<?php
|
|
|
|
|
declare(strict_types=1);
|
|
|
|
|
|
|
|
|
|
namespace App\Services\Cloud {
|
|
|
|
|
/** In-memory envelope used only by this isolated state-machine test. */
|
|
|
|
|
final class CloudSecrets
|
|
|
|
|
{
|
|
|
|
|
private static array $values = [];
|
|
|
|
|
|
|
|
|
|
public static function encrypt(array $value, string $purpose): string
|
|
|
|
|
{
|
|
|
|
|
$key = bin2hex(random_bytes(16));
|
|
|
|
|
self::$values[$key] = $value;
|
|
|
|
|
return 'storage-test:' . $key;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static function decrypt(string $value, string $purpose): array
|
|
|
|
|
{
|
|
|
|
|
return self::$values[substr($value, strlen('storage-test:'))];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudHostStorageConfiguration;
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
use App\Services\Cloud\CloudHostStorageConfigurationService;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) {
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
$expect = static function (callable $callable, int $status,
|
|
|
|
|
string $message) use ($assert): void {
|
|
|
|
|
try {
|
|
|
|
|
$callable();
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === $status,
|
|
|
|
|
$message . ' Unexpected status: ' . $error->httpStatus);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
};
|
|
|
|
|
$terminalize = static function (CloudJob $job, string $outcome,
|
|
|
|
|
bool $effectMayHaveApplied = false): void {
|
|
|
|
|
$now = time();
|
|
|
|
|
$job->phase = 'terminal';
|
|
|
|
|
$job->outcome = $outcome;
|
|
|
|
|
$job->status = $outcome;
|
|
|
|
|
$job->attempts = 1;
|
|
|
|
|
$job->dispatch_attempts = 1;
|
|
|
|
|
$job->dispatch_started_at = $now - 2;
|
|
|
|
|
$job->dispatched_at = $now - 2;
|
|
|
|
|
$job->agent_accepted_at = $now - 1;
|
|
|
|
|
$job->receipt_deadline_at = $now + 900;
|
|
|
|
|
$job->terminal_at = $now;
|
|
|
|
|
$job->effect_may_have_applied = $effectMayHaveApplied ? 1 : 0;
|
|
|
|
|
$job->safe_to_resubmit = 0;
|
|
|
|
|
$job->effect_resolution = $outcome === 'succeeded'
|
|
|
|
|
? 'applied' : ($effectMayHaveApplied ? 'unknown' : 'not_applied');
|
|
|
|
|
$job->lease_token = null;
|
|
|
|
|
$job->lease_until = 0;
|
|
|
|
|
$job->available_at = 0;
|
|
|
|
|
$job->updated_at = $now;
|
|
|
|
|
$job->save();
|
|
|
|
|
};
|
|
|
|
|
$makeLegacyActive = static function (CloudJob $job,
|
|
|
|
|
string $phase = 'awaiting_receipt'): void {
|
|
|
|
|
$now = time();
|
|
|
|
|
$job->phase = $phase;
|
|
|
|
|
$job->status = $phase === 'accepted' ? 'queued' : 'running';
|
|
|
|
|
$job->attempts = $phase === 'accepted' ? 0 : 1;
|
|
|
|
|
$job->dispatch_attempts = $phase === 'accepted' ? 0 : 1;
|
|
|
|
|
$job->created_at = $now - 181;
|
|
|
|
|
$job->dispatch_deadline_at = $now + 119;
|
|
|
|
|
$job->dispatch_started_at = $phase === 'accepted' ? null : $now - 181;
|
|
|
|
|
$job->dispatched_at = $phase === 'accepted' ? null : $now - 181;
|
|
|
|
|
$job->agent_accepted_at = $phase === 'accepted' ? null : $now - 180;
|
|
|
|
|
$job->receipt_deadline_at = $phase === 'accepted'
|
|
|
|
|
? null : $now + 779;
|
|
|
|
|
$job->lease_token = null;
|
|
|
|
|
$job->lease_until = 0;
|
|
|
|
|
$job->updated_at = $now;
|
|
|
|
|
$job->save();
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$capsule = new Capsule();
|
|
|
|
|
$capsule->addConnection([
|
|
|
|
|
'driver' => 'sqlite', 'database' => ':memory:',
|
|
|
|
|
], 'default');
|
|
|
|
|
$capsule->setAsGlobal();
|
|
|
|
|
$capsule->bootEloquent();
|
|
|
|
|
$schema = $capsule->schema('default');
|
|
|
|
|
|
|
|
|
|
$schema->create('user', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_hosts', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('name');
|
|
|
|
|
$table->string('region');
|
|
|
|
|
$table->string('country_code', 2);
|
|
|
|
|
$table->boolean('enabled');
|
|
|
|
|
$table->string('default_network');
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_host_agents', static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id')->primary();
|
|
|
|
|
$table->text('sealed_api_secret');
|
|
|
|
|
$table->boolean('enabled')->default(false);
|
|
|
|
|
$table->integer('revision')->default(1);
|
|
|
|
|
$table->integer('tested_revision')->default(0);
|
|
|
|
|
$table->string('last_test_status')->default('never');
|
|
|
|
|
$table->string('last_error_code')->default('');
|
|
|
|
|
$table->string('last_agent_version')->default('');
|
|
|
|
|
$table->integer('last_tested_at')->default(0);
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_host_storage_configurations',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->integer('host_id')->primary();
|
|
|
|
|
$table->integer('revision')->default(0);
|
|
|
|
|
foreach (['managed', 'template', 'seed', 'user_iso', 'rescue'] as $role) {
|
|
|
|
|
$table->text($role . '_directory');
|
|
|
|
|
}
|
|
|
|
|
$table->string('state')->default('applying');
|
|
|
|
|
$table->string('operation_id')->nullable()->unique();
|
|
|
|
|
$table->integer('job_id')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_jobs', static function (Blueprint $table): void {
|
|
|
|
|
$table->increments('id');
|
|
|
|
|
$table->string('operation_id')->unique();
|
|
|
|
|
$table->string('request_idempotency_key');
|
|
|
|
|
$table->string('request_fingerprint');
|
|
|
|
|
$table->integer('parent_job_id')->nullable();
|
|
|
|
|
$table->integer('order_id')->nullable();
|
|
|
|
|
$table->integer('invoice_id')->nullable();
|
|
|
|
|
$table->integer('instance_id')->nullable();
|
|
|
|
|
$table->integer('snapshot_id')->nullable();
|
|
|
|
|
$table->integer('user_id');
|
|
|
|
|
$table->integer('host_id')->nullable();
|
|
|
|
|
$table->string('kind');
|
|
|
|
|
$table->string('execution_mode');
|
|
|
|
|
$table->string('phase');
|
|
|
|
|
$table->string('outcome')->nullable();
|
|
|
|
|
$table->string('status');
|
|
|
|
|
$table->text('sealed_payload');
|
|
|
|
|
$table->integer('attempts');
|
|
|
|
|
$table->integer('dispatch_attempts');
|
|
|
|
|
$table->integer('dispatch_deadline_at');
|
|
|
|
|
$table->integer('dispatch_started_at')->nullable();
|
|
|
|
|
$table->integer('agent_accepted_at')->nullable();
|
|
|
|
|
$table->integer('receipt_deadline_at')->nullable();
|
|
|
|
|
$table->integer('terminal_at')->nullable();
|
|
|
|
|
$table->boolean('effect_may_have_applied')->nullable();
|
|
|
|
|
$table->boolean('safe_to_resubmit')->nullable();
|
|
|
|
|
$table->string('effect_resolution');
|
|
|
|
|
$table->integer('effect_resolved_at')->nullable();
|
|
|
|
|
$table->integer('effect_resolved_by')->nullable();
|
|
|
|
|
$table->text('effect_resolution_reason')->nullable();
|
|
|
|
|
$table->string('completion_event_id')->nullable();
|
|
|
|
|
$table->integer('read_at')->nullable();
|
|
|
|
|
$table->integer('dispatched_at')->nullable();
|
|
|
|
|
$table->string('lease_token')->nullable();
|
|
|
|
|
$table->integer('lease_until')->default(0);
|
|
|
|
|
$table->integer('available_at');
|
|
|
|
|
$table->string('error_code')->nullable();
|
|
|
|
|
$table->boolean('finalization_incident')->default(0);
|
|
|
|
|
$table->string('finalization_error_code')->nullable();
|
|
|
|
|
$table->text('result_json')->nullable();
|
|
|
|
|
$table->integer('created_at');
|
|
|
|
|
$table->integer('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_agent_completion_events',
|
|
|
|
|
static function (Blueprint $table): void {
|
|
|
|
|
$table->string('event_id')->primary();
|
|
|
|
|
$table->integer('host_id');
|
|
|
|
|
$table->string('operation_id');
|
|
|
|
|
$table->string('payload_sha256');
|
|
|
|
|
$table->string('outcome');
|
|
|
|
|
$table->boolean('effect_may_have_applied');
|
|
|
|
|
$table->boolean('safe_to_resubmit');
|
|
|
|
|
$table->string('error_code')->default('');
|
|
|
|
|
$table->text('result_json')->nullable();
|
|
|
|
|
$table->string('disposition')->default('received');
|
|
|
|
|
$table->integer('received_at');
|
|
|
|
|
$table->integer('applied_at')->nullable();
|
|
|
|
|
$table->unique(['host_id', 'operation_id']);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
Capsule::table('user')->insert(['id' => 1]);
|
|
|
|
|
foreach ([1, 2, 3, 4] as $hostId) {
|
|
|
|
|
Capsule::table('cloud_hosts')->insert([
|
|
|
|
|
'id' => $hostId, 'name' => 'Host ' . $hostId,
|
|
|
|
|
'region' => 'HongKong', 'country_code' => 'HK', 'enabled' => 1,
|
|
|
|
|
'default_network' => 'bridge:br0',
|
|
|
|
|
'created_at' => 1, 'updated_at' => 1,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_host_agents')->insert([
|
|
|
|
|
'host_id' => $hostId, 'sealed_api_secret' => 'test-secret',
|
|
|
|
|
'enabled' => 1, 'revision' => 1, 'tested_revision' => 1,
|
|
|
|
|
'last_test_status' => 'healthy', 'last_error_code' => '',
|
|
|
|
|
'last_agent_version' => 'test', 'last_tested_at' => 1,
|
|
|
|
|
'created_at' => 1, 'updated_at' => 1,
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$actor = [
|
|
|
|
|
'user_id' => 1, 'is_admin' => true, 'is_super_admin' => true,
|
|
|
|
|
'can_manage' => true, 'session_id' => 'storage-test',
|
|
|
|
|
];
|
|
|
|
|
$first = [
|
|
|
|
|
'expected_revision' => 0,
|
|
|
|
|
'managed_directory' => '/home/libvirt/kvm',
|
|
|
|
|
'template_directory' => '/home/libvirt/images',
|
|
|
|
|
'seed_directory' => '/home/libvirt/seed',
|
|
|
|
|
'user_iso_directory' => '/home/libvirt/iso',
|
|
|
|
|
'rescue_directory' => '/home/libvirt/rescue-iso',
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
$empty = CloudHostStorageConfigurationService::read(1);
|
|
|
|
|
$assert($empty['state'] === 'unconfigured'
|
|
|
|
|
&& $empty['revision'] === 0 && $empty['can_submit'] === true,
|
|
|
|
|
'An unconfigured host was not writable.');
|
|
|
|
|
|
|
|
|
|
$receipt1 = CloudHostStorageConfigurationService::save(1, $first, $actor);
|
|
|
|
|
$job1 = CloudJob::find((int) $receipt1['task_id']);
|
|
|
|
|
$row = CloudHostStorageConfiguration::find(1);
|
|
|
|
|
$assert($job1 !== null && $row !== null
|
|
|
|
|
&& (string) $row->operation_id === (string) $job1->operation_id
|
|
|
|
|
&& (int) $row->job_id === (int) $job1->id
|
|
|
|
|
&& (int) $job1->dispatch_deadline_at - (int) $job1->created_at === 60
|
|
|
|
|
&& (int) $row->revision === 1 && (string) $row->state === 'applying',
|
|
|
|
|
'Initial storage submission did not create one linked task.');
|
|
|
|
|
$activeView = CloudHostStorageConfigurationService::read(1);
|
|
|
|
|
$assert($activeView['task_id'] === (string) $job1->id
|
|
|
|
|
&& $activeView['status_url'] === '/api/v1/tasks/' . (int) $job1->id,
|
|
|
|
|
'An active storage projection did not expose its bounded task observer.');
|
|
|
|
|
|
|
|
|
|
$sameActive = $first;
|
|
|
|
|
$sameActive['expected_revision'] = 1;
|
|
|
|
|
$activeReceipt = CloudHostStorageConfigurationService::save(
|
|
|
|
|
1, $sameActive, $actor
|
|
|
|
|
);
|
|
|
|
|
$assert($activeReceipt['task_id'] === $receipt1['task_id']
|
|
|
|
|
&& CloudJob::count() === 1,
|
|
|
|
|
'An active same-path request created another task.');
|
|
|
|
|
$changedActive = $sameActive;
|
|
|
|
|
$changedActive['rescue_directory'] = '/home/libvirt/rescue-corrected';
|
|
|
|
|
$expect(static fn () => CloudHostStorageConfigurationService::save(
|
|
|
|
|
1, $changedActive, $actor
|
|
|
|
|
), 409, 'An active task allowed concurrent path replacement.');
|
|
|
|
|
|
|
|
|
|
CloudHostStorageConfigurationService::finishJobLocked(
|
|
|
|
|
$job1, 'failed', null, ['effect_may_have_applied' => true]
|
|
|
|
|
);
|
|
|
|
|
$terminalize($job1, 'failed', true);
|
|
|
|
|
$failed = CloudHostStorageConfigurationService::read(1);
|
|
|
|
|
$assert($failed['state'] === 'failed'
|
|
|
|
|
&& $failed['can_submit'] === true
|
|
|
|
|
&& $failed['operation_id'] === (string) $job1->operation_id,
|
|
|
|
|
'A terminal failure remained blocked.');
|
|
|
|
|
|
|
|
|
|
$receipt2 = CloudHostStorageConfigurationService::save(
|
|
|
|
|
1, $sameActive, $actor
|
|
|
|
|
);
|
|
|
|
|
$job2 = CloudJob::find((int) $receipt2['task_id']);
|
|
|
|
|
$row = CloudHostStorageConfiguration::find(1);
|
|
|
|
|
$assert($job2 !== null
|
|
|
|
|
&& (int) $job2->id !== (int) $job1->id
|
|
|
|
|
&& (string) $job2->operation_id !== (string) $job1->operation_id
|
|
|
|
|
&& (int) $row->revision === 1
|
|
|
|
|
&& (string) $row->operation_id === (string) $job2->operation_id
|
|
|
|
|
&& (string) $row->state === 'applying'
|
|
|
|
|
&& (string) $job1->phase === 'terminal'
|
|
|
|
|
&& (string) $job1->outcome === 'failed',
|
|
|
|
|
'A same-path correction did not create a new operation while preserving the failed audit.');
|
|
|
|
|
|
|
|
|
|
CloudHostStorageConfigurationService::finishJobLocked(
|
|
|
|
|
$job1, 'succeeded', ['configured' => true], []
|
|
|
|
|
);
|
|
|
|
|
$row->refresh();
|
|
|
|
|
$assert((string) $row->operation_id === (string) $job2->operation_id
|
|
|
|
|
&& (string) $row->state === 'applying',
|
|
|
|
|
'A late superseded receipt overwrote the current generation.');
|
|
|
|
|
|
|
|
|
|
CloudHostStorageConfigurationService::finishJobLocked(
|
|
|
|
|
$job2, 'failed', null, []
|
|
|
|
|
);
|
|
|
|
|
$terminalize($job2, 'failed');
|
|
|
|
|
$changedAfterFailure = $changedActive;
|
|
|
|
|
$receipt3 = CloudHostStorageConfigurationService::save(
|
|
|
|
|
1, $changedAfterFailure, $actor
|
|
|
|
|
);
|
|
|
|
|
$job3 = CloudJob::find((int) $receipt3['task_id']);
|
|
|
|
|
$row = CloudHostStorageConfiguration::find(1);
|
|
|
|
|
$assert($job3 !== null
|
|
|
|
|
&& (string) $job3->operation_id !== (string) $job2->operation_id
|
|
|
|
|
&& (int) $row->revision === 2
|
|
|
|
|
&& (string) $row->rescue_directory
|
|
|
|
|
=== '/home/libvirt/rescue-corrected',
|
|
|
|
|
'A corrected path after terminal failure did not create the next revision.');
|
|
|
|
|
|
|
|
|
|
CloudHostStorageConfigurationService::finishJobLocked(
|
|
|
|
|
$job3, 'succeeded', ['configured' => true], []
|
|
|
|
|
);
|
|
|
|
|
$terminalize($job3, 'succeeded');
|
|
|
|
|
$applied = CloudHostStorageConfigurationService::read(1);
|
|
|
|
|
$assert($applied['state'] === 'applied'
|
|
|
|
|
&& $applied['configured'] === true
|
|
|
|
|
&& $applied['can_submit'] === false,
|
|
|
|
|
'A confirmed storage layout was not locked as applied.');
|
|
|
|
|
$changedAfterFailure['expected_revision'] = 2;
|
|
|
|
|
$sameApplied = CloudHostStorageConfigurationService::save(
|
|
|
|
|
1, $changedAfterFailure, $actor
|
|
|
|
|
);
|
|
|
|
|
$assert($sameApplied['task_id'] === $receipt3['task_id']
|
|
|
|
|
&& CloudJob::count() === 3,
|
|
|
|
|
'An exact applied request was not idempotent.');
|
|
|
|
|
$differentApplied = $changedAfterFailure;
|
|
|
|
|
$differentApplied['managed_directory'] = '/home/libvirt/other-kvm';
|
|
|
|
|
$expect(static fn () => CloudHostStorageConfigurationService::save(
|
|
|
|
|
1, $differentApplied, $actor
|
|
|
|
|
), 409, 'An applied storage layout was mutable.');
|
|
|
|
|
|
|
|
|
|
// A worker-finalization incident must not turn a known terminal failure
|
|
|
|
|
// into a permanent "applying" fence.
|
|
|
|
|
$receipt4 = CloudHostStorageConfigurationService::save(2, $first, $actor);
|
|
|
|
|
$job4 = CloudJob::find((int) $receipt4['task_id']);
|
|
|
|
|
$terminalize($job4, 'failed', true);
|
|
|
|
|
$staleProjection = CloudHostStorageConfigurationService::read(2);
|
|
|
|
|
$assert($staleProjection['state'] === 'failed'
|
|
|
|
|
&& $staleProjection['can_submit'] === true,
|
|
|
|
|
'A terminally failed job remained visually locked behind stale applying state.');
|
|
|
|
|
$staleApplying = $first;
|
|
|
|
|
$staleApplying['expected_revision'] = 1;
|
|
|
|
|
$receipt5 = CloudHostStorageConfigurationService::save(
|
|
|
|
|
2, $staleApplying, $actor
|
|
|
|
|
);
|
|
|
|
|
$job5 = CloudJob::find((int) $receipt5['task_id']);
|
|
|
|
|
$row2 = CloudHostStorageConfiguration::find(2);
|
|
|
|
|
$assert($job5 !== null
|
|
|
|
|
&& (int) $job5->id !== (int) $job4->id
|
|
|
|
|
&& (string) $row2->operation_id === (string) $job5->operation_id
|
|
|
|
|
&& (int) $row2->revision === 1,
|
|
|
|
|
'A linked terminal failure with a stale applying row remained blocked.');
|
|
|
|
|
|
|
|
|
|
// A reopened storage page locally closes a receipt-less task at the new
|
|
|
|
|
// effective 180-second boundary, even when it persisted the old 960-second
|
|
|
|
|
// deadline. This path performs no Agent request and no redispatch.
|
|
|
|
|
$receipt6 = CloudHostStorageConfigurationService::save(3, $first, $actor);
|
|
|
|
|
$job6 = CloudJob::find((int) $receipt6['task_id']);
|
|
|
|
|
$makeLegacyActive($job6);
|
|
|
|
|
$expiredRead = CloudHostStorageConfigurationService::read(3);
|
|
|
|
|
$job6->refresh();
|
|
|
|
|
$row3 = CloudHostStorageConfiguration::find(3);
|
|
|
|
|
$assert((string) $job6->phase === 'terminal'
|
|
|
|
|
&& (string) $job6->outcome === 'failed'
|
|
|
|
|
&& (string) $row3->state === 'failed'
|
|
|
|
|
&& $expiredRead['state'] === 'failed'
|
|
|
|
|
&& $expiredRead['can_submit'] === true
|
|
|
|
|
&& $expiredRead['task_id'] === null
|
|
|
|
|
&& $expiredRead['status_url'] === null,
|
|
|
|
|
'Storage read did not locally converge and reread an expired task.');
|
|
|
|
|
|
|
|
|
|
// Save has the same fallback. An accepted task carrying the former 300s
|
|
|
|
|
// dispatch deadline is closed after 60s, then a new operation is created.
|
|
|
|
|
$receipt7 = CloudHostStorageConfigurationService::save(4, $first, $actor);
|
|
|
|
|
$job7 = CloudJob::find((int) $receipt7['task_id']);
|
|
|
|
|
$makeLegacyActive($job7, 'accepted');
|
|
|
|
|
$sameLegacy = $first;
|
|
|
|
|
$sameLegacy['expected_revision'] = 1;
|
|
|
|
|
$receipt8 = CloudHostStorageConfigurationService::save(
|
|
|
|
|
4, $sameLegacy, $actor
|
|
|
|
|
);
|
|
|
|
|
$job7->refresh();
|
|
|
|
|
$job8 = CloudJob::find((int) $receipt8['task_id']);
|
|
|
|
|
$row4 = CloudHostStorageConfiguration::find(4);
|
|
|
|
|
$assert((string) $job7->phase === 'terminal'
|
|
|
|
|
&& (string) $job7->outcome === 'failed'
|
|
|
|
|
&& $job8 !== null && (int) $job8->id !== (int) $job7->id
|
|
|
|
|
&& (string) $row4->operation_id === (string) $job8->operation_id
|
|
|
|
|
&& (string) $row4->state === 'applying',
|
|
|
|
|
'Storage save remained blocked by an expired accepted task.');
|
|
|
|
|
|
|
|
|
|
echo "Host storage failure supersession state machine passed.\n";
|
|
|
|
|
}
|