|
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudInstanceBackup;
|
|
|
|
|
use App\Models\CloudBackupSubscription;
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
use App\Models\CloudStorageArtifact;
|
|
|
|
|
use App\Models\CloudStorageOperation;
|
|
|
|
|
use App\Models\CloudStoragePublication;
|
|
|
|
|
use App\Models\CloudStorageReplica;
|
|
|
|
|
use App\Services\Cloud\CloudBackupService;
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
use App\Services\Cloud\CloudPortableRestoreCompatibility;
|
|
|
|
|
use App\Services\Cloud\CloudStorageArtifactService;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
|
|
|
|
|
|
|
|
$assert = static function (bool $condition, string $message): void {
|
|
|
|
|
if (!$condition) {
|
|
|
|
|
throw new RuntimeException($message);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
$expectConflict = static function (callable $callback) use ($assert): void {
|
|
|
|
|
try {
|
|
|
|
|
$callback();
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
'Unexpected conflict status: ' . $error->httpStatus);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
throw new RuntimeException('Expected storage identity conflict.');
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$db = new Capsule();
|
|
|
|
|
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
|
|
|
|
|
$db->setAsGlobal();
|
|
|
|
|
$db->bootEloquent();
|
|
|
|
|
$schema = $db->schema('default');
|
|
|
|
|
|
|
|
|
|
$schema->create('cloud_storage_backends', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id');
|
|
|
|
|
$t->string('backend_key'); $t->string('identity_sha256');
|
|
|
|
|
$t->string('name'); $t->string('driver'); $t->string('endpoint');
|
|
|
|
|
$t->string('region'); $t->string('bucket'); $t->string('prefix');
|
|
|
|
|
$t->boolean('path_style'); $t->string('failure_domain');
|
|
|
|
|
$t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision');
|
|
|
|
|
$t->unsignedBigInteger('tested_revision'); $t->string('admin_state');
|
|
|
|
|
$t->string('health_state'); $t->unsignedInteger('write_priority');
|
|
|
|
|
$t->unsignedInteger('consecutive_failures')->default(0);
|
|
|
|
|
$t->unsignedInteger('consecutive_successes')->default(0);
|
|
|
|
|
$t->string('last_error_code')->default('');
|
|
|
|
|
$t->unsignedBigInteger('last_tested_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('last_healthy_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('next_probe_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_storage_artifacts', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('artifact_key')->unique();
|
|
|
|
|
$t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes');
|
|
|
|
|
$t->unsignedBigInteger('size_bytes'); $t->string('sha256');
|
|
|
|
|
$t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies');
|
|
|
|
|
$t->unsignedTinyInteger('required_copies'); $t->string('state');
|
|
|
|
|
$t->string('last_operation_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('verified_at');
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_storage_replicas', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('replica_key')->unique();
|
|
|
|
|
$t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id');
|
|
|
|
|
$t->string('role'); $t->string('object_key');
|
|
|
|
|
$t->string('object_version_id')->default('');
|
|
|
|
|
$t->string('object_etag')->default('');
|
|
|
|
|
$t->unsignedBigInteger('size_bytes')->default(0);
|
|
|
|
|
$t->string('sha256')->default(''); $t->string('state');
|
|
|
|
|
$t->string('publish_operation_id')->nullable();
|
|
|
|
|
$t->string('delete_operation_id')->nullable();
|
|
|
|
|
$t->unsignedInteger('delete_generation')->default(0);
|
|
|
|
|
$t->unsignedInteger('delete_cycle_attempts')->default(0);
|
|
|
|
|
$t->unsignedInteger('delete_presence_confirmations')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('delete_last_checked_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('delete_next_check_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('delete_attention_at')->nullable();
|
|
|
|
|
$t->string('lease_token')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('lease_until')->default(0);
|
|
|
|
|
$t->string('error_code')->default('');
|
|
|
|
|
$t->unsignedBigInteger('verified_at')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_storage_operations', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('operation_id')->unique();
|
|
|
|
|
$t->string('request_idempotency_key')->unique();
|
|
|
|
|
$t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id');
|
|
|
|
|
$t->unsignedBigInteger('replica_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('source_backend_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('target_backend_id')->nullable();
|
|
|
|
|
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
|
|
|
|
|
$t->string('effect_resolution');
|
|
|
|
|
$t->boolean('effect_may_have_applied')->nullable();
|
|
|
|
|
$t->boolean('safe_to_resubmit')->nullable();
|
|
|
|
|
$t->unsignedInteger('attempts'); $t->string('lease_token')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('lease_until');
|
|
|
|
|
$t->unsignedBigInteger('available_at'); $t->unsignedBigInteger('next_check_at');
|
|
|
|
|
$t->string('error_code'); $t->text('result_json')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('terminal_at')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_storage_publications', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('publication_key')->unique();
|
|
|
|
|
$t->unsignedBigInteger('artifact_id')->unique();
|
|
|
|
|
$t->unsignedBigInteger('backup_id')->nullable()->unique();
|
|
|
|
|
$t->string('template_import_intent_id')->nullable()->unique();
|
|
|
|
|
$t->string('rescue_template_id')->nullable()->unique();
|
|
|
|
|
$t->string('state'); $t->unsignedInteger('attempts')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('available_at')->default(0);
|
|
|
|
|
$t->string('lease_token')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('lease_until')->default(0);
|
|
|
|
|
$t->string('last_error_code')->default('');
|
|
|
|
|
$t->unsignedBigInteger('coverage_ready_at')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('finalized_at')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_instance_backups', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('backup_key')->unique();
|
|
|
|
|
$t->unsignedBigInteger('storage_artifact_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('source_instance_id');
|
|
|
|
|
$t->unsignedBigInteger('source_host_id'); $t->unsignedBigInteger('user_id');
|
|
|
|
|
$t->string('source_uuid'); $t->string('name');
|
|
|
|
|
$t->string('purpose')->default('portable'); $t->string('source')->default('manual');
|
|
|
|
|
$t->unsignedBigInteger('subscription_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('scheduled_for')->nullable();
|
|
|
|
|
$t->unsignedInteger('schedule_attempt')->default(0);
|
|
|
|
|
$t->string('state'); $t->string('consistency'); $t->string('object_key');
|
|
|
|
|
$t->unsignedBigInteger('object_size_bytes')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('virtual_size_bytes')->default(0);
|
|
|
|
|
$t->string('sha256')->default(''); $t->string('object_etag')->default('');
|
|
|
|
|
$t->string('object_version_id')->default('');
|
|
|
|
|
$t->text('source_portable_restore_contract_json')->default('{}');
|
|
|
|
|
$t->string('source_portable_restore_contract_sha256')->default('');
|
|
|
|
|
$t->text('spec_json');
|
|
|
|
|
$t->text('sealed_credential')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('storage_revision')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('active_job_id')->nullable();
|
|
|
|
|
$t->string('last_operation_id')->nullable();
|
|
|
|
|
$t->string('error_code')->default(''); $t->unsignedBigInteger('ready_at')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_jobs', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id'); $t->string('kind'); $t->unsignedBigInteger('host_id');
|
|
|
|
|
$t->unsignedBigInteger('instance_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('backup_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('user_id')->nullable();
|
|
|
|
|
$t->string('operation_id')->unique();
|
|
|
|
|
$t->string('effect_resolution')->default('not_applicable');
|
|
|
|
|
$t->boolean('finalization_incident')->default(false);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_backup_subscriptions',
|
|
|
|
|
static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id');
|
|
|
|
|
$t->unsignedBigInteger('instance_id')->unique();
|
|
|
|
|
$t->unsignedBigInteger('user_id');
|
|
|
|
|
$t->unsignedBigInteger('offer_id');
|
|
|
|
|
$t->boolean('enabled');
|
|
|
|
|
$t->boolean('renew_next_period');
|
|
|
|
|
$t->string('frequency');
|
|
|
|
|
$t->unsignedTinyInteger('weekday');
|
|
|
|
|
$t->unsignedTinyInteger('retention_count');
|
|
|
|
|
$t->string('timezone');
|
|
|
|
|
$t->string('price');
|
|
|
|
|
$t->string('currency');
|
|
|
|
|
$t->unsignedInteger('price_version');
|
|
|
|
|
$t->unsignedInteger('state_revision')->default(1);
|
|
|
|
|
$t->unsignedBigInteger('next_run_at');
|
|
|
|
|
$t->unsignedBigInteger('deferred_until')->default(0);
|
|
|
|
|
$t->string('defer_reason')->default('');
|
|
|
|
|
$t->unsignedBigInteger('deferred_at')->nullable();
|
|
|
|
|
$t->unsignedInteger('defer_count')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('last_scheduled_at')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('last_backup_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('created_at');
|
|
|
|
|
$t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
$now = time();
|
|
|
|
|
foreach ([[1, 'provider-a', 10], [2, 'provider-b', 20]] as [$id, $name, $priority]) {
|
|
|
|
|
Capsule::table('cloud_storage_backends')->insert([
|
|
|
|
|
'id' => $id, 'backend_key' => str_repeat((string) $id, 32),
|
|
|
|
|
'identity_sha256' => str_repeat((string) $id, 64),
|
|
|
|
|
'name' => $name, 'driver' => 's3',
|
|
|
|
|
'endpoint' => 'https://' . $name . '.example.test',
|
|
|
|
|
'region' => 'us-east-1', 'bucket' => $name . '-bucket',
|
|
|
|
|
'prefix' => $name, 'path_style' => 1,
|
|
|
|
|
'failure_domain' => $name, 'sealed_credentials' => 'sealed',
|
|
|
|
|
'config_revision' => 1, 'tested_revision' => 1,
|
|
|
|
|
'admin_state' => 'active', 'health_state' => 'healthy',
|
|
|
|
|
'write_priority' => $priority, 'consecutive_failures' => 0,
|
|
|
|
|
'consecutive_successes' => 1, 'last_error_code' => '',
|
|
|
|
|
'last_tested_at' => $now, 'last_healthy_at' => $now,
|
|
|
|
|
'next_probe_at' => $now + 300, 'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$portableRestore = [
|
|
|
|
|
'contract' => 1, 'architecture' => 'x86_64', 'machine' => 'q35',
|
|
|
|
|
'firmware' => 'uefi', 'secure_boot' => false, 'disk_format' => 'qcow2',
|
|
|
|
|
'system_disk_only' => true, 'public_network_contract' => 1,
|
|
|
|
|
'vpc_inherited' => false,
|
|
|
|
|
];
|
|
|
|
|
$compatibility = CloudPortableRestoreCompatibility::fromPayload(
|
|
|
|
|
$portableRestore
|
|
|
|
|
);
|
|
|
|
|
$sourceSpec = json_encode([
|
|
|
|
|
'managed' => true, 'vcpu' => 2, 'memory_mib' => 2048,
|
|
|
|
|
'disk_gib' => 20, 'bandwidth_mbps' => 1000,
|
|
|
|
|
], JSON_THROW_ON_ERROR);
|
|
|
|
|
|
|
|
|
|
$fixture = static function (int $id, bool $automatic = false) use (
|
|
|
|
|
$now,
|
|
|
|
|
$sourceSpec,
|
|
|
|
|
$compatibility
|
|
|
|
|
): array {
|
|
|
|
|
$backupKey = substr(hash('sha256', 'storage-finalizer-' . $id), 0, 32);
|
|
|
|
|
$agentOperationId = substr(hash('sha256', 'agent-' . $id), 0, 32);
|
|
|
|
|
$scheduledFor = $automatic ? $now - 60 : null;
|
|
|
|
|
$subscriptionId = $automatic ? 100 + $id : null;
|
|
|
|
|
if ($automatic) {
|
|
|
|
|
Capsule::table('cloud_backup_subscriptions')->insert([
|
|
|
|
|
'id' => $subscriptionId,
|
|
|
|
|
'instance_id' => 100 + $id,
|
|
|
|
|
'user_id' => 9,
|
|
|
|
|
'offer_id' => 1,
|
|
|
|
|
'enabled' => 1,
|
|
|
|
|
'renew_next_period' => 1,
|
|
|
|
|
'frequency' => 'daily',
|
|
|
|
|
'weekday' => 0,
|
|
|
|
|
'retention_count' => 2,
|
|
|
|
|
'timezone' => 'UTC',
|
|
|
|
|
'price' => '2.00',
|
|
|
|
|
'currency' => 'USD',
|
|
|
|
|
'price_version' => 1,
|
|
|
|
|
'state_revision' => 1,
|
|
|
|
|
'next_run_at' => $scheduledFor,
|
|
|
|
|
'deferred_until' => 0,
|
|
|
|
|
'defer_reason' => '',
|
|
|
|
|
'deferred_at' => null,
|
|
|
|
|
'defer_count' => 0,
|
|
|
|
|
'last_scheduled_at' => null,
|
|
|
|
|
'last_backup_id' => null,
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
Capsule::table('cloud_instance_backups')->insert([
|
|
|
|
|
'id' => $id, 'backup_key' => $backupKey,
|
|
|
|
|
'storage_artifact_id' => null,
|
|
|
|
|
'source_instance_id' => 100 + $id, 'source_host_id' => 3,
|
|
|
|
|
'user_id' => 9,
|
|
|
|
|
'source_uuid' => sprintf('00000000-0000-4000-8000-%012d', $id),
|
|
|
|
|
'name' => 'storage-finalizer-' . $id, 'state' => 'creating',
|
|
|
|
|
'source' => $automatic ? 'automatic' : 'manual',
|
|
|
|
|
'subscription_id' => $subscriptionId,
|
|
|
|
|
'scheduled_for' => $scheduledFor,
|
|
|
|
|
'schedule_attempt' => $automatic ? 1 : 0,
|
|
|
|
|
'consistency' => 'filesystem',
|
|
|
|
|
'object_key' => 'vm-backups/' . $backupKey . '/system.qcow2',
|
|
|
|
|
'source_portable_restore_contract_json' => $compatibility->json(),
|
|
|
|
|
'source_portable_restore_contract_sha256' => $compatibility->sha256(),
|
|
|
|
|
'spec_json' => $sourceSpec,
|
|
|
|
|
'sealed_credential' => 'sealed-guest', 'storage_revision' => 0,
|
|
|
|
|
'active_job_id' => $id, 'last_operation_id' => $agentOperationId,
|
|
|
|
|
'created_at' => $now, 'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
$publication = CloudStorageArtifactService::beginAgentPublication(
|
|
|
|
|
'backup', 'vm-backups/' . $backupKey . '/system.qcow2',
|
|
|
|
|
21 * 1073741824, 'storage-finalizer-' . str_pad((string) $id, 4, '0', STR_PAD_LEFT),
|
|
|
|
|
hash('sha256', 'storage-finalizer-fingerprint-' . $id),
|
|
|
|
|
['backup_id' => $id]
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_instance_backups')->where('id', $id)->update([
|
|
|
|
|
'storage_artifact_id' => (int) $publication['artifact_id'],
|
|
|
|
|
'object_key' => (string) $publication['object_key'],
|
|
|
|
|
]);
|
|
|
|
|
$work = CloudStorageArtifactService::claimPrimaryPreparation(
|
|
|
|
|
(string) $publication['operation_id'], $now + $id, 300
|
|
|
|
|
);
|
|
|
|
|
CloudStorageArtifactService::markPrimaryAwaitingReceipt(
|
|
|
|
|
(string) $publication['operation_id'],
|
|
|
|
|
(string) $work['lease_token'],
|
|
|
|
|
[
|
|
|
|
|
'upload_id' => 'upload-' . $id,
|
|
|
|
|
'part_size_bytes' => 67108864,
|
|
|
|
|
'part_count' => 336,
|
|
|
|
|
'maximum_size_bytes' => 21 * 1073741824,
|
|
|
|
|
],
|
|
|
|
|
$now + $id
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_jobs')->insert([
|
|
|
|
|
'id' => $id, 'kind' => 'backup_create', 'host_id' => 3,
|
|
|
|
|
'instance_id' => 100 + $id, 'backup_id' => $id,
|
|
|
|
|
'user_id' => 9, 'operation_id' => $agentOperationId,
|
|
|
|
|
'effect_resolution' => 'not_applicable',
|
|
|
|
|
]);
|
|
|
|
|
return [
|
|
|
|
|
'backup' => CloudInstanceBackup::findOrFail($id),
|
|
|
|
|
'job' => CloudJob::findOrFail($id),
|
|
|
|
|
'storage_operation_id' => (string) $publication['operation_id'],
|
|
|
|
|
'artifact_id' => (int) $publication['artifact_id'],
|
|
|
|
|
'subscription_id' => $subscriptionId,
|
|
|
|
|
'scheduled_for' => $scheduledFor,
|
|
|
|
|
];
|
|
|
|
|
};
|
|
|
|
|
$payload = static fn (array $item): array => [
|
|
|
|
|
'owner_id' => 9,
|
|
|
|
|
'backup_key' => (string) $item['backup']->backup_key,
|
|
|
|
|
'maximum_size_bytes' => 21 * 1073741824,
|
|
|
|
|
'storage_artifact_id' => (int) $item['artifact_id'],
|
|
|
|
|
'storage_publication_operation_id' =>
|
|
|
|
|
(string) $item['storage_operation_id'],
|
|
|
|
|
'source_portable_restore_contract_sha256' => $compatibility->sha256(),
|
|
|
|
|
];
|
|
|
|
|
$receipt = static fn (array $item): array => [
|
|
|
|
|
'backup_key' => (string) $item['backup']->backup_key,
|
|
|
|
|
'size_bytes' => 4096,
|
|
|
|
|
'virtual_size_bytes' => 20 * 1073741824,
|
|
|
|
|
'sha256' => str_repeat(dechex(((int) $item['backup']->id % 14) + 1), 64),
|
|
|
|
|
'object_etag' => '"etag-' . (int) $item['backup']->id . '"',
|
|
|
|
|
'object_version_id' => 'version-' . (int) $item['backup']->id,
|
|
|
|
|
'consistency' => 'filesystem', 'cleanup_pending' => false,
|
|
|
|
|
'source_contract' => $portableRestore,
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
$success = $fixture(1);
|
|
|
|
|
$successPayload = $payload($success);
|
|
|
|
|
$successReceipt = $receipt($success);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$success['job'], 'succeeded', $successReceipt, $successPayload
|
|
|
|
|
);
|
|
|
|
|
$ready = CloudInstanceBackup::findOrFail(1);
|
|
|
|
|
$artifact = CloudStorageArtifact::findOrFail($success['artifact_id']);
|
|
|
|
|
$primary = CloudStorageReplica::where(
|
|
|
|
|
'publish_operation_id', $success['storage_operation_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$operation = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $success['storage_operation_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$successPublication = CloudStoragePublication::where(
|
|
|
|
|
'artifact_id', $success['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$successFinalizedAt = (int) $successPublication->finalized_at;
|
|
|
|
|
$assert((string) $ready->state === 'ready'
|
|
|
|
|
&& $ready->active_job_id === null
|
|
|
|
|
&& (string) $operation->outcome === 'succeeded'
|
|
|
|
|
&& (string) $artifact->sha256 === (string) $ready->sha256
|
|
|
|
|
&& (int) $artifact->size_bytes === (int) $ready->object_size_bytes
|
|
|
|
|
&& (string) $primary->state === 'ready'
|
|
|
|
|
&& (string) $primary->object_version_id
|
|
|
|
|
=== (string) $ready->object_version_id
|
|
|
|
|
&& (string) $successPublication->state === 'completed'
|
|
|
|
|
&& (int) $successPublication->attempts === 1
|
|
|
|
|
&& $successFinalizedAt > 0,
|
|
|
|
|
'Agent success did not settle primary storage before exposing the backup.');
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$success['job'], 'succeeded', $successReceipt, $successPayload
|
|
|
|
|
);
|
|
|
|
|
$assert(CloudStorageArtifact::count() === 1
|
|
|
|
|
&& CloudStorageOperation::where('kind', 'publish')->count() === 1
|
|
|
|
|
&& (int) CloudStoragePublication::where(
|
|
|
|
|
'artifact_id', $success['artifact_id']
|
|
|
|
|
)->value('attempts') === 1
|
|
|
|
|
&& (int) CloudStoragePublication::where(
|
|
|
|
|
'artifact_id', $success['artifact_id']
|
|
|
|
|
)->value('finalized_at') === $successFinalizedAt,
|
|
|
|
|
'Identical finalizer replay allocated or replaced storage identity.');
|
|
|
|
|
|
|
|
|
|
$mismatch = $fixture(2);
|
|
|
|
|
$badPayload = $payload($mismatch);
|
|
|
|
|
$badPayload['storage_artifact_id'] = $success['artifact_id'];
|
|
|
|
|
$expectConflict(static fn () => CloudBackupService::finishJobLocked(
|
|
|
|
|
$mismatch['job'], 'succeeded', $receipt($mismatch), $badPayload
|
|
|
|
|
));
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(2)->state === 'creating'
|
|
|
|
|
&& (string) CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $mismatch['storage_operation_id']
|
|
|
|
|
)->value('phase') === 'awaiting_receipt',
|
|
|
|
|
'Mismatched artifact payload mutated backup or storage state.');
|
|
|
|
|
|
|
|
|
|
$uncertain = $fixture(3);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$uncertain['job'], 'uncertain', null, $payload($uncertain)
|
|
|
|
|
);
|
|
|
|
|
$uncertainOperation = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $uncertain['storage_operation_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(3)->state === 'uncertain'
|
|
|
|
|
&& (string) $uncertainOperation->phase === 'reconciling'
|
|
|
|
|
&& (string) $uncertainOperation->effect_resolution === 'unknown'
|
|
|
|
|
&& (string) $uncertainOperation->error_code
|
|
|
|
|
=== 'backup_operation_effect_unknown',
|
|
|
|
|
'Unknown Agent effect did not preserve original primary reconciliation.');
|
|
|
|
|
|
|
|
|
|
$failed = $fixture(4);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$failed['job'], 'failed', null, $payload($failed)
|
|
|
|
|
);
|
|
|
|
|
$failedOperation = CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $failed['storage_operation_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$failedPublication = CloudStoragePublication::where(
|
|
|
|
|
'artifact_id', $failed['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(4)->state === 'failed'
|
|
|
|
|
&& (string) $failedOperation->phase === 'terminal'
|
|
|
|
|
&& (string) $failedOperation->effect_resolution === 'not_applied'
|
|
|
|
|
&& (string) $failedPublication->state === 'failed',
|
|
|
|
|
'A failed Agent result did not close the primary publication as definitely not applied.');
|
|
|
|
|
|
|
|
|
|
$bound = $fixture(5);
|
|
|
|
|
$foreign = $fixture(6);
|
|
|
|
|
$wrongOperationPayload = $payload($bound);
|
|
|
|
|
$wrongOperationPayload['storage_publication_operation_id'] =
|
|
|
|
|
$foreign['storage_operation_id'];
|
|
|
|
|
$expectConflict(static fn () => CloudBackupService::finishJobLocked(
|
|
|
|
|
$bound['job'],
|
|
|
|
|
'succeeded',
|
|
|
|
|
$receipt($bound),
|
|
|
|
|
$wrongOperationPayload
|
|
|
|
|
));
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating'
|
|
|
|
|
&& (string) CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $bound['storage_operation_id']
|
|
|
|
|
)->value('phase') === 'awaiting_receipt'
|
|
|
|
|
&& (string) CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $foreign['storage_operation_id']
|
|
|
|
|
)->value('phase') === 'awaiting_receipt',
|
|
|
|
|
'A foreign successful receipt mutated either artifact ledger.');
|
|
|
|
|
$expectConflict(static fn () => CloudBackupService::finishJobLocked(
|
|
|
|
|
$bound['job'], 'failed', null, $wrongOperationPayload
|
|
|
|
|
));
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating'
|
|
|
|
|
&& (string) CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $bound['storage_operation_id']
|
|
|
|
|
)->value('phase') === 'awaiting_receipt'
|
|
|
|
|
&& (string) CloudStorageOperation::where(
|
|
|
|
|
'operation_id', $foreign['storage_operation_id']
|
|
|
|
|
)->value('phase') === 'awaiting_receipt',
|
|
|
|
|
'A foreign storage operation mutated either artifact ledger.');
|
|
|
|
|
|
|
|
|
|
$safeTransient = [
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
'cause_code' => 'backup_disk_transition_active',
|
|
|
|
|
];
|
|
|
|
|
$automaticRetry = $fixture(7, true);
|
|
|
|
|
$retryStartedAt = time();
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$automaticRetry['job'],
|
|
|
|
|
'failed',
|
|
|
|
|
null,
|
|
|
|
|
$payload($automaticRetry),
|
|
|
|
|
false,
|
|
|
|
|
$safeTransient
|
|
|
|
|
);
|
|
|
|
|
$retiredBackup = CloudInstanceBackup::findOrFail(7);
|
|
|
|
|
$retiredArtifact = CloudStorageArtifact::findOrFail(
|
|
|
|
|
$automaticRetry['artifact_id']
|
|
|
|
|
);
|
|
|
|
|
$retiredPublication = CloudStoragePublication::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$automaticRetry['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$retiredSubscription = CloudBackupSubscription::findOrFail(
|
|
|
|
|
$automaticRetry['subscription_id']
|
|
|
|
|
);
|
|
|
|
|
$retiredOperations = CloudStorageOperation::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$automaticRetry['artifact_id']
|
|
|
|
|
)->get();
|
|
|
|
|
$assert((string) $retiredBackup->state === 'deleted'
|
|
|
|
|
&& (string) $retiredBackup->error_code
|
|
|
|
|
=== 'automatic_transient_retried'
|
|
|
|
|
&& $retiredBackup->sealed_credential === null
|
|
|
|
|
&& $retiredBackup->active_job_id === null
|
|
|
|
|
&& (string) $retiredArtifact->state === 'deleted'
|
|
|
|
|
&& (string) $retiredPublication->state === 'retired'
|
|
|
|
|
&& $retiredPublication->coverage_ready_at === null
|
|
|
|
|
&& (int) $retiredPublication->finalized_at >= $retryStartedAt
|
|
|
|
|
&& CloudStorageReplica::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$automaticRetry['artifact_id']
|
|
|
|
|
)->where('state', '<>', 'deleted')->count() === 0
|
|
|
|
|
&& $retiredOperations->count() === 2
|
|
|
|
|
&& $retiredOperations->every(
|
|
|
|
|
static fn (CloudStorageOperation $operation): bool =>
|
|
|
|
|
(string) $operation->phase === 'terminal'
|
|
|
|
|
&& (string) $operation->outcome === 'failed'
|
|
|
|
|
&& (string) $operation->effect_resolution === 'not_applied'
|
|
|
|
|
)
|
|
|
|
|
&& (int) $retiredSubscription->next_run_at
|
|
|
|
|
=== (int) $automaticRetry['scheduled_for']
|
|
|
|
|
&& (string) $retiredSubscription->defer_reason
|
|
|
|
|
=== 'backup_disk_transition_active'
|
|
|
|
|
&& (int) $retiredSubscription->defer_count === 1
|
|
|
|
|
&& (int) $retiredSubscription->deferred_until
|
|
|
|
|
>= $retryStartedAt + 300
|
|
|
|
|
&& $retiredSubscription->last_scheduled_at === null
|
|
|
|
|
&& $retiredSubscription->last_backup_id === null,
|
|
|
|
|
'A proven not-applied automatic race was not retired and re-armed on the same slot.');
|
|
|
|
|
$retryDeferredUntil = (int) $retiredSubscription->deferred_until;
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$automaticRetry['job'],
|
|
|
|
|
'failed',
|
|
|
|
|
null,
|
|
|
|
|
$payload($automaticRetry),
|
|
|
|
|
false,
|
|
|
|
|
$safeTransient
|
|
|
|
|
);
|
|
|
|
|
$retiredSubscription->refresh();
|
|
|
|
|
$assert((int) $retiredSubscription->defer_count === 1
|
|
|
|
|
&& (int) $retiredSubscription->deferred_until === $retryDeferredUntil
|
|
|
|
|
&& CloudStorageOperation::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$automaticRetry['artifact_id']
|
|
|
|
|
)->count() === 2,
|
|
|
|
|
'A repeated transient failure receipt re-armed or replaced durable evidence.');
|
|
|
|
|
|
|
|
|
|
$abortFailure = $fixture(8, true);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$abortFailure['job'],
|
|
|
|
|
'failed',
|
|
|
|
|
null,
|
|
|
|
|
$payload($abortFailure),
|
|
|
|
|
false,
|
|
|
|
|
[
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
'cause_code' => 'backup_upload_abort_failed',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$abortSubscription = CloudBackupSubscription::findOrFail(
|
|
|
|
|
$abortFailure['subscription_id']
|
|
|
|
|
);
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(8)->state === 'failed'
|
|
|
|
|
&& (string) CloudStorageArtifact::findOrFail(
|
|
|
|
|
$abortFailure['artifact_id']
|
|
|
|
|
)->state === 'publishing'
|
|
|
|
|
&& (string) CloudStoragePublication::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$abortFailure['artifact_id']
|
|
|
|
|
)->value('state') === 'failed'
|
|
|
|
|
&& (int) $abortSubscription->last_scheduled_at
|
|
|
|
|
=== (int) $abortFailure['scheduled_for']
|
|
|
|
|
&& (int) $abortSubscription->last_backup_id === 8
|
|
|
|
|
&& (int) $abortSubscription->next_run_at
|
|
|
|
|
> (int) $abortFailure['scheduled_for']
|
|
|
|
|
&& (int) $abortSubscription->deferred_until === 0,
|
|
|
|
|
'Multipart abort failure was incorrectly treated as a safe transient retry.');
|
|
|
|
|
|
|
|
|
|
$automaticSuccess = $fixture(9, true);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$automaticSuccess['job'],
|
|
|
|
|
'succeeded',
|
|
|
|
|
$receipt($automaticSuccess),
|
|
|
|
|
$payload($automaticSuccess)
|
|
|
|
|
);
|
|
|
|
|
$successSubscription = CloudBackupSubscription::findOrFail(
|
|
|
|
|
$automaticSuccess['subscription_id']
|
|
|
|
|
);
|
|
|
|
|
$successNextRun = (int) $successSubscription->next_run_at;
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$automaticSuccess['job'],
|
|
|
|
|
'succeeded',
|
|
|
|
|
$receipt($automaticSuccess),
|
|
|
|
|
$payload($automaticSuccess)
|
|
|
|
|
);
|
|
|
|
|
$successSubscription->refresh();
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(9)->state === 'ready'
|
|
|
|
|
&& (int) $successSubscription->last_scheduled_at
|
|
|
|
|
=== (int) $automaticSuccess['scheduled_for']
|
|
|
|
|
&& (int) $successSubscription->last_backup_id === 9
|
|
|
|
|
&& (int) $successSubscription->next_run_at === $successNextRun,
|
|
|
|
|
'Ordinary automatic terminal success did not advance exactly once.');
|
|
|
|
|
|
|
|
|
|
$unknownAutomatic = $fixture(10, true);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$unknownAutomatic['job'],
|
|
|
|
|
'uncertain',
|
|
|
|
|
null,
|
|
|
|
|
$payload($unknownAutomatic),
|
|
|
|
|
false,
|
|
|
|
|
[
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => true,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'cause_code' => 'backup_disk_transition_active',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$unknownSubscription = CloudBackupSubscription::findOrFail(
|
|
|
|
|
$unknownAutomatic['subscription_id']
|
|
|
|
|
);
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(10)->state === 'uncertain'
|
|
|
|
|
&& (string) CloudStorageArtifact::findOrFail(
|
|
|
|
|
$unknownAutomatic['artifact_id']
|
|
|
|
|
)->state !== 'deleted'
|
|
|
|
|
&& (string) CloudStoragePublication::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$unknownAutomatic['artifact_id']
|
|
|
|
|
)->value('state') !== 'retired'
|
|
|
|
|
&& (int) $unknownSubscription->next_run_at
|
|
|
|
|
> (int) $unknownAutomatic['scheduled_for']
|
|
|
|
|
&& (int) $unknownSubscription->deferred_until === 0,
|
|
|
|
|
'Unknown automatic backup effect was replayed as a safe transient failure.');
|
|
|
|
|
|
|
|
|
|
$manualTransient = $fixture(11);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$manualTransient['job'],
|
|
|
|
|
'failed',
|
|
|
|
|
null,
|
|
|
|
|
$payload($manualTransient),
|
|
|
|
|
false,
|
|
|
|
|
$safeTransient
|
|
|
|
|
);
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(11)->state === 'failed'
|
|
|
|
|
&& (string) CloudStorageArtifact::findOrFail(
|
|
|
|
|
$manualTransient['artifact_id']
|
|
|
|
|
)->state === 'publishing'
|
|
|
|
|
&& (string) CloudStoragePublication::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$manualTransient['artifact_id']
|
|
|
|
|
)->value('state') === 'failed',
|
|
|
|
|
'A manual backup failure was incorrectly armed for automatic retry.');
|
|
|
|
|
|
|
|
|
|
$rollback = $fixture(12, true);
|
|
|
|
|
$connection = Capsule::connection('default');
|
|
|
|
|
$connection->beginTransaction();
|
|
|
|
|
try {
|
|
|
|
|
$poisonedReplicaId = CloudStorageReplica::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$rollback['artifact_id']
|
|
|
|
|
)->orderBy('id')->value('id');
|
|
|
|
|
Capsule::table('cloud_storage_replicas')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$poisonedReplicaId
|
|
|
|
|
)->update([
|
|
|
|
|
'object_etag' => 'unexpected-provider-identity',
|
|
|
|
|
]);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$rollback['job'],
|
|
|
|
|
'failed',
|
|
|
|
|
null,
|
|
|
|
|
$payload($rollback),
|
|
|
|
|
false,
|
|
|
|
|
$safeTransient
|
|
|
|
|
);
|
|
|
|
|
throw new RuntimeException('Expected blank retirement conflict.');
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
'Unexpected blank retirement conflict status.');
|
|
|
|
|
$connection->rollBack();
|
|
|
|
|
}
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(12)->state === 'creating'
|
|
|
|
|
&& (string) CloudStorageArtifact::findOrFail(
|
|
|
|
|
$rollback['artifact_id']
|
|
|
|
|
)->state === 'publishing'
|
|
|
|
|
&& (string) CloudStorageOperation::where(
|
|
|
|
|
'operation_id',
|
|
|
|
|
$rollback['storage_operation_id']
|
|
|
|
|
)->value('phase') === 'awaiting_receipt'
|
|
|
|
|
&& (string) CloudStoragePublication::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$rollback['artifact_id']
|
|
|
|
|
)->value('state') === 'waiting_primary',
|
|
|
|
|
'A failed transient finalization did not roll back the entire ledger.');
|
|
|
|
|
|
|
|
|
|
echo "Cloud backup storage finalizer tests passed.\n";
|