You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudBackupStorageFinalizer...

678 lines
28 KiB
PHTML

<?php
require dirname(__DIR__) . '/vendor/autoload.php';
use App\Models\CloudInstanceBackup;
use App\Models\CloudBackupSubscription;
use App\Models\CloudJob;
use App\Models\CloudStorageArtifact;
use App\Models\CloudStorageOperation;
use App\Models\CloudStoragePublication;
use App\Models\CloudStorageReplica;
use App\Services\Cloud\CloudBackupService;
use App\Services\Cloud\CloudError;
use App\Services\Cloud\CloudPortableRestoreCompatibility;
use App\Services\Cloud\CloudStorageArtifactService;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
$assert = static function (bool $condition, string $message): void {
if (!$condition) {
throw new RuntimeException($message);
}
};
$expectConflict = static function (callable $callback) use ($assert): void {
try {
$callback();
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Unexpected conflict status: ' . $error->httpStatus);
return;
}
throw new RuntimeException('Expected storage identity conflict.');
};
$db = new Capsule();
$db->addConnection(['driver' => 'sqlite', 'database' => ':memory:'], 'default');
$db->setAsGlobal();
$db->bootEloquent();
$schema = $db->schema('default');
$schema->create('cloud_storage_backends', static function (Blueprint $t): void {
$t->increments('id');
$t->string('backend_key'); $t->string('identity_sha256');
$t->string('name'); $t->string('driver'); $t->string('endpoint');
$t->string('region'); $t->string('bucket'); $t->string('prefix');
$t->boolean('path_style'); $t->string('failure_domain');
$t->text('sealed_credentials'); $t->unsignedBigInteger('config_revision');
$t->unsignedBigInteger('tested_revision'); $t->string('admin_state');
$t->string('health_state'); $t->unsignedInteger('write_priority');
$t->unsignedInteger('consecutive_failures')->default(0);
$t->unsignedInteger('consecutive_successes')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('last_tested_at')->default(0);
$t->unsignedBigInteger('last_healthy_at')->default(0);
$t->unsignedBigInteger('next_probe_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_artifacts', static function (Blueprint $t): void {
$t->increments('id'); $t->string('artifact_key')->unique();
$t->string('kind'); $t->unsignedBigInteger('maximum_size_bytes');
$t->unsignedBigInteger('size_bytes'); $t->string('sha256');
$t->string('durability_policy'); $t->unsignedTinyInteger('desired_copies');
$t->unsignedTinyInteger('required_copies'); $t->string('state');
$t->string('last_operation_id')->nullable();
$t->unsignedBigInteger('verified_at');
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_replicas', static function (Blueprint $t): void {
$t->increments('id'); $t->string('replica_key')->unique();
$t->unsignedBigInteger('artifact_id'); $t->unsignedBigInteger('backend_id');
$t->string('role'); $t->string('object_key');
$t->string('object_version_id')->default('');
$t->string('object_etag')->default('');
$t->unsignedBigInteger('size_bytes')->default(0);
$t->string('sha256')->default(''); $t->string('state');
$t->string('publish_operation_id')->nullable();
$t->string('delete_operation_id')->nullable();
$t->unsignedInteger('delete_generation')->default(0);
$t->unsignedInteger('delete_cycle_attempts')->default(0);
$t->unsignedInteger('delete_presence_confirmations')->default(0);
$t->unsignedBigInteger('delete_last_checked_at')->default(0);
$t->unsignedBigInteger('delete_next_check_at')->default(0);
$t->unsignedBigInteger('delete_attention_at')->nullable();
$t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until')->default(0);
$t->string('error_code')->default('');
$t->unsignedBigInteger('verified_at')->default(0);
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_operations', static function (Blueprint $t): void {
$t->increments('id'); $t->string('operation_id')->unique();
$t->string('request_idempotency_key')->unique();
$t->string('request_fingerprint'); $t->unsignedBigInteger('artifact_id');
$t->unsignedBigInteger('replica_id')->nullable();
$t->unsignedBigInteger('source_backend_id')->nullable();
$t->unsignedBigInteger('target_backend_id')->nullable();
$t->string('kind'); $t->string('phase'); $t->string('outcome')->nullable();
$t->string('effect_resolution');
$t->boolean('effect_may_have_applied')->nullable();
$t->boolean('safe_to_resubmit')->nullable();
$t->unsignedInteger('attempts'); $t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until');
$t->unsignedBigInteger('available_at'); $t->unsignedBigInteger('next_check_at');
$t->string('error_code'); $t->text('result_json')->nullable();
$t->unsignedBigInteger('terminal_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_storage_publications', static function (Blueprint $t): void {
$t->increments('id'); $t->string('publication_key')->unique();
$t->unsignedBigInteger('artifact_id')->unique();
$t->unsignedBigInteger('backup_id')->nullable()->unique();
$t->string('template_import_intent_id')->nullable()->unique();
$t->string('rescue_template_id')->nullable()->unique();
$t->string('state'); $t->unsignedInteger('attempts')->default(0);
$t->unsignedBigInteger('available_at')->default(0);
$t->string('lease_token')->nullable();
$t->unsignedBigInteger('lease_until')->default(0);
$t->string('last_error_code')->default('');
$t->unsignedBigInteger('coverage_ready_at')->nullable();
$t->unsignedBigInteger('finalized_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_instance_backups', static function (Blueprint $t): void {
$t->increments('id'); $t->string('backup_key')->unique();
$t->unsignedBigInteger('storage_artifact_id')->nullable();
$t->unsignedBigInteger('source_instance_id');
$t->unsignedBigInteger('source_host_id'); $t->unsignedBigInteger('user_id');
$t->string('source_uuid'); $t->string('name');
$t->string('purpose')->default('portable'); $t->string('source')->default('manual');
$t->unsignedBigInteger('subscription_id')->nullable();
$t->unsignedBigInteger('scheduled_for')->nullable();
$t->unsignedInteger('schedule_attempt')->default(0);
$t->string('state'); $t->string('consistency'); $t->string('object_key');
$t->unsignedBigInteger('object_size_bytes')->default(0);
$t->unsignedBigInteger('virtual_size_bytes')->default(0);
$t->string('sha256')->default(''); $t->string('object_etag')->default('');
$t->string('object_version_id')->default('');
$t->text('source_portable_restore_contract_json')->default('{}');
$t->string('source_portable_restore_contract_sha256')->default('');
$t->text('spec_json');
$t->text('sealed_credential')->nullable();
$t->unsignedBigInteger('storage_revision')->default(0);
$t->unsignedBigInteger('active_job_id')->nullable();
$t->string('last_operation_id')->nullable();
$t->string('error_code')->default(''); $t->unsignedBigInteger('ready_at')->nullable();
$t->unsignedBigInteger('created_at'); $t->unsignedBigInteger('updated_at');
});
$schema->create('cloud_jobs', static function (Blueprint $t): void {
$t->increments('id'); $t->string('kind'); $t->unsignedBigInteger('host_id');
$t->unsignedBigInteger('instance_id')->nullable();
$t->unsignedBigInteger('backup_id')->nullable();
$t->unsignedBigInteger('user_id')->nullable();
$t->string('operation_id')->unique();
$t->string('effect_resolution')->default('not_applicable');
$t->boolean('finalization_incident')->default(false);
});
$schema->create('cloud_backup_subscriptions',
static function (Blueprint $t): void {
$t->increments('id');
$t->unsignedBigInteger('instance_id')->unique();
$t->unsignedBigInteger('user_id');
$t->unsignedBigInteger('offer_id');
$t->boolean('enabled');
$t->boolean('renew_next_period');
$t->string('frequency');
$t->unsignedTinyInteger('weekday');
$t->unsignedTinyInteger('retention_count');
$t->string('timezone');
$t->string('price');
$t->string('currency');
$t->unsignedInteger('price_version');
$t->unsignedInteger('state_revision')->default(1);
$t->unsignedBigInteger('next_run_at');
$t->unsignedBigInteger('deferred_until')->default(0);
$t->string('defer_reason')->default('');
$t->unsignedBigInteger('deferred_at')->nullable();
$t->unsignedInteger('defer_count')->default(0);
$t->unsignedBigInteger('last_scheduled_at')->nullable();
$t->unsignedBigInteger('last_backup_id')->nullable();
$t->unsignedBigInteger('created_at');
$t->unsignedBigInteger('updated_at');
});
$now = time();
foreach ([[1, 'provider-a', 10], [2, 'provider-b', 20]] as [$id, $name, $priority]) {
Capsule::table('cloud_storage_backends')->insert([
'id' => $id, 'backend_key' => str_repeat((string) $id, 32),
'identity_sha256' => str_repeat((string) $id, 64),
'name' => $name, 'driver' => 's3',
'endpoint' => 'https://' . $name . '.example.test',
'region' => 'us-east-1', 'bucket' => $name . '-bucket',
'prefix' => $name, 'path_style' => 1,
'failure_domain' => $name, 'sealed_credentials' => 'sealed',
'config_revision' => 1, 'tested_revision' => 1,
'admin_state' => 'active', 'health_state' => 'healthy',
'write_priority' => $priority, 'consecutive_failures' => 0,
'consecutive_successes' => 1, 'last_error_code' => '',
'last_tested_at' => $now, 'last_healthy_at' => $now,
'next_probe_at' => $now + 300, 'created_at' => $now,
'updated_at' => $now,
]);
}
$portableRestore = [
'contract' => 1, 'architecture' => 'x86_64', 'machine' => 'q35',
'firmware' => 'uefi', 'secure_boot' => false, 'disk_format' => 'qcow2',
'system_disk_only' => true, 'public_network_contract' => 1,
'vpc_inherited' => false,
];
$compatibility = CloudPortableRestoreCompatibility::fromPayload(
$portableRestore
);
$sourceSpec = json_encode([
'managed' => true, 'vcpu' => 2, 'memory_mib' => 2048,
'disk_gib' => 20, 'bandwidth_mbps' => 1000,
], JSON_THROW_ON_ERROR);
$fixture = static function (int $id, bool $automatic = false) use (
$now,
$sourceSpec,
$compatibility
): array {
$backupKey = substr(hash('sha256', 'storage-finalizer-' . $id), 0, 32);
$agentOperationId = substr(hash('sha256', 'agent-' . $id), 0, 32);
$scheduledFor = $automatic ? $now - 60 : null;
$subscriptionId = $automatic ? 100 + $id : null;
if ($automatic) {
Capsule::table('cloud_backup_subscriptions')->insert([
'id' => $subscriptionId,
'instance_id' => 100 + $id,
'user_id' => 9,
'offer_id' => 1,
'enabled' => 1,
'renew_next_period' => 1,
'frequency' => 'daily',
'weekday' => 0,
'retention_count' => 2,
'timezone' => 'UTC',
'price' => '2.00',
'currency' => 'USD',
'price_version' => 1,
'state_revision' => 1,
'next_run_at' => $scheduledFor,
'deferred_until' => 0,
'defer_reason' => '',
'deferred_at' => null,
'defer_count' => 0,
'last_scheduled_at' => null,
'last_backup_id' => null,
'created_at' => $now,
'updated_at' => $now,
]);
}
Capsule::table('cloud_instance_backups')->insert([
'id' => $id, 'backup_key' => $backupKey,
'storage_artifact_id' => null,
'source_instance_id' => 100 + $id, 'source_host_id' => 3,
'user_id' => 9,
'source_uuid' => sprintf('00000000-0000-4000-8000-%012d', $id),
'name' => 'storage-finalizer-' . $id, 'state' => 'creating',
'source' => $automatic ? 'automatic' : 'manual',
'subscription_id' => $subscriptionId,
'scheduled_for' => $scheduledFor,
'schedule_attempt' => $automatic ? 1 : 0,
'consistency' => 'filesystem',
'object_key' => 'vm-backups/' . $backupKey . '/system.qcow2',
'source_portable_restore_contract_json' => $compatibility->json(),
'source_portable_restore_contract_sha256' => $compatibility->sha256(),
'spec_json' => $sourceSpec,
'sealed_credential' => 'sealed-guest', 'storage_revision' => 0,
'active_job_id' => $id, 'last_operation_id' => $agentOperationId,
'created_at' => $now, 'updated_at' => $now,
]);
$publication = CloudStorageArtifactService::beginAgentPublication(
'backup', 'vm-backups/' . $backupKey . '/system.qcow2',
21 * 1073741824, 'storage-finalizer-' . str_pad((string) $id, 4, '0', STR_PAD_LEFT),
hash('sha256', 'storage-finalizer-fingerprint-' . $id),
['backup_id' => $id]
);
Capsule::table('cloud_instance_backups')->where('id', $id)->update([
'storage_artifact_id' => (int) $publication['artifact_id'],
'object_key' => (string) $publication['object_key'],
]);
$work = CloudStorageArtifactService::claimPrimaryPreparation(
(string) $publication['operation_id'], $now + $id, 300
);
CloudStorageArtifactService::markPrimaryAwaitingReceipt(
(string) $publication['operation_id'],
(string) $work['lease_token'],
[
'upload_id' => 'upload-' . $id,
'part_size_bytes' => 67108864,
'part_count' => 336,
'maximum_size_bytes' => 21 * 1073741824,
],
$now + $id
);
Capsule::table('cloud_jobs')->insert([
'id' => $id, 'kind' => 'backup_create', 'host_id' => 3,
'instance_id' => 100 + $id, 'backup_id' => $id,
'user_id' => 9, 'operation_id' => $agentOperationId,
'effect_resolution' => 'not_applicable',
]);
return [
'backup' => CloudInstanceBackup::findOrFail($id),
'job' => CloudJob::findOrFail($id),
'storage_operation_id' => (string) $publication['operation_id'],
'artifact_id' => (int) $publication['artifact_id'],
'subscription_id' => $subscriptionId,
'scheduled_for' => $scheduledFor,
];
};
$payload = static fn (array $item): array => [
'owner_id' => 9,
'backup_key' => (string) $item['backup']->backup_key,
'maximum_size_bytes' => 21 * 1073741824,
'storage_artifact_id' => (int) $item['artifact_id'],
'storage_publication_operation_id' =>
(string) $item['storage_operation_id'],
'source_portable_restore_contract_sha256' => $compatibility->sha256(),
];
$receipt = static fn (array $item): array => [
'backup_key' => (string) $item['backup']->backup_key,
'size_bytes' => 4096,
'virtual_size_bytes' => 20 * 1073741824,
'sha256' => str_repeat(dechex(((int) $item['backup']->id % 14) + 1), 64),
'object_etag' => '"etag-' . (int) $item['backup']->id . '"',
'object_version_id' => 'version-' . (int) $item['backup']->id,
'consistency' => 'filesystem', 'cleanup_pending' => false,
'source_contract' => $portableRestore,
];
$success = $fixture(1);
$successPayload = $payload($success);
$successReceipt = $receipt($success);
CloudBackupService::finishJobLocked(
$success['job'], 'succeeded', $successReceipt, $successPayload
);
$ready = CloudInstanceBackup::findOrFail(1);
$artifact = CloudStorageArtifact::findOrFail($success['artifact_id']);
$primary = CloudStorageReplica::where(
'publish_operation_id', $success['storage_operation_id']
)->firstOrFail();
$operation = CloudStorageOperation::where(
'operation_id', $success['storage_operation_id']
)->firstOrFail();
$successPublication = CloudStoragePublication::where(
'artifact_id', $success['artifact_id']
)->firstOrFail();
$successFinalizedAt = (int) $successPublication->finalized_at;
$assert((string) $ready->state === 'ready'
&& $ready->active_job_id === null
&& (string) $operation->outcome === 'succeeded'
&& (string) $artifact->sha256 === (string) $ready->sha256
&& (int) $artifact->size_bytes === (int) $ready->object_size_bytes
&& (string) $primary->state === 'ready'
&& (string) $primary->object_version_id
=== (string) $ready->object_version_id
&& (string) $successPublication->state === 'completed'
&& (int) $successPublication->attempts === 1
&& $successFinalizedAt > 0,
'Agent success did not settle primary storage before exposing the backup.');
CloudBackupService::finishJobLocked(
$success['job'], 'succeeded', $successReceipt, $successPayload
);
$assert(CloudStorageArtifact::count() === 1
&& CloudStorageOperation::where('kind', 'publish')->count() === 1
&& (int) CloudStoragePublication::where(
'artifact_id', $success['artifact_id']
)->value('attempts') === 1
&& (int) CloudStoragePublication::where(
'artifact_id', $success['artifact_id']
)->value('finalized_at') === $successFinalizedAt,
'Identical finalizer replay allocated or replaced storage identity.');
$mismatch = $fixture(2);
$badPayload = $payload($mismatch);
$badPayload['storage_artifact_id'] = $success['artifact_id'];
$expectConflict(static fn () => CloudBackupService::finishJobLocked(
$mismatch['job'], 'succeeded', $receipt($mismatch), $badPayload
));
$assert((string) CloudInstanceBackup::findOrFail(2)->state === 'creating'
&& (string) CloudStorageOperation::where(
'operation_id', $mismatch['storage_operation_id']
)->value('phase') === 'awaiting_receipt',
'Mismatched artifact payload mutated backup or storage state.');
$uncertain = $fixture(3);
CloudBackupService::finishJobLocked(
$uncertain['job'], 'uncertain', null, $payload($uncertain)
);
$uncertainOperation = CloudStorageOperation::where(
'operation_id', $uncertain['storage_operation_id']
)->firstOrFail();
$assert((string) CloudInstanceBackup::findOrFail(3)->state === 'uncertain'
&& (string) $uncertainOperation->phase === 'reconciling'
&& (string) $uncertainOperation->effect_resolution === 'unknown'
&& (string) $uncertainOperation->error_code
=== 'backup_operation_effect_unknown',
'Unknown Agent effect did not preserve original primary reconciliation.');
$failed = $fixture(4);
CloudBackupService::finishJobLocked(
$failed['job'], 'failed', null, $payload($failed)
);
$failedOperation = CloudStorageOperation::where(
'operation_id', $failed['storage_operation_id']
)->firstOrFail();
$failedPublication = CloudStoragePublication::where(
'artifact_id', $failed['artifact_id']
)->firstOrFail();
$assert((string) CloudInstanceBackup::findOrFail(4)->state === 'failed'
&& (string) $failedOperation->phase === 'terminal'
&& (string) $failedOperation->effect_resolution === 'not_applied'
&& (string) $failedPublication->state === 'failed',
'A failed Agent result did not close the primary publication as definitely not applied.');
$bound = $fixture(5);
$foreign = $fixture(6);
$wrongOperationPayload = $payload($bound);
$wrongOperationPayload['storage_publication_operation_id'] =
$foreign['storage_operation_id'];
$expectConflict(static fn () => CloudBackupService::finishJobLocked(
$bound['job'],
'succeeded',
$receipt($bound),
$wrongOperationPayload
));
$assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating'
&& (string) CloudStorageOperation::where(
'operation_id', $bound['storage_operation_id']
)->value('phase') === 'awaiting_receipt'
&& (string) CloudStorageOperation::where(
'operation_id', $foreign['storage_operation_id']
)->value('phase') === 'awaiting_receipt',
'A foreign successful receipt mutated either artifact ledger.');
$expectConflict(static fn () => CloudBackupService::finishJobLocked(
$bound['job'], 'failed', null, $wrongOperationPayload
));
$assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating'
&& (string) CloudStorageOperation::where(
'operation_id', $bound['storage_operation_id']
)->value('phase') === 'awaiting_receipt'
&& (string) CloudStorageOperation::where(
'operation_id', $foreign['storage_operation_id']
)->value('phase') === 'awaiting_receipt',
'A foreign storage operation mutated either artifact ledger.');
$safeTransient = [
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'cause_code' => 'backup_disk_transition_active',
];
$automaticRetry = $fixture(7, true);
$retryStartedAt = time();
CloudBackupService::finishJobLocked(
$automaticRetry['job'],
'failed',
null,
$payload($automaticRetry),
false,
$safeTransient
);
$retiredBackup = CloudInstanceBackup::findOrFail(7);
$retiredArtifact = CloudStorageArtifact::findOrFail(
$automaticRetry['artifact_id']
);
$retiredPublication = CloudStoragePublication::where(
'artifact_id',
$automaticRetry['artifact_id']
)->firstOrFail();
$retiredSubscription = CloudBackupSubscription::findOrFail(
$automaticRetry['subscription_id']
);
$retiredOperations = CloudStorageOperation::where(
'artifact_id',
$automaticRetry['artifact_id']
)->get();
$assert((string) $retiredBackup->state === 'deleted'
&& (string) $retiredBackup->error_code
=== 'automatic_transient_retried'
&& $retiredBackup->sealed_credential === null
&& $retiredBackup->active_job_id === null
&& (string) $retiredArtifact->state === 'deleted'
&& (string) $retiredPublication->state === 'retired'
&& $retiredPublication->coverage_ready_at === null
&& (int) $retiredPublication->finalized_at >= $retryStartedAt
&& CloudStorageReplica::where(
'artifact_id',
$automaticRetry['artifact_id']
)->where('state', '<>', 'deleted')->count() === 0
&& $retiredOperations->count() === 2
&& $retiredOperations->every(
static fn (CloudStorageOperation $operation): bool =>
(string) $operation->phase === 'terminal'
&& (string) $operation->outcome === 'failed'
&& (string) $operation->effect_resolution === 'not_applied'
)
&& (int) $retiredSubscription->next_run_at
=== (int) $automaticRetry['scheduled_for']
&& (string) $retiredSubscription->defer_reason
=== 'backup_disk_transition_active'
&& (int) $retiredSubscription->defer_count === 1
&& (int) $retiredSubscription->deferred_until
>= $retryStartedAt + 300
&& $retiredSubscription->last_scheduled_at === null
&& $retiredSubscription->last_backup_id === null,
'A proven not-applied automatic race was not retired and re-armed on the same slot.');
$retryDeferredUntil = (int) $retiredSubscription->deferred_until;
CloudBackupService::finishJobLocked(
$automaticRetry['job'],
'failed',
null,
$payload($automaticRetry),
false,
$safeTransient
);
$retiredSubscription->refresh();
$assert((int) $retiredSubscription->defer_count === 1
&& (int) $retiredSubscription->deferred_until === $retryDeferredUntil
&& CloudStorageOperation::where(
'artifact_id',
$automaticRetry['artifact_id']
)->count() === 2,
'A repeated transient failure receipt re-armed or replaced durable evidence.');
$abortFailure = $fixture(8, true);
CloudBackupService::finishJobLocked(
$abortFailure['job'],
'failed',
null,
$payload($abortFailure),
false,
[
'status' => 'failed',
'effect_may_have_applied' => false,
'safe_to_resubmit' => true,
'cause_code' => 'backup_upload_abort_failed',
]
);
$abortSubscription = CloudBackupSubscription::findOrFail(
$abortFailure['subscription_id']
);
$assert((string) CloudInstanceBackup::findOrFail(8)->state === 'failed'
&& (string) CloudStorageArtifact::findOrFail(
$abortFailure['artifact_id']
)->state === 'publishing'
&& (string) CloudStoragePublication::where(
'artifact_id',
$abortFailure['artifact_id']
)->value('state') === 'failed'
&& (int) $abortSubscription->last_scheduled_at
=== (int) $abortFailure['scheduled_for']
&& (int) $abortSubscription->last_backup_id === 8
&& (int) $abortSubscription->next_run_at
> (int) $abortFailure['scheduled_for']
&& (int) $abortSubscription->deferred_until === 0,
'Multipart abort failure was incorrectly treated as a safe transient retry.');
$automaticSuccess = $fixture(9, true);
CloudBackupService::finishJobLocked(
$automaticSuccess['job'],
'succeeded',
$receipt($automaticSuccess),
$payload($automaticSuccess)
);
$successSubscription = CloudBackupSubscription::findOrFail(
$automaticSuccess['subscription_id']
);
$successNextRun = (int) $successSubscription->next_run_at;
CloudBackupService::finishJobLocked(
$automaticSuccess['job'],
'succeeded',
$receipt($automaticSuccess),
$payload($automaticSuccess)
);
$successSubscription->refresh();
$assert((string) CloudInstanceBackup::findOrFail(9)->state === 'ready'
&& (int) $successSubscription->last_scheduled_at
=== (int) $automaticSuccess['scheduled_for']
&& (int) $successSubscription->last_backup_id === 9
&& (int) $successSubscription->next_run_at === $successNextRun,
'Ordinary automatic terminal success did not advance exactly once.');
$unknownAutomatic = $fixture(10, true);
CloudBackupService::finishJobLocked(
$unknownAutomatic['job'],
'uncertain',
null,
$payload($unknownAutomatic),
false,
[
'status' => 'failed',
'effect_may_have_applied' => true,
'safe_to_resubmit' => false,
'cause_code' => 'backup_disk_transition_active',
]
);
$unknownSubscription = CloudBackupSubscription::findOrFail(
$unknownAutomatic['subscription_id']
);
$assert((string) CloudInstanceBackup::findOrFail(10)->state === 'uncertain'
&& (string) CloudStorageArtifact::findOrFail(
$unknownAutomatic['artifact_id']
)->state !== 'deleted'
&& (string) CloudStoragePublication::where(
'artifact_id',
$unknownAutomatic['artifact_id']
)->value('state') !== 'retired'
&& (int) $unknownSubscription->next_run_at
> (int) $unknownAutomatic['scheduled_for']
&& (int) $unknownSubscription->deferred_until === 0,
'Unknown automatic backup effect was replayed as a safe transient failure.');
$manualTransient = $fixture(11);
CloudBackupService::finishJobLocked(
$manualTransient['job'],
'failed',
null,
$payload($manualTransient),
false,
$safeTransient
);
$assert((string) CloudInstanceBackup::findOrFail(11)->state === 'failed'
&& (string) CloudStorageArtifact::findOrFail(
$manualTransient['artifact_id']
)->state === 'publishing'
&& (string) CloudStoragePublication::where(
'artifact_id',
$manualTransient['artifact_id']
)->value('state') === 'failed',
'A manual backup failure was incorrectly armed for automatic retry.');
$rollback = $fixture(12, true);
$connection = Capsule::connection('default');
$connection->beginTransaction();
try {
$poisonedReplicaId = CloudStorageReplica::where(
'artifact_id',
$rollback['artifact_id']
)->orderBy('id')->value('id');
Capsule::table('cloud_storage_replicas')->where(
'id',
$poisonedReplicaId
)->update([
'object_etag' => 'unexpected-provider-identity',
]);
CloudBackupService::finishJobLocked(
$rollback['job'],
'failed',
null,
$payload($rollback),
false,
$safeTransient
);
throw new RuntimeException('Expected blank retirement conflict.');
} catch (CloudError $error) {
$assert($error->httpStatus === 409,
'Unexpected blank retirement conflict status.');
$connection->rollBack();
}
$assert((string) CloudInstanceBackup::findOrFail(12)->state === 'creating'
&& (string) CloudStorageArtifact::findOrFail(
$rollback['artifact_id']
)->state === 'publishing'
&& (string) CloudStorageOperation::where(
'operation_id',
$rollback['storage_operation_id']
)->value('phase') === 'awaiting_receipt'
&& (string) CloudStoragePublication::where(
'artifact_id',
$rollback['artifact_id']
)->value('state') === 'waiting_primary',
'A failed transient finalization did not roll back the entire ledger.');
echo "Cloud backup storage finalizer tests passed.\n";