|
|
|
|
@ -3,6 +3,7 @@
|
|
|
|
|
require dirname(__DIR__) . '/vendor/autoload.php';
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudInstanceBackup;
|
|
|
|
|
use App\Models\CloudBackupSubscription;
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
use App\Models\CloudStorageArtifact;
|
|
|
|
|
use App\Models\CloudStorageOperation;
|
|
|
|
|
@ -128,6 +129,7 @@ $schema->create('cloud_instance_backups', static function (Blueprint $t): void {
|
|
|
|
|
$t->string('purpose')->default('portable'); $t->string('source')->default('manual');
|
|
|
|
|
$t->unsignedBigInteger('subscription_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('scheduled_for')->nullable();
|
|
|
|
|
$t->unsignedInteger('schedule_attempt')->default(0);
|
|
|
|
|
$t->string('state'); $t->string('consistency'); $t->string('object_key');
|
|
|
|
|
$t->unsignedBigInteger('object_size_bytes')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('virtual_size_bytes')->default(0);
|
|
|
|
|
@ -150,6 +152,33 @@ $schema->create('cloud_jobs', static function (Blueprint $t): void {
|
|
|
|
|
$t->unsignedBigInteger('user_id')->nullable();
|
|
|
|
|
$t->string('operation_id')->unique();
|
|
|
|
|
$t->string('effect_resolution')->default('not_applicable');
|
|
|
|
|
$t->boolean('finalization_incident')->default(false);
|
|
|
|
|
});
|
|
|
|
|
$schema->create('cloud_backup_subscriptions',
|
|
|
|
|
static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id');
|
|
|
|
|
$t->unsignedBigInteger('instance_id')->unique();
|
|
|
|
|
$t->unsignedBigInteger('user_id');
|
|
|
|
|
$t->unsignedBigInteger('offer_id');
|
|
|
|
|
$t->boolean('enabled');
|
|
|
|
|
$t->boolean('renew_next_period');
|
|
|
|
|
$t->string('frequency');
|
|
|
|
|
$t->unsignedTinyInteger('weekday');
|
|
|
|
|
$t->unsignedTinyInteger('retention_count');
|
|
|
|
|
$t->string('timezone');
|
|
|
|
|
$t->string('price');
|
|
|
|
|
$t->string('currency');
|
|
|
|
|
$t->unsignedInteger('price_version');
|
|
|
|
|
$t->unsignedInteger('state_revision')->default(1);
|
|
|
|
|
$t->unsignedBigInteger('next_run_at');
|
|
|
|
|
$t->unsignedBigInteger('deferred_until')->default(0);
|
|
|
|
|
$t->string('defer_reason')->default('');
|
|
|
|
|
$t->unsignedBigInteger('deferred_at')->nullable();
|
|
|
|
|
$t->unsignedInteger('defer_count')->default(0);
|
|
|
|
|
$t->unsignedBigInteger('last_scheduled_at')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('last_backup_id')->nullable();
|
|
|
|
|
$t->unsignedBigInteger('created_at');
|
|
|
|
|
$t->unsignedBigInteger('updated_at');
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
$now = time();
|
|
|
|
|
@ -186,13 +215,42 @@ $sourceSpec = json_encode([
|
|
|
|
|
'disk_gib' => 20, 'bandwidth_mbps' => 1000,
|
|
|
|
|
], JSON_THROW_ON_ERROR);
|
|
|
|
|
|
|
|
|
|
$fixture = static function (int $id) use (
|
|
|
|
|
$fixture = static function (int $id, bool $automatic = false) use (
|
|
|
|
|
$now,
|
|
|
|
|
$sourceSpec,
|
|
|
|
|
$compatibility
|
|
|
|
|
): array {
|
|
|
|
|
$backupKey = substr(hash('sha256', 'storage-finalizer-' . $id), 0, 32);
|
|
|
|
|
$agentOperationId = substr(hash('sha256', 'agent-' . $id), 0, 32);
|
|
|
|
|
$scheduledFor = $automatic ? $now - 60 : null;
|
|
|
|
|
$subscriptionId = $automatic ? 100 + $id : null;
|
|
|
|
|
if ($automatic) {
|
|
|
|
|
Capsule::table('cloud_backup_subscriptions')->insert([
|
|
|
|
|
'id' => $subscriptionId,
|
|
|
|
|
'instance_id' => 100 + $id,
|
|
|
|
|
'user_id' => 9,
|
|
|
|
|
'offer_id' => 1,
|
|
|
|
|
'enabled' => 1,
|
|
|
|
|
'renew_next_period' => 1,
|
|
|
|
|
'frequency' => 'daily',
|
|
|
|
|
'weekday' => 0,
|
|
|
|
|
'retention_count' => 2,
|
|
|
|
|
'timezone' => 'UTC',
|
|
|
|
|
'price' => '2.00',
|
|
|
|
|
'currency' => 'USD',
|
|
|
|
|
'price_version' => 1,
|
|
|
|
|
'state_revision' => 1,
|
|
|
|
|
'next_run_at' => $scheduledFor,
|
|
|
|
|
'deferred_until' => 0,
|
|
|
|
|
'defer_reason' => '',
|
|
|
|
|
'deferred_at' => null,
|
|
|
|
|
'defer_count' => 0,
|
|
|
|
|
'last_scheduled_at' => null,
|
|
|
|
|
'last_backup_id' => null,
|
|
|
|
|
'created_at' => $now,
|
|
|
|
|
'updated_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
Capsule::table('cloud_instance_backups')->insert([
|
|
|
|
|
'id' => $id, 'backup_key' => $backupKey,
|
|
|
|
|
'storage_artifact_id' => null,
|
|
|
|
|
@ -200,6 +258,10 @@ $fixture = static function (int $id) use (
|
|
|
|
|
'user_id' => 9,
|
|
|
|
|
'source_uuid' => sprintf('00000000-0000-4000-8000-%012d', $id),
|
|
|
|
|
'name' => 'storage-finalizer-' . $id, 'state' => 'creating',
|
|
|
|
|
'source' => $automatic ? 'automatic' : 'manual',
|
|
|
|
|
'subscription_id' => $subscriptionId,
|
|
|
|
|
'scheduled_for' => $scheduledFor,
|
|
|
|
|
'schedule_attempt' => $automatic ? 1 : 0,
|
|
|
|
|
'consistency' => 'filesystem',
|
|
|
|
|
'object_key' => 'vm-backups/' . $backupKey . '/system.qcow2',
|
|
|
|
|
'source_portable_restore_contract_json' => $compatibility->json(),
|
|
|
|
|
@ -244,6 +306,8 @@ $fixture = static function (int $id) use (
|
|
|
|
|
'job' => CloudJob::findOrFail($id),
|
|
|
|
|
'storage_operation_id' => (string) $publication['operation_id'],
|
|
|
|
|
'artifact_id' => (int) $publication['artifact_id'],
|
|
|
|
|
'subscription_id' => $subscriptionId,
|
|
|
|
|
'scheduled_for' => $scheduledFor,
|
|
|
|
|
];
|
|
|
|
|
};
|
|
|
|
|
$payload = static fn (array $item): array => [
|
|
|
|
|
@ -382,4 +446,232 @@ $assert((string) CloudInstanceBackup::findOrFail(5)->state === 'creating'
|
|
|
|
|
)->value('phase') === 'awaiting_receipt',
|
|
|
|
|
'A foreign storage operation mutated either artifact ledger.');
|
|
|
|
|
|
|
|
|
|
$safeTransient = [
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
'cause_code' => 'backup_disk_transition_active',
|
|
|
|
|
];
|
|
|
|
|
$automaticRetry = $fixture(7, true);
|
|
|
|
|
$retryStartedAt = time();
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$automaticRetry['job'],
|
|
|
|
|
'failed',
|
|
|
|
|
null,
|
|
|
|
|
$payload($automaticRetry),
|
|
|
|
|
false,
|
|
|
|
|
$safeTransient
|
|
|
|
|
);
|
|
|
|
|
$retiredBackup = CloudInstanceBackup::findOrFail(7);
|
|
|
|
|
$retiredArtifact = CloudStorageArtifact::findOrFail(
|
|
|
|
|
$automaticRetry['artifact_id']
|
|
|
|
|
);
|
|
|
|
|
$retiredPublication = CloudStoragePublication::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$automaticRetry['artifact_id']
|
|
|
|
|
)->firstOrFail();
|
|
|
|
|
$retiredSubscription = CloudBackupSubscription::findOrFail(
|
|
|
|
|
$automaticRetry['subscription_id']
|
|
|
|
|
);
|
|
|
|
|
$retiredOperations = CloudStorageOperation::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$automaticRetry['artifact_id']
|
|
|
|
|
)->get();
|
|
|
|
|
$assert((string) $retiredBackup->state === 'deleted'
|
|
|
|
|
&& (string) $retiredBackup->error_code
|
|
|
|
|
=== 'automatic_transient_retried'
|
|
|
|
|
&& $retiredBackup->sealed_credential === null
|
|
|
|
|
&& $retiredBackup->active_job_id === null
|
|
|
|
|
&& (string) $retiredArtifact->state === 'deleted'
|
|
|
|
|
&& (string) $retiredPublication->state === 'retired'
|
|
|
|
|
&& $retiredPublication->coverage_ready_at === null
|
|
|
|
|
&& (int) $retiredPublication->finalized_at >= $retryStartedAt
|
|
|
|
|
&& CloudStorageReplica::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$automaticRetry['artifact_id']
|
|
|
|
|
)->where('state', '<>', 'deleted')->count() === 0
|
|
|
|
|
&& $retiredOperations->count() === 2
|
|
|
|
|
&& $retiredOperations->every(
|
|
|
|
|
static fn (CloudStorageOperation $operation): bool =>
|
|
|
|
|
(string) $operation->phase === 'terminal'
|
|
|
|
|
&& (string) $operation->outcome === 'failed'
|
|
|
|
|
&& (string) $operation->effect_resolution === 'not_applied'
|
|
|
|
|
)
|
|
|
|
|
&& (int) $retiredSubscription->next_run_at
|
|
|
|
|
=== (int) $automaticRetry['scheduled_for']
|
|
|
|
|
&& (string) $retiredSubscription->defer_reason
|
|
|
|
|
=== 'backup_disk_transition_active'
|
|
|
|
|
&& (int) $retiredSubscription->defer_count === 1
|
|
|
|
|
&& (int) $retiredSubscription->deferred_until
|
|
|
|
|
>= $retryStartedAt + 300
|
|
|
|
|
&& $retiredSubscription->last_scheduled_at === null
|
|
|
|
|
&& $retiredSubscription->last_backup_id === null,
|
|
|
|
|
'A proven not-applied automatic race was not retired and re-armed on the same slot.');
|
|
|
|
|
$retryDeferredUntil = (int) $retiredSubscription->deferred_until;
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$automaticRetry['job'],
|
|
|
|
|
'failed',
|
|
|
|
|
null,
|
|
|
|
|
$payload($automaticRetry),
|
|
|
|
|
false,
|
|
|
|
|
$safeTransient
|
|
|
|
|
);
|
|
|
|
|
$retiredSubscription->refresh();
|
|
|
|
|
$assert((int) $retiredSubscription->defer_count === 1
|
|
|
|
|
&& (int) $retiredSubscription->deferred_until === $retryDeferredUntil
|
|
|
|
|
&& CloudStorageOperation::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$automaticRetry['artifact_id']
|
|
|
|
|
)->count() === 2,
|
|
|
|
|
'A repeated transient failure receipt re-armed or replaced durable evidence.');
|
|
|
|
|
|
|
|
|
|
$abortFailure = $fixture(8, true);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$abortFailure['job'],
|
|
|
|
|
'failed',
|
|
|
|
|
null,
|
|
|
|
|
$payload($abortFailure),
|
|
|
|
|
false,
|
|
|
|
|
[
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => true,
|
|
|
|
|
'cause_code' => 'backup_upload_abort_failed',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$abortSubscription = CloudBackupSubscription::findOrFail(
|
|
|
|
|
$abortFailure['subscription_id']
|
|
|
|
|
);
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(8)->state === 'failed'
|
|
|
|
|
&& (string) CloudStorageArtifact::findOrFail(
|
|
|
|
|
$abortFailure['artifact_id']
|
|
|
|
|
)->state === 'publishing'
|
|
|
|
|
&& (string) CloudStoragePublication::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$abortFailure['artifact_id']
|
|
|
|
|
)->value('state') === 'failed'
|
|
|
|
|
&& (int) $abortSubscription->last_scheduled_at
|
|
|
|
|
=== (int) $abortFailure['scheduled_for']
|
|
|
|
|
&& (int) $abortSubscription->last_backup_id === 8
|
|
|
|
|
&& (int) $abortSubscription->next_run_at
|
|
|
|
|
> (int) $abortFailure['scheduled_for']
|
|
|
|
|
&& (int) $abortSubscription->deferred_until === 0,
|
|
|
|
|
'Multipart abort failure was incorrectly treated as a safe transient retry.');
|
|
|
|
|
|
|
|
|
|
$automaticSuccess = $fixture(9, true);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$automaticSuccess['job'],
|
|
|
|
|
'succeeded',
|
|
|
|
|
$receipt($automaticSuccess),
|
|
|
|
|
$payload($automaticSuccess)
|
|
|
|
|
);
|
|
|
|
|
$successSubscription = CloudBackupSubscription::findOrFail(
|
|
|
|
|
$automaticSuccess['subscription_id']
|
|
|
|
|
);
|
|
|
|
|
$successNextRun = (int) $successSubscription->next_run_at;
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$automaticSuccess['job'],
|
|
|
|
|
'succeeded',
|
|
|
|
|
$receipt($automaticSuccess),
|
|
|
|
|
$payload($automaticSuccess)
|
|
|
|
|
);
|
|
|
|
|
$successSubscription->refresh();
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(9)->state === 'ready'
|
|
|
|
|
&& (int) $successSubscription->last_scheduled_at
|
|
|
|
|
=== (int) $automaticSuccess['scheduled_for']
|
|
|
|
|
&& (int) $successSubscription->last_backup_id === 9
|
|
|
|
|
&& (int) $successSubscription->next_run_at === $successNextRun,
|
|
|
|
|
'Ordinary automatic terminal success did not advance exactly once.');
|
|
|
|
|
|
|
|
|
|
$unknownAutomatic = $fixture(10, true);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$unknownAutomatic['job'],
|
|
|
|
|
'uncertain',
|
|
|
|
|
null,
|
|
|
|
|
$payload($unknownAutomatic),
|
|
|
|
|
false,
|
|
|
|
|
[
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => true,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'cause_code' => 'backup_disk_transition_active',
|
|
|
|
|
]
|
|
|
|
|
);
|
|
|
|
|
$unknownSubscription = CloudBackupSubscription::findOrFail(
|
|
|
|
|
$unknownAutomatic['subscription_id']
|
|
|
|
|
);
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(10)->state === 'uncertain'
|
|
|
|
|
&& (string) CloudStorageArtifact::findOrFail(
|
|
|
|
|
$unknownAutomatic['artifact_id']
|
|
|
|
|
)->state !== 'deleted'
|
|
|
|
|
&& (string) CloudStoragePublication::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$unknownAutomatic['artifact_id']
|
|
|
|
|
)->value('state') !== 'retired'
|
|
|
|
|
&& (int) $unknownSubscription->next_run_at
|
|
|
|
|
> (int) $unknownAutomatic['scheduled_for']
|
|
|
|
|
&& (int) $unknownSubscription->deferred_until === 0,
|
|
|
|
|
'Unknown automatic backup effect was replayed as a safe transient failure.');
|
|
|
|
|
|
|
|
|
|
$manualTransient = $fixture(11);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$manualTransient['job'],
|
|
|
|
|
'failed',
|
|
|
|
|
null,
|
|
|
|
|
$payload($manualTransient),
|
|
|
|
|
false,
|
|
|
|
|
$safeTransient
|
|
|
|
|
);
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(11)->state === 'failed'
|
|
|
|
|
&& (string) CloudStorageArtifact::findOrFail(
|
|
|
|
|
$manualTransient['artifact_id']
|
|
|
|
|
)->state === 'publishing'
|
|
|
|
|
&& (string) CloudStoragePublication::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$manualTransient['artifact_id']
|
|
|
|
|
)->value('state') === 'failed',
|
|
|
|
|
'A manual backup failure was incorrectly armed for automatic retry.');
|
|
|
|
|
|
|
|
|
|
$rollback = $fixture(12, true);
|
|
|
|
|
$connection = Capsule::connection('default');
|
|
|
|
|
$connection->beginTransaction();
|
|
|
|
|
try {
|
|
|
|
|
$poisonedReplicaId = CloudStorageReplica::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$rollback['artifact_id']
|
|
|
|
|
)->orderBy('id')->value('id');
|
|
|
|
|
Capsule::table('cloud_storage_replicas')->where(
|
|
|
|
|
'id',
|
|
|
|
|
$poisonedReplicaId
|
|
|
|
|
)->update([
|
|
|
|
|
'object_etag' => 'unexpected-provider-identity',
|
|
|
|
|
]);
|
|
|
|
|
CloudBackupService::finishJobLocked(
|
|
|
|
|
$rollback['job'],
|
|
|
|
|
'failed',
|
|
|
|
|
null,
|
|
|
|
|
$payload($rollback),
|
|
|
|
|
false,
|
|
|
|
|
$safeTransient
|
|
|
|
|
);
|
|
|
|
|
throw new RuntimeException('Expected blank retirement conflict.');
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
$assert($error->httpStatus === 409,
|
|
|
|
|
'Unexpected blank retirement conflict status.');
|
|
|
|
|
$connection->rollBack();
|
|
|
|
|
}
|
|
|
|
|
$assert((string) CloudInstanceBackup::findOrFail(12)->state === 'creating'
|
|
|
|
|
&& (string) CloudStorageArtifact::findOrFail(
|
|
|
|
|
$rollback['artifact_id']
|
|
|
|
|
)->state === 'publishing'
|
|
|
|
|
&& (string) CloudStorageOperation::where(
|
|
|
|
|
'operation_id',
|
|
|
|
|
$rollback['storage_operation_id']
|
|
|
|
|
)->value('phase') === 'awaiting_receipt'
|
|
|
|
|
&& (string) CloudStoragePublication::where(
|
|
|
|
|
'artifact_id',
|
|
|
|
|
$rollback['artifact_id']
|
|
|
|
|
)->value('state') === 'waiting_primary',
|
|
|
|
|
'A failed transient finalization did not roll back the entire ledger.');
|
|
|
|
|
|
|
|
|
|
echo "Cloud backup storage finalizer tests passed.\n";
|
|
|
|
|
|