|
|
|
|
<?php
|
|
|
|
|
|
|
|
|
|
declare(strict_types=1);
|
|
|
|
|
|
|
|
|
|
use App\Models\CloudJob;
|
|
|
|
|
use App\Services\Cloud\CloudJobService;
|
|
|
|
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Reuse the complete aggregate-ledger fixture. The focused checks below cover
|
|
|
|
|
* the extra catalog-pointer evidence path without maintaining a second schema.
|
|
|
|
|
*/
|
|
|
|
|
require __DIR__ . '/CloudAggregateRecoveryConvergenceTest.php';
|
|
|
|
|
|
|
|
|
|
foreach ([
|
|
|
|
|
'assert' => $assert ?? null,
|
|
|
|
|
'insertParent' => $insertParent ?? null,
|
|
|
|
|
'insertResolvedChild' => $insertResolvedChild ?? null,
|
|
|
|
|
'insertProjectionForParent' => $insertProjectionForParent ?? null,
|
|
|
|
|
'makeLegacyEligible' => $makeLegacyEligible ?? null,
|
|
|
|
|
] as $fixture => $callable) {
|
|
|
|
|
if (!is_callable($callable)) {
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
'Missing aggregate pointer recovery fixture: ' . $fixture
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$parkOtherParents = static function (int $parentId): void {
|
|
|
|
|
$future = time() + 3600;
|
|
|
|
|
Capsule::table('cloud_jobs')
|
|
|
|
|
->whereNull('parent_job_id')
|
|
|
|
|
->where('id', '<>', $parentId)
|
|
|
|
|
->update([
|
|
|
|
|
'available_at' => $future,
|
|
|
|
|
'lease_until' => $future,
|
|
|
|
|
]);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$runLegacySweep = static function (int $parentId) use (
|
|
|
|
|
$parkOtherParents,
|
|
|
|
|
$makeLegacyEligible,
|
|
|
|
|
$assert
|
|
|
|
|
): void {
|
|
|
|
|
$parkOtherParents($parentId);
|
|
|
|
|
$makeLegacyEligible($parentId);
|
|
|
|
|
$assert(
|
|
|
|
|
CloudJobService::reconcileOneLegacyAggregateEffect(),
|
|
|
|
|
'The catalog pointer recovery sweep did not claim its exact parent.'
|
|
|
|
|
);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
$runPublishedSweep = static function (int $parentId) use (
|
|
|
|
|
$parkOtherParents,
|
|
|
|
|
$assert
|
|
|
|
|
): void {
|
|
|
|
|
$parkOtherParents($parentId);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$assert(
|
|
|
|
|
CloudJobService::reconcileOneLegacyAggregateEffect(),
|
|
|
|
|
'The published catalog pointer anomaly stayed outside the local sweep.'
|
|
|
|
|
);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// A NULL template pointer is repairable only because the immutable child and
|
|
|
|
|
// replica ledgers name the same host, operation and resource exactly once.
|
|
|
|
|
$templateParentId = $insertParent('template_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$templateParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$templateId = $insertProjectionForParent($templateParentId);
|
|
|
|
|
Capsule::table('cloud_templates')->where('id', $templateId)->update([
|
|
|
|
|
'state' => 'importing',
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
]);
|
|
|
|
|
$runLegacySweep($templateParentId);
|
|
|
|
|
$templateParent = CloudJob::findOrFail($templateParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $templateId)
|
|
|
|
|
->value('active_job_id') === $templateParentId
|
|
|
|
|
&& (string) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $templateId)
|
|
|
|
|
->value('state') === 'active'
|
|
|
|
|
&& (string) $templateParent->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $templateParent->finalization_incident,
|
|
|
|
|
'A uniquely proven NULL template active_job_id did not recover.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// The same proof must recover an integer-zero ISO delete pointer without
|
|
|
|
|
// replaying the already completed Agent deletion.
|
|
|
|
|
$isoParentId = $insertParent('user_iso_delete');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$isoParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'user_iso_delete_replica'
|
|
|
|
|
);
|
|
|
|
|
$isoId = $insertProjectionForParent($isoParentId);
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $isoId)->update([
|
|
|
|
|
'state' => 'deleting',
|
|
|
|
|
'active_job_id' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$runLegacySweep($isoParentId);
|
|
|
|
|
$isoParent = CloudJob::findOrFail($isoParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $isoId)
|
|
|
|
|
->value('active_job_id') === $isoParentId
|
|
|
|
|
&& (string) Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $isoId)
|
|
|
|
|
->value('state') === 'retired'
|
|
|
|
|
&& !Capsule::table('cloud_user_iso_replicas')
|
|
|
|
|
->where('user_iso_id', $isoId)
|
|
|
|
|
->exists()
|
|
|
|
|
&& (string) $isoParent->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $isoParent->finalization_incident,
|
|
|
|
|
'A uniquely proven zero ISO active_job_id did not recover locally.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Deadline/finalizer failure states are application-reachable before the
|
|
|
|
|
// parent-first sweep runs. Clearing the pointer in that window must not strand
|
|
|
|
|
// the resource outside recovery.
|
|
|
|
|
$errorTemplateParentId = $insertParent('template_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$errorTemplateParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$errorTemplateId = $insertProjectionForParent($errorTemplateParentId);
|
|
|
|
|
Capsule::table('cloud_templates')->where('id', $errorTemplateId)->update([
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
]);
|
|
|
|
|
$runLegacySweep($errorTemplateParentId);
|
|
|
|
|
$errorTemplateParent = CloudJob::findOrFail($errorTemplateParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $errorTemplateId)
|
|
|
|
|
->value('active_job_id') === $errorTemplateParentId
|
|
|
|
|
&& (string) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $errorTemplateId)->value('state') === 'active'
|
|
|
|
|
&& (string) $errorTemplateParent->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $errorTemplateParent->finalization_incident,
|
|
|
|
|
'A NULL pointer in template import error state was not recovered.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$failedTemplateDeleteParentId = $insertParent('template_delete');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$failedTemplateDeleteParentId,
|
|
|
|
|
'not_applied',
|
|
|
|
|
'template_delete_replica'
|
|
|
|
|
);
|
|
|
|
|
$failedTemplateDeleteId = $insertProjectionForParent(
|
|
|
|
|
$failedTemplateDeleteParentId
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $failedTemplateDeleteId)
|
|
|
|
|
->update(['active_job_id' => 0]);
|
|
|
|
|
$runLegacySweep($failedTemplateDeleteParentId);
|
|
|
|
|
$failedTemplateDeleteParent = CloudJob::findOrFail(
|
|
|
|
|
$failedTemplateDeleteParentId
|
|
|
|
|
);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $failedTemplateDeleteId)
|
|
|
|
|
->value('active_job_id') === $failedTemplateDeleteParentId
|
|
|
|
|
&& (string) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $failedTemplateDeleteId)->value('state') === 'active'
|
|
|
|
|
&& (string) Capsule::table('cloud_template_replicas')
|
|
|
|
|
->where('template_id', $failedTemplateDeleteId)
|
|
|
|
|
->value('state') === 'active'
|
|
|
|
|
&& (string) $failedTemplateDeleteParent->effect_resolution
|
|
|
|
|
=== 'not_applied'
|
|
|
|
|
&& !(bool) $failedTemplateDeleteParent->finalization_incident,
|
|
|
|
|
'A zero pointer in template delete_failed state was not recovered.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$failedIsoImportParentId = $insertParent('user_iso_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$failedIsoImportParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'user_iso_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$failedIsoImportId = $insertProjectionForParent($failedIsoImportParentId);
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $failedIsoImportId)->update([
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
]);
|
|
|
|
|
$runLegacySweep($failedIsoImportParentId);
|
|
|
|
|
$failedIsoImportParent = CloudJob::findOrFail($failedIsoImportParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $failedIsoImportId)
|
|
|
|
|
->value('active_job_id') === $failedIsoImportParentId
|
|
|
|
|
&& (string) Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $failedIsoImportId)->value('state') === 'active'
|
|
|
|
|
&& (string) $failedIsoImportParent->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $failedIsoImportParent->finalization_incident,
|
|
|
|
|
'A NULL pointer in ISO import_failed state was not recovered.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$failedIsoDeleteParentId = $insertParent('user_iso_delete');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$failedIsoDeleteParentId,
|
|
|
|
|
'not_applied',
|
|
|
|
|
'user_iso_delete_replica'
|
|
|
|
|
);
|
|
|
|
|
$failedIsoDeleteId = $insertProjectionForParent($failedIsoDeleteParentId);
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $failedIsoDeleteId)->update([
|
|
|
|
|
'active_job_id' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$runLegacySweep($failedIsoDeleteParentId);
|
|
|
|
|
$failedIsoDeleteParent = CloudJob::findOrFail($failedIsoDeleteParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $failedIsoDeleteId)
|
|
|
|
|
->value('active_job_id') === $failedIsoDeleteParentId
|
|
|
|
|
&& (string) Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $failedIsoDeleteId)->value('state') === 'active'
|
|
|
|
|
&& (string) Capsule::table('cloud_user_iso_replicas')
|
|
|
|
|
->where('user_iso_id', $failedIsoDeleteId)
|
|
|
|
|
->value('state') === 'active'
|
|
|
|
|
&& (string) $failedIsoDeleteParent->effect_resolution === 'not_applied'
|
|
|
|
|
&& !(bool) $failedIsoDeleteParent->finalization_incident,
|
|
|
|
|
'A zero pointer in ISO delete_failed state was not recovered.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Two resources claiming the same immutable child are deliberately not
|
|
|
|
|
// guessed. The parent remains fenced and the task center gets an identity code.
|
|
|
|
|
$ambiguousParentId = $insertParent('template_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$ambiguousParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$firstTemplateId = $insertProjectionForParent($ambiguousParentId);
|
|
|
|
|
$secondTemplateId = $firstTemplateId . '-duplicate';
|
|
|
|
|
$firstTemplate = (array) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $firstTemplateId)
|
|
|
|
|
->first();
|
|
|
|
|
$firstTemplate['id'] = $secondTemplateId;
|
|
|
|
|
$firstTemplate['state'] = 'importing';
|
|
|
|
|
$firstTemplate['active_job_id'] = null;
|
|
|
|
|
Capsule::table('cloud_templates')->insert($firstTemplate);
|
|
|
|
|
$firstReplica = (array) Capsule::table('cloud_template_replicas')
|
|
|
|
|
->where('template_id', $firstTemplateId)
|
|
|
|
|
->first();
|
|
|
|
|
$firstReplica['template_id'] = $secondTemplateId;
|
|
|
|
|
Capsule::table('cloud_template_replicas')->insert($firstReplica);
|
|
|
|
|
Capsule::table('cloud_templates')->where('id', $firstTemplateId)->update([
|
|
|
|
|
'state' => 'importing',
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
]);
|
|
|
|
|
$runLegacySweep($ambiguousParentId);
|
|
|
|
|
$ambiguousParent = CloudJob::findOrFail($ambiguousParentId);
|
|
|
|
|
$ambiguousRecovery = CloudJobService::recoveryPresentation(
|
|
|
|
|
$ambiguousParent,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$assert(
|
|
|
|
|
(string) $ambiguousParent->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $ambiguousParent->finalization_incident
|
|
|
|
|
&& (string) $ambiguousParent->finalization_error_code
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& ($ambiguousRecovery['code'] ?? null)
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& Capsule::table('cloud_templates')
|
|
|
|
|
->whereIn('id', [$firstTemplateId, $secondTemplateId])
|
|
|
|
|
->whereNotNull('active_job_id')
|
|
|
|
|
->count() === 0,
|
|
|
|
|
'An ambiguous template pointer was guessed or not shown as an identity conflict.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Already-published parents normally avoid the legacy sweep. A lost pointer
|
|
|
|
|
// plus an in-flight catalog state is an explicit exception: repair the unique
|
|
|
|
|
// pointer, then keep the contradictory projection fenced and visible.
|
|
|
|
|
$publishedTemplateParentId = $insertParent('template_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$publishedTemplateParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$publishedTemplateId = $insertProjectionForParent(
|
|
|
|
|
$publishedTemplateParentId
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $publishedTemplateId)
|
|
|
|
|
->update(['state' => 'importing', 'active_job_id' => null]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $publishedTemplateParentId)->update([
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'effect_resolved_at' => time(),
|
|
|
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$runPublishedSweep($publishedTemplateParentId);
|
|
|
|
|
$publishedTemplateParent = CloudJob::findOrFail(
|
|
|
|
|
$publishedTemplateParentId
|
|
|
|
|
);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $publishedTemplateId)
|
|
|
|
|
->value('active_job_id') === $publishedTemplateParentId
|
|
|
|
|
&& (string) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $publishedTemplateId)
|
|
|
|
|
->value('state') === 'importing'
|
|
|
|
|
&& (bool) $publishedTemplateParent->finalization_incident
|
|
|
|
|
&& (string) $publishedTemplateParent->finalization_error_code
|
|
|
|
|
=== 'recovery_instance_state_conflict'
|
|
|
|
|
&& (CloudJobService::recoveryPresentation(
|
|
|
|
|
$publishedTemplateParent,
|
|
|
|
|
true
|
|
|
|
|
)['code'] ?? null) === 'recovery_instance_state_conflict',
|
|
|
|
|
'A published template with a lost pointer stayed invisible or was falsely unlocked.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$publishedIsoParentId = $insertParent('user_iso_delete');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$publishedIsoParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'user_iso_delete_replica'
|
|
|
|
|
);
|
|
|
|
|
$publishedIsoId = $insertProjectionForParent($publishedIsoParentId);
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $publishedIsoId)->update([
|
|
|
|
|
'state' => 'deleting',
|
|
|
|
|
'active_job_id' => 0,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $publishedIsoParentId)->update([
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'effect_resolved_at' => time(),
|
|
|
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
]);
|
|
|
|
|
$runPublishedSweep($publishedIsoParentId);
|
|
|
|
|
$publishedIsoParent = CloudJob::findOrFail($publishedIsoParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $publishedIsoId)
|
|
|
|
|
->value('active_job_id') === $publishedIsoParentId
|
|
|
|
|
&& (string) Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $publishedIsoId)
|
|
|
|
|
->value('state') === 'deleting'
|
|
|
|
|
&& (bool) $publishedIsoParent->finalization_incident
|
|
|
|
|
&& (string) $publishedIsoParent->finalization_error_code
|
|
|
|
|
=== 'recovery_instance_state_conflict'
|
|
|
|
|
&& (CloudJobService::recoveryPresentation(
|
|
|
|
|
$publishedIsoParent,
|
|
|
|
|
true
|
|
|
|
|
)['code'] ?? null) === 'recovery_instance_state_conflict',
|
|
|
|
|
'A published ISO with a lost pointer stayed invisible or was falsely unlocked.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Metadata sync keeps the main template state active, so recovery needs an
|
|
|
|
|
// additional in-flight proof. A failed/unknown parent plus the exact current
|
|
|
|
|
// child/replica identity is sufficient and must still fence new mutations.
|
|
|
|
|
$metadataParentId = $insertParent('template_metadata_sync');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$metadataParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_metadata_replica'
|
|
|
|
|
);
|
|
|
|
|
$metadataTemplateId = $insertProjectionForParent($metadataParentId);
|
|
|
|
|
Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $metadataTemplateId)
|
|
|
|
|
->update(['active_job_id' => null]);
|
|
|
|
|
$metadataBoundary = new ReflectionMethod(
|
|
|
|
|
\App\Services\Cloud\CloudTemplateCatalogService::class,
|
|
|
|
|
'activeJobSafelySettled'
|
|
|
|
|
);
|
|
|
|
|
$assert(
|
|
|
|
|
$metadataBoundary->invoke(
|
|
|
|
|
null,
|
|
|
|
|
\App\Models\CloudTemplate::findOrFail($metadataTemplateId),
|
|
|
|
|
true
|
|
|
|
|
) === false,
|
|
|
|
|
'A missing metadata pointer allowed a new overlapping template mutation.'
|
|
|
|
|
);
|
|
|
|
|
$assert(
|
|
|
|
|
Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $metadataTemplateId)
|
|
|
|
|
->whereNull('active_job_id')
|
|
|
|
|
->exists(),
|
|
|
|
|
'The mutation boundary repaired the pointer in reverse lock order.'
|
|
|
|
|
);
|
|
|
|
|
$runLegacySweep($metadataParentId);
|
|
|
|
|
$metadataParent = CloudJob::findOrFail($metadataParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $metadataTemplateId)
|
|
|
|
|
->value('active_job_id') === $metadataParentId
|
|
|
|
|
&& (string) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $metadataTemplateId)
|
|
|
|
|
->value('state') === 'active'
|
|
|
|
|
&& (string) $metadataParent->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $metadataParent->finalization_incident,
|
|
|
|
|
'A uniquely proven metadata-sync pointer did not recover.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// A healthy, already-published metadata history with only active replicas is
|
|
|
|
|
// not evidence of an in-flight operation and must never be rebound by guess.
|
|
|
|
|
$healthyMetadataParentId = $insertParent('template_metadata_sync');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$healthyMetadataParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_metadata_replica'
|
|
|
|
|
);
|
|
|
|
|
$healthyMetadataTemplateId = $insertProjectionForParent(
|
|
|
|
|
$healthyMetadataParentId
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $healthyMetadataTemplateId)
|
|
|
|
|
->update(['active_job_id' => null]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $healthyMetadataParentId)->update([
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'effect_resolved_at' => time(),
|
|
|
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$parkOtherParents($healthyMetadataParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
!CloudJobService::reconcileOneLegacyAggregateEffect(),
|
|
|
|
|
'A healthy metadata history was incorrectly selected for pointer repair.'
|
|
|
|
|
);
|
|
|
|
|
$healthyRepairRejected = false;
|
|
|
|
|
try {
|
|
|
|
|
\App\Services\Cloud\CloudTemplateCatalogService::
|
|
|
|
|
repairMissingParentBindingLocked(
|
|
|
|
|
CloudJob::findOrFail($healthyMetadataParentId)
|
|
|
|
|
);
|
|
|
|
|
} catch (\App\Services\Cloud\CloudRecoveryInstanceStateConflict $error) {
|
|
|
|
|
$healthyRepairRejected = true;
|
|
|
|
|
}
|
|
|
|
|
$assert(
|
|
|
|
|
$healthyRepairRejected
|
|
|
|
|
&& Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $healthyMetadataTemplateId)
|
|
|
|
|
->whereNull('active_job_id')
|
|
|
|
|
->exists(),
|
|
|
|
|
'A healthy metadata history was rebound without in-flight evidence.'
|
|
|
|
|
);
|
|
|
|
|
$assert(
|
|
|
|
|
$metadataBoundary->invoke(
|
|
|
|
|
null,
|
|
|
|
|
\App\Models\CloudTemplate::findOrFail($healthyMetadataTemplateId),
|
|
|
|
|
true
|
|
|
|
|
) === true,
|
|
|
|
|
'A healthy, fully settled metadata history permanently blocked mutations.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// The same pre-mutation fence applies to an ISO whose visible state was
|
|
|
|
|
// already changed back to active while its uniquely linked parent is still
|
|
|
|
|
// unresolved. This is the exact window before the parent-first sweep runs.
|
|
|
|
|
$isoBoundaryParentId = $insertParent('user_iso_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$isoBoundaryParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'user_iso_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$isoBoundaryId = $insertProjectionForParent($isoBoundaryParentId);
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $isoBoundaryId)->update([
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
]);
|
|
|
|
|
$isoBoundary = new ReflectionMethod(
|
|
|
|
|
\App\Services\Cloud\CloudUserIsoService::class,
|
|
|
|
|
'activeJobSafelySettled'
|
|
|
|
|
);
|
|
|
|
|
$assert(
|
|
|
|
|
$isoBoundary->invoke(
|
|
|
|
|
null,
|
|
|
|
|
\App\Models\CloudUserIso::findOrFail($isoBoundaryId),
|
|
|
|
|
true
|
|
|
|
|
) === false
|
|
|
|
|
&& Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $isoBoundaryId)
|
|
|
|
|
->whereNull('active_job_id')
|
|
|
|
|
->exists(),
|
|
|
|
|
'A lost ISO pointer allowed a second mutation before recovery.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Even with a failed/unknown parent, two current metadata resources remain an
|
|
|
|
|
// identity conflict. Neither pointer may be written.
|
|
|
|
|
$ambiguousMetadataParentId = $insertParent('template_metadata_sync');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$ambiguousMetadataParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_metadata_replica'
|
|
|
|
|
);
|
|
|
|
|
$firstMetadataId = $insertProjectionForParent($ambiguousMetadataParentId);
|
|
|
|
|
$secondMetadataId = $firstMetadataId . '-duplicate';
|
|
|
|
|
$firstMetadata = (array) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $firstMetadataId)
|
|
|
|
|
->first();
|
|
|
|
|
$firstMetadata['id'] = $secondMetadataId;
|
|
|
|
|
$firstMetadata['active_job_id'] = null;
|
|
|
|
|
Capsule::table('cloud_templates')->insert($firstMetadata);
|
|
|
|
|
$firstMetadataReplica = (array) Capsule::table('cloud_template_replicas')
|
|
|
|
|
->where('template_id', $firstMetadataId)
|
|
|
|
|
->first();
|
|
|
|
|
$firstMetadataReplica['template_id'] = $secondMetadataId;
|
|
|
|
|
Capsule::table('cloud_template_replicas')->insert($firstMetadataReplica);
|
|
|
|
|
Capsule::table('cloud_templates')->where('id', $firstMetadataId)->update([
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
]);
|
|
|
|
|
$runLegacySweep($ambiguousMetadataParentId);
|
|
|
|
|
$ambiguousMetadataParent = CloudJob::findOrFail(
|
|
|
|
|
$ambiguousMetadataParentId
|
|
|
|
|
);
|
|
|
|
|
$assert(
|
|
|
|
|
(string) $ambiguousMetadataParent->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $ambiguousMetadataParent->finalization_incident
|
|
|
|
|
&& (string) $ambiguousMetadataParent->finalization_error_code
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& Capsule::table('cloud_templates')
|
|
|
|
|
->whereIn('id', [$firstMetadataId, $secondMetadataId])
|
|
|
|
|
->whereNotNull('active_job_id')
|
|
|
|
|
->count() === 0,
|
|
|
|
|
'An ambiguous metadata-sync pointer was guessed or falsely released.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$publishParent = static function (int $parentId): void {
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
|
|
|
|
|
'outcome' => 'succeeded',
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'effect_resolved_at' => time(),
|
|
|
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
|
|
|
'error_code' => null,
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// A quota-rejected import is a legitimate published failed/applied projection:
|
|
|
|
|
// its active replicas remain for explicit cleanup. A lost pointer must be
|
|
|
|
|
// repaired without turning that state into a generic incident.
|
|
|
|
|
$quotaIsoParentId = $insertParent('user_iso_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$quotaIsoParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'user_iso_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$quotaIsoId = $insertProjectionForParent($quotaIsoParentId);
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $quotaIsoId)->update([
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $quotaIsoParentId)->update([
|
|
|
|
|
'outcome' => 'failed',
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => 1,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'applied',
|
|
|
|
|
'effect_resolved_at' => time(),
|
|
|
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
|
|
|
'error_code' => 'user_iso_quota_exceeded',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$runPublishedSweep($quotaIsoParentId);
|
|
|
|
|
$quotaIsoParent = CloudJob::findOrFail($quotaIsoParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_user_isos')->where('id', $quotaIsoId)
|
|
|
|
|
->value('active_job_id') === $quotaIsoParentId
|
|
|
|
|
&& (string) Capsule::table('cloud_user_isos')->where('id', $quotaIsoId)
|
|
|
|
|
->value('state') === 'import_failed'
|
|
|
|
|
&& (string) $quotaIsoParent->outcome === 'failed'
|
|
|
|
|
&& (string) $quotaIsoParent->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $quotaIsoParent->finalization_incident
|
|
|
|
|
&& $isoBoundary->invoke(
|
|
|
|
|
null,
|
|
|
|
|
\App\Models\CloudUserIso::findOrFail($quotaIsoId),
|
|
|
|
|
false
|
|
|
|
|
) === true,
|
|
|
|
|
'A settled quota cleanup projection was not uniquely rebound.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// A published not-applied delete may legitimately retain an older stale
|
|
|
|
|
// replica. It is already safe before pointer repair and remains strictly
|
|
|
|
|
// verifiable after the parent-first sweep restores the pointer.
|
|
|
|
|
$staleDeleteParentId = $insertParent('template_delete');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$staleDeleteParentId,
|
|
|
|
|
'not_applied',
|
|
|
|
|
'template_delete_replica'
|
|
|
|
|
);
|
|
|
|
|
$staleDeleteTemplateId = $insertProjectionForParent($staleDeleteParentId);
|
|
|
|
|
Capsule::table('cloud_templates')->where('id', $staleDeleteTemplateId)->update([
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_template_replicas')
|
|
|
|
|
->where('template_id', $staleDeleteTemplateId)->update([
|
|
|
|
|
'state' => 'stale',
|
|
|
|
|
'revision' => 0,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $staleDeleteParentId)->update([
|
|
|
|
|
'outcome' => 'failed',
|
|
|
|
|
'status' => 'failed',
|
|
|
|
|
'effect_may_have_applied' => 0,
|
|
|
|
|
'safe_to_resubmit' => 1,
|
|
|
|
|
'effect_resolution' => 'not_applied',
|
|
|
|
|
'effect_resolved_at' => time(),
|
|
|
|
|
'effect_resolution_reason' => 'normal_completion',
|
|
|
|
|
'error_code' => 'template_delete_not_applied',
|
|
|
|
|
'finalization_incident' => 0,
|
|
|
|
|
'finalization_error_code' => null,
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$assert(
|
|
|
|
|
$metadataBoundary->invoke(
|
|
|
|
|
null,
|
|
|
|
|
\App\Models\CloudTemplate::findOrFail($staleDeleteTemplateId),
|
|
|
|
|
false
|
|
|
|
|
) === true,
|
|
|
|
|
'A valid not-applied delete with a stale replica stayed falsely fenced.'
|
|
|
|
|
);
|
|
|
|
|
$runPublishedSweep($staleDeleteParentId);
|
|
|
|
|
$staleDeleteParent = CloudJob::findOrFail($staleDeleteParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $staleDeleteTemplateId)
|
|
|
|
|
->value('active_job_id') === $staleDeleteParentId
|
|
|
|
|
&& (string) Capsule::table('cloud_template_replicas')
|
|
|
|
|
->where('template_id', $staleDeleteTemplateId)
|
|
|
|
|
->value('state') === 'stale'
|
|
|
|
|
&& !(bool) $staleDeleteParent->finalization_incident,
|
|
|
|
|
'A valid not-applied delete pointer was not repaired safely.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Published healthy resources with a positive wrong pointer remain eligible
|
|
|
|
|
// for diagnosis, but the positive value is never overwritten automatically.
|
|
|
|
|
$publishedWrongTemplateParentId = $insertParent('template_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$publishedWrongTemplateParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$publishedWrongTemplateId = $insertProjectionForParent(
|
|
|
|
|
$publishedWrongTemplateParentId
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $publishedWrongTemplateId)->update([
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'active_job_id' => 999999991,
|
|
|
|
|
]);
|
|
|
|
|
$publishParent($publishedWrongTemplateParentId);
|
|
|
|
|
$runPublishedSweep($publishedWrongTemplateParentId);
|
|
|
|
|
$publishedWrongTemplateParent = CloudJob::findOrFail(
|
|
|
|
|
$publishedWrongTemplateParentId
|
|
|
|
|
);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $publishedWrongTemplateId)
|
|
|
|
|
->value('active_job_id') === 999999991
|
|
|
|
|
&& (string) $publishedWrongTemplateParent->finalization_error_code
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& (int) $publishedWrongTemplateParent->available_at >= time() + 290,
|
|
|
|
|
'A published template positive pointer was overwritten or not queued.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$publishedWrongIsoParentId = $insertParent('user_iso_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$publishedWrongIsoParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'user_iso_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$publishedWrongIsoId = $insertProjectionForParent($publishedWrongIsoParentId);
|
|
|
|
|
Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $publishedWrongIsoId)->update([
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'active_job_id' => 999999992,
|
|
|
|
|
]);
|
|
|
|
|
$publishParent($publishedWrongIsoParentId);
|
|
|
|
|
$runPublishedSweep($publishedWrongIsoParentId);
|
|
|
|
|
$publishedWrongIsoParent = CloudJob::findOrFail($publishedWrongIsoParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $publishedWrongIsoId)
|
|
|
|
|
->value('active_job_id') === 999999992
|
|
|
|
|
&& (string) $publishedWrongIsoParent->finalization_error_code
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& (int) $publishedWrongIsoParent->available_at >= time() + 290,
|
|
|
|
|
'A published ISO positive pointer was overwritten or not queued.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Full content is part of the binding proof. A NULL pointer plus a matching
|
|
|
|
|
// identity chain must stay fenced and visible when bytes or hashes diverge.
|
|
|
|
|
$corruptTemplateParentId = $insertParent('template_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$corruptTemplateParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$corruptTemplateId = $insertProjectionForParent($corruptTemplateParentId);
|
|
|
|
|
Capsule::table('cloud_templates')->where('id', $corruptTemplateId)->update([
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_template_replicas')
|
|
|
|
|
->where('template_id', $corruptTemplateId)->update([
|
|
|
|
|
'sha256' => str_repeat('f', 64),
|
|
|
|
|
]);
|
|
|
|
|
$publishParent($corruptTemplateParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
$metadataBoundary->invoke(
|
|
|
|
|
null,
|
|
|
|
|
\App\Models\CloudTemplate::findOrFail($corruptTemplateId),
|
|
|
|
|
false
|
|
|
|
|
) === false,
|
|
|
|
|
'A NULL template pointer with corrupt content falsely unlocked mutation.'
|
|
|
|
|
);
|
|
|
|
|
$runPublishedSweep($corruptTemplateParentId);
|
|
|
|
|
$corruptTemplateParent = CloudJob::findOrFail($corruptTemplateParentId);
|
|
|
|
|
$corruptTemplatePointer = Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $corruptTemplateId)->value('active_job_id');
|
|
|
|
|
$assert(
|
|
|
|
|
(int) $corruptTemplatePointer === $corruptTemplateParentId
|
|
|
|
|
&& (string) $corruptTemplateParent->finalization_error_code
|
|
|
|
|
=== 'recovery_instance_state_conflict',
|
|
|
|
|
'Corrupt template content was unlocked or dropped from recovery: pointer='
|
|
|
|
|
. var_export($corruptTemplatePointer, true) . ', code='
|
|
|
|
|
. (string) $corruptTemplateParent->finalization_error_code
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$corruptIsoParentId = $insertParent('user_iso_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$corruptIsoParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'user_iso_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$corruptIsoId = $insertProjectionForParent($corruptIsoParentId);
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $corruptIsoId)->update([
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'active_job_id' => 0,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_user_iso_replicas')
|
|
|
|
|
->where('user_iso_id', $corruptIsoId)->update([
|
|
|
|
|
'size_bytes' => 8192,
|
|
|
|
|
]);
|
|
|
|
|
$publishParent($corruptIsoParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
$isoBoundary->invoke(
|
|
|
|
|
null,
|
|
|
|
|
\App\Models\CloudUserIso::findOrFail($corruptIsoId),
|
|
|
|
|
false
|
|
|
|
|
) === false,
|
|
|
|
|
'A zero ISO pointer with corrupt content falsely unlocked mutation.'
|
|
|
|
|
);
|
|
|
|
|
$runPublishedSweep($corruptIsoParentId);
|
|
|
|
|
$corruptIsoParent = CloudJob::findOrFail($corruptIsoParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_user_isos')->where('id', $corruptIsoId)
|
|
|
|
|
->value('active_job_id') === $corruptIsoParentId
|
|
|
|
|
&& (string) $corruptIsoParent->finalization_error_code
|
|
|
|
|
=== 'recovery_instance_state_conflict',
|
|
|
|
|
'Corrupt ISO content was unlocked or dropped from recovery.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// The same child id cannot belong to both catalog domains. Cross-table reuse
|
|
|
|
|
// is an identity incident and may not authorize a NULL-pointer repair.
|
|
|
|
|
$crossTableTemplateParentId = $insertParent('template_import');
|
|
|
|
|
$crossTableTemplateChildId = $insertResolvedChild(
|
|
|
|
|
$crossTableTemplateParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$crossTableTemplateId = $insertProjectionForParent(
|
|
|
|
|
$crossTableTemplateParentId
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $crossTableTemplateId)
|
|
|
|
|
->update(['active_job_id' => null]);
|
|
|
|
|
$crossTableTemplateChild = CloudJob::findOrFail($crossTableTemplateChildId);
|
|
|
|
|
Capsule::table('cloud_user_iso_replicas')->insert([
|
|
|
|
|
'host_id' => 800000 + $crossTableTemplateChildId,
|
|
|
|
|
'user_iso_id' => md5('cross-table-iso:' . $crossTableTemplateChildId),
|
|
|
|
|
'user_id' => 1,
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'size_bytes' => 4096,
|
|
|
|
|
'sha256' => str_repeat('a', 64),
|
|
|
|
|
'operation_id' => (string) $crossTableTemplateChild->operation_id,
|
|
|
|
|
'job_id' => $crossTableTemplateChildId,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'created_at' => time(),
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
]);
|
|
|
|
|
$runLegacySweep($crossTableTemplateParentId);
|
|
|
|
|
$crossTableTemplateParent = CloudJob::findOrFail(
|
|
|
|
|
$crossTableTemplateParentId
|
|
|
|
|
);
|
|
|
|
|
$assert(
|
|
|
|
|
Capsule::table('cloud_templates')->where('id', $crossTableTemplateId)
|
|
|
|
|
->whereNull('active_job_id')->exists()
|
|
|
|
|
&& (string) $crossTableTemplateParent->finalization_error_code
|
|
|
|
|
=== 'recovery_operation_identity_conflict',
|
|
|
|
|
'A template child duplicated in the ISO ledger repaired its pointer.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$crossTableIsoParentId = $insertParent('user_iso_import');
|
|
|
|
|
$crossTableIsoChildId = $insertResolvedChild(
|
|
|
|
|
$crossTableIsoParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'user_iso_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$crossTableIsoId = $insertProjectionForParent($crossTableIsoParentId);
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $crossTableIsoId)->update([
|
|
|
|
|
'active_job_id' => null,
|
|
|
|
|
]);
|
|
|
|
|
$crossTableIsoChild = CloudJob::findOrFail($crossTableIsoChildId);
|
|
|
|
|
Capsule::table('cloud_template_replicas')->insert([
|
|
|
|
|
'host_id' => 900000 + $crossTableIsoChildId,
|
|
|
|
|
'template_id' => 'cross-table-template-' . $crossTableIsoChildId,
|
|
|
|
|
'revision' => 1,
|
|
|
|
|
'generation_token' => md5(
|
|
|
|
|
'cross-table-template:' . $crossTableIsoChildId
|
|
|
|
|
),
|
|
|
|
|
'state' => 'active',
|
|
|
|
|
'image_bytes' => 4096,
|
|
|
|
|
'sha256' => str_repeat('b', 64),
|
|
|
|
|
'operation_id' => (string) $crossTableIsoChild->operation_id,
|
|
|
|
|
'job_id' => $crossTableIsoChildId,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'created_at' => time(),
|
|
|
|
|
'updated_at' => time(),
|
|
|
|
|
]);
|
|
|
|
|
$runLegacySweep($crossTableIsoParentId);
|
|
|
|
|
$crossTableIsoParent = CloudJob::findOrFail($crossTableIsoParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $crossTableIsoId)
|
|
|
|
|
->whereNull('active_job_id')->exists()
|
|
|
|
|
&& (string) $crossTableIsoParent->finalization_error_code
|
|
|
|
|
=== 'recovery_operation_identity_conflict',
|
|
|
|
|
'An ISO child duplicated in the template ledger repaired its pointer.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// A unique positive pointer is not sufficient: swapping two otherwise valid
|
|
|
|
|
// parents must fail closed when their children and replicas belong elsewhere.
|
|
|
|
|
$crossTemplateParentA = $insertParent('template_metadata_sync');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$crossTemplateParentA,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_metadata_replica'
|
|
|
|
|
);
|
|
|
|
|
$crossTemplateA = $insertProjectionForParent($crossTemplateParentA);
|
|
|
|
|
$publishParent($crossTemplateParentA);
|
|
|
|
|
$crossTemplateParentB = $insertParent('template_metadata_sync');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$crossTemplateParentB,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_metadata_replica'
|
|
|
|
|
);
|
|
|
|
|
$crossTemplateB = $insertProjectionForParent($crossTemplateParentB);
|
|
|
|
|
$publishParent($crossTemplateParentB);
|
|
|
|
|
Capsule::table('cloud_templates')->where('id', $crossTemplateA)->update([
|
|
|
|
|
'active_job_id' => $crossTemplateParentB,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_templates')->where('id', $crossTemplateB)->update([
|
|
|
|
|
'active_job_id' => $crossTemplateParentA,
|
|
|
|
|
]);
|
|
|
|
|
$assert(
|
|
|
|
|
$metadataBoundary->invoke(
|
|
|
|
|
null,
|
|
|
|
|
\App\Models\CloudTemplate::findOrFail($crossTemplateA),
|
|
|
|
|
false
|
|
|
|
|
) === false
|
|
|
|
|
&& $metadataBoundary->invoke(
|
|
|
|
|
null,
|
|
|
|
|
\App\Models\CloudTemplate::findOrFail($crossTemplateB),
|
|
|
|
|
false
|
|
|
|
|
) === false,
|
|
|
|
|
'Cross-wired template parents falsely unlocked both resources.'
|
|
|
|
|
);
|
|
|
|
|
$runPublishedSweep($crossTemplateParentA);
|
|
|
|
|
$runPublishedSweep($crossTemplateParentB);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_templates')->where('id', $crossTemplateA)
|
|
|
|
|
->value('active_job_id') === $crossTemplateParentB
|
|
|
|
|
&& (int) Capsule::table('cloud_templates')->where('id', $crossTemplateB)
|
|
|
|
|
->value('active_job_id') === $crossTemplateParentA
|
|
|
|
|
&& (string) CloudJob::findOrFail($crossTemplateParentA)
|
|
|
|
|
->finalization_error_code === 'recovery_operation_identity_conflict'
|
|
|
|
|
&& (string) CloudJob::findOrFail($crossTemplateParentB)
|
|
|
|
|
->finalization_error_code === 'recovery_operation_identity_conflict',
|
|
|
|
|
'Cross-wired template pointers were rewritten or removed from recovery.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$crossIsoParentA = $insertParent('user_iso_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$crossIsoParentA,
|
|
|
|
|
'applied',
|
|
|
|
|
'user_iso_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$crossIsoA = $insertProjectionForParent($crossIsoParentA);
|
|
|
|
|
$publishParent($crossIsoParentA);
|
|
|
|
|
$crossIsoParentB = $insertParent('user_iso_import');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$crossIsoParentB,
|
|
|
|
|
'applied',
|
|
|
|
|
'user_iso_import_replica'
|
|
|
|
|
);
|
|
|
|
|
$crossIsoB = $insertProjectionForParent($crossIsoParentB);
|
|
|
|
|
$publishParent($crossIsoParentB);
|
|
|
|
|
Capsule::table('cloud_user_isos')->whereIn(
|
|
|
|
|
'id',
|
|
|
|
|
[$crossIsoA, $crossIsoB]
|
|
|
|
|
)->update(['state' => 'active']);
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $crossIsoA)->update([
|
|
|
|
|
'active_job_id' => $crossIsoParentB,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $crossIsoB)->update([
|
|
|
|
|
'active_job_id' => $crossIsoParentA,
|
|
|
|
|
]);
|
|
|
|
|
$assert(
|
|
|
|
|
$isoBoundary->invoke(
|
|
|
|
|
null,
|
|
|
|
|
\App\Models\CloudUserIso::findOrFail($crossIsoA),
|
|
|
|
|
false
|
|
|
|
|
) === false
|
|
|
|
|
&& $isoBoundary->invoke(
|
|
|
|
|
null,
|
|
|
|
|
\App\Models\CloudUserIso::findOrFail($crossIsoB),
|
|
|
|
|
false
|
|
|
|
|
) === false,
|
|
|
|
|
'Cross-wired ISO parents falsely unlocked both resources.'
|
|
|
|
|
);
|
|
|
|
|
$runPublishedSweep($crossIsoParentA);
|
|
|
|
|
$runPublishedSweep($crossIsoParentB);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_user_isos')->where('id', $crossIsoA)
|
|
|
|
|
->value('active_job_id') === $crossIsoParentB
|
|
|
|
|
&& (int) Capsule::table('cloud_user_isos')->where('id', $crossIsoB)
|
|
|
|
|
->value('active_job_id') === $crossIsoParentA
|
|
|
|
|
&& (string) CloudJob::findOrFail($crossIsoParentA)
|
|
|
|
|
->finalization_error_code === 'recovery_operation_identity_conflict'
|
|
|
|
|
&& (string) CloudJob::findOrFail($crossIsoParentB)
|
|
|
|
|
->finalization_error_code === 'recovery_operation_identity_conflict',
|
|
|
|
|
'Cross-wired ISO pointers were rewritten or removed from recovery.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// Positive damage is never overwritten automatically. Even a unique replica
|
|
|
|
|
// chain only proves the expected parent; it cannot authorize replacing a
|
|
|
|
|
// positive pointer that may represent a separate damaged generation.
|
|
|
|
|
$danglingTemplateParentId = $insertParent('template_metadata_sync');
|
|
|
|
|
$insertResolvedChild(
|
|
|
|
|
$danglingTemplateParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'template_metadata_replica'
|
|
|
|
|
);
|
|
|
|
|
$danglingTemplateId = $insertProjectionForParent(
|
|
|
|
|
$danglingTemplateParentId
|
|
|
|
|
);
|
|
|
|
|
Capsule::table('cloud_templates')->where('id', $danglingTemplateId)->update([
|
|
|
|
|
'active_job_id' => 999999999,
|
|
|
|
|
]);
|
|
|
|
|
$runLegacySweep($danglingTemplateParentId);
|
|
|
|
|
$danglingTemplateParent = CloudJob::findOrFail($danglingTemplateParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $danglingTemplateId)
|
|
|
|
|
->value('active_job_id') === 999999999
|
|
|
|
|
&& (string) $danglingTemplateParent->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $danglingTemplateParent->finalization_incident
|
|
|
|
|
&& (string) $danglingTemplateParent->finalization_error_code
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& (int) $danglingTemplateParent->available_at >= time() + 290,
|
|
|
|
|
'A positive dangling template pointer was overwritten or left invisible.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
$wrongIsoParentId = $insertParent('user_iso_delete');
|
|
|
|
|
$wrongIsoChildId = $insertResolvedChild(
|
|
|
|
|
$wrongIsoParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
'user_iso_delete_replica'
|
|
|
|
|
);
|
|
|
|
|
$wrongIsoId = $insertProjectionForParent($wrongIsoParentId);
|
|
|
|
|
Capsule::table('cloud_user_isos')->where('id', $wrongIsoId)->update([
|
|
|
|
|
'state' => 'deleting',
|
|
|
|
|
'active_job_id' => $wrongIsoChildId,
|
|
|
|
|
]);
|
|
|
|
|
$runLegacySweep($wrongIsoParentId);
|
|
|
|
|
$wrongIsoParent = CloudJob::findOrFail($wrongIsoParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
(int) Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $wrongIsoId)
|
|
|
|
|
->value('active_job_id') === $wrongIsoChildId
|
|
|
|
|
&& (string) $wrongIsoParent->effect_resolution === 'unknown'
|
|
|
|
|
&& (bool) $wrongIsoParent->finalization_incident
|
|
|
|
|
&& (string) $wrongIsoParent->finalization_error_code
|
|
|
|
|
=== 'recovery_operation_identity_conflict'
|
|
|
|
|
&& (int) $wrongIsoParent->available_at >= time() + 290,
|
|
|
|
|
'An ISO pointer to a leaf job was overwritten or left invisible.'
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// A NULL pointer is intentional after an exactly proven mixed aggregate. The
|
|
|
|
|
// background pointer-repair sweep must not rebind that completed parent and
|
|
|
|
|
// recreate the lock before an operator starts a fresh cleanup operation.
|
|
|
|
|
foreach ([
|
|
|
|
|
'template_import' => 'template_import_replica',
|
|
|
|
|
'template_delete' => 'template_delete_replica',
|
|
|
|
|
'user_iso_import' => 'user_iso_import_replica',
|
|
|
|
|
'user_iso_delete' => 'user_iso_delete_replica',
|
|
|
|
|
] as $parentKind => $childKind) {
|
|
|
|
|
$partialParentId = $insertParent($parentKind);
|
|
|
|
|
$appliedChildId = $insertResolvedChild(
|
|
|
|
|
$partialParentId,
|
|
|
|
|
'applied',
|
|
|
|
|
$childKind
|
|
|
|
|
);
|
|
|
|
|
$insertResolvedChild($partialParentId, 'not_applied', $childKind);
|
|
|
|
|
$partialResourceId = $insertProjectionForParent($partialParentId);
|
|
|
|
|
CloudJobService::reconcileAggregateParentEffectLocked(
|
|
|
|
|
CloudJob::findOrFail($appliedChildId),
|
|
|
|
|
null
|
|
|
|
|
);
|
|
|
|
|
// Earlier corruption fixtures deliberately include aggregate-shaped rows
|
|
|
|
|
// with a non-null parent id. Park every other row so this assertion tests
|
|
|
|
|
// only whether the released partial itself remains discoverable.
|
|
|
|
|
$future = time() + 3600;
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', '<>', $partialParentId)->update([
|
|
|
|
|
'available_at' => $future,
|
|
|
|
|
'lease_until' => $future,
|
|
|
|
|
]);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $partialParentId)->update([
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$assert(
|
|
|
|
|
!CloudJobService::reconcileOneLegacyAggregateEffect(),
|
|
|
|
|
'A settled mixed aggregate re-entered pointer repair: ' . $parentKind
|
|
|
|
|
);
|
|
|
|
|
$pointer = str_starts_with($parentKind, 'template_')
|
|
|
|
|
? Capsule::table('cloud_templates')->where('id', $partialResourceId)
|
|
|
|
|
->value('active_job_id')
|
|
|
|
|
: Capsule::table('cloud_user_isos')->where('id', $partialResourceId)
|
|
|
|
|
->value('active_job_id');
|
|
|
|
|
$assert(
|
|
|
|
|
$pointer === null
|
|
|
|
|
&& !(bool) CloudJob::findOrFail($partialParentId)
|
|
|
|
|
->finalization_incident,
|
|
|
|
|
'A settled mixed aggregate was rebound or refenced: ' . $parentKind
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
// A later local projection incident must make the released parent
|
|
|
|
|
// discoverable again. Recovery may temporarily restore the pointer only
|
|
|
|
|
// to validate the immutable mixed ledger; a successful recheck releases
|
|
|
|
|
// it again instead of recreating a permanent catalog lock.
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $partialParentId)->update([
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_instance_state_conflict',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$assert(
|
|
|
|
|
CloudJobService::reconcileOneLegacyAggregateEffect(),
|
|
|
|
|
'A settled mixed aggregate incident was not rechecked: ' . $parentKind
|
|
|
|
|
);
|
|
|
|
|
$recheckedPointer = str_starts_with($parentKind, 'template_')
|
|
|
|
|
? Capsule::table('cloud_templates')->where('id', $partialResourceId)
|
|
|
|
|
->value('active_job_id')
|
|
|
|
|
: Capsule::table('cloud_user_isos')->where('id', $partialResourceId)
|
|
|
|
|
->value('active_job_id');
|
|
|
|
|
$recheckedParent = CloudJob::findOrFail($partialParentId);
|
|
|
|
|
$assert(
|
|
|
|
|
$recheckedPointer === null
|
|
|
|
|
&& !(bool) $recheckedParent->finalization_incident
|
|
|
|
|
&& $recheckedParent->finalization_error_code === null
|
|
|
|
|
&& CloudJobService::effectScope($recheckedParent) === 'partial',
|
|
|
|
|
'A valid settled mixed incident did not release cleanly: ' . $parentKind
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
echo "Cloud catalog active-job pointer recovery checks passed.\n";
|