You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/tests/CloudCatalogActiveJobPointe...

1108 lines
41 KiB
PHTML

<?php
declare(strict_types=1);
use App\Models\CloudJob;
use App\Services\Cloud\CloudJobService;
use Illuminate\Database\Capsule\Manager as Capsule;
/*
* Reuse the complete aggregate-ledger fixture. The focused checks below cover
* the extra catalog-pointer evidence path without maintaining a second schema.
*/
require __DIR__ . '/CloudAggregateRecoveryConvergenceTest.php';
foreach ([
'assert' => $assert ?? null,
'insertParent' => $insertParent ?? null,
'insertResolvedChild' => $insertResolvedChild ?? null,
'insertProjectionForParent' => $insertProjectionForParent ?? null,
'makeLegacyEligible' => $makeLegacyEligible ?? null,
] as $fixture => $callable) {
if (!is_callable($callable)) {
throw new RuntimeException(
'Missing aggregate pointer recovery fixture: ' . $fixture
);
}
}
$parkOtherParents = static function (int $parentId): void {
$future = time() + 3600;
Capsule::table('cloud_jobs')
->whereNull('parent_job_id')
->where('id', '<>', $parentId)
->update([
'available_at' => $future,
'lease_until' => $future,
]);
};
$runLegacySweep = static function (int $parentId) use (
$parkOtherParents,
$makeLegacyEligible,
$assert
): void {
$parkOtherParents($parentId);
$makeLegacyEligible($parentId);
$assert(
CloudJobService::reconcileOneLegacyAggregateEffect(),
'The catalog pointer recovery sweep did not claim its exact parent.'
);
};
$runPublishedSweep = static function (int $parentId) use (
$parkOtherParents,
$assert
): void {
$parkOtherParents($parentId);
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$assert(
CloudJobService::reconcileOneLegacyAggregateEffect(),
'The published catalog pointer anomaly stayed outside the local sweep.'
);
};
// A NULL template pointer is repairable only because the immutable child and
// replica ledgers name the same host, operation and resource exactly once.
$templateParentId = $insertParent('template_import');
$insertResolvedChild(
$templateParentId,
'applied',
'template_import_replica'
);
$templateId = $insertProjectionForParent($templateParentId);
Capsule::table('cloud_templates')->where('id', $templateId)->update([
'state' => 'importing',
'active_job_id' => null,
]);
$runLegacySweep($templateParentId);
$templateParent = CloudJob::findOrFail($templateParentId);
$assert(
(int) Capsule::table('cloud_templates')
->where('id', $templateId)
->value('active_job_id') === $templateParentId
&& (string) Capsule::table('cloud_templates')
->where('id', $templateId)
->value('state') === 'active'
&& (string) $templateParent->effect_resolution === 'applied'
&& !(bool) $templateParent->finalization_incident,
'A uniquely proven NULL template active_job_id did not recover.'
);
// The same proof must recover an integer-zero ISO delete pointer without
// replaying the already completed Agent deletion.
$isoParentId = $insertParent('user_iso_delete');
$insertResolvedChild(
$isoParentId,
'applied',
'user_iso_delete_replica'
);
$isoId = $insertProjectionForParent($isoParentId);
Capsule::table('cloud_user_isos')->where('id', $isoId)->update([
'state' => 'deleting',
'active_job_id' => 0,
]);
$runLegacySweep($isoParentId);
$isoParent = CloudJob::findOrFail($isoParentId);
$assert(
(int) Capsule::table('cloud_user_isos')
->where('id', $isoId)
->value('active_job_id') === $isoParentId
&& (string) Capsule::table('cloud_user_isos')
->where('id', $isoId)
->value('state') === 'retired'
&& !Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $isoId)
->exists()
&& (string) $isoParent->effect_resolution === 'applied'
&& !(bool) $isoParent->finalization_incident,
'A uniquely proven zero ISO active_job_id did not recover locally.'
);
// Deadline/finalizer failure states are application-reachable before the
// parent-first sweep runs. Clearing the pointer in that window must not strand
// the resource outside recovery.
$errorTemplateParentId = $insertParent('template_import');
$insertResolvedChild(
$errorTemplateParentId,
'applied',
'template_import_replica'
);
$errorTemplateId = $insertProjectionForParent($errorTemplateParentId);
Capsule::table('cloud_templates')->where('id', $errorTemplateId)->update([
'active_job_id' => null,
]);
$runLegacySweep($errorTemplateParentId);
$errorTemplateParent = CloudJob::findOrFail($errorTemplateParentId);
$assert(
(int) Capsule::table('cloud_templates')
->where('id', $errorTemplateId)
->value('active_job_id') === $errorTemplateParentId
&& (string) Capsule::table('cloud_templates')
->where('id', $errorTemplateId)->value('state') === 'active'
&& (string) $errorTemplateParent->effect_resolution === 'applied'
&& !(bool) $errorTemplateParent->finalization_incident,
'A NULL pointer in template import error state was not recovered.'
);
$failedTemplateDeleteParentId = $insertParent('template_delete');
$insertResolvedChild(
$failedTemplateDeleteParentId,
'not_applied',
'template_delete_replica'
);
$failedTemplateDeleteId = $insertProjectionForParent(
$failedTemplateDeleteParentId
);
Capsule::table('cloud_templates')
->where('id', $failedTemplateDeleteId)
->update(['active_job_id' => 0]);
$runLegacySweep($failedTemplateDeleteParentId);
$failedTemplateDeleteParent = CloudJob::findOrFail(
$failedTemplateDeleteParentId
);
$assert(
(int) Capsule::table('cloud_templates')
->where('id', $failedTemplateDeleteId)
->value('active_job_id') === $failedTemplateDeleteParentId
&& (string) Capsule::table('cloud_templates')
->where('id', $failedTemplateDeleteId)->value('state') === 'active'
&& (string) Capsule::table('cloud_template_replicas')
->where('template_id', $failedTemplateDeleteId)
->value('state') === 'active'
&& (string) $failedTemplateDeleteParent->effect_resolution
=== 'not_applied'
&& !(bool) $failedTemplateDeleteParent->finalization_incident,
'A zero pointer in template delete_failed state was not recovered.'
);
$failedIsoImportParentId = $insertParent('user_iso_import');
$insertResolvedChild(
$failedIsoImportParentId,
'applied',
'user_iso_import_replica'
);
$failedIsoImportId = $insertProjectionForParent($failedIsoImportParentId);
Capsule::table('cloud_user_isos')->where('id', $failedIsoImportId)->update([
'active_job_id' => null,
]);
$runLegacySweep($failedIsoImportParentId);
$failedIsoImportParent = CloudJob::findOrFail($failedIsoImportParentId);
$assert(
(int) Capsule::table('cloud_user_isos')
->where('id', $failedIsoImportId)
->value('active_job_id') === $failedIsoImportParentId
&& (string) Capsule::table('cloud_user_isos')
->where('id', $failedIsoImportId)->value('state') === 'active'
&& (string) $failedIsoImportParent->effect_resolution === 'applied'
&& !(bool) $failedIsoImportParent->finalization_incident,
'A NULL pointer in ISO import_failed state was not recovered.'
);
$failedIsoDeleteParentId = $insertParent('user_iso_delete');
$insertResolvedChild(
$failedIsoDeleteParentId,
'not_applied',
'user_iso_delete_replica'
);
$failedIsoDeleteId = $insertProjectionForParent($failedIsoDeleteParentId);
Capsule::table('cloud_user_isos')->where('id', $failedIsoDeleteId)->update([
'active_job_id' => 0,
]);
$runLegacySweep($failedIsoDeleteParentId);
$failedIsoDeleteParent = CloudJob::findOrFail($failedIsoDeleteParentId);
$assert(
(int) Capsule::table('cloud_user_isos')
->where('id', $failedIsoDeleteId)
->value('active_job_id') === $failedIsoDeleteParentId
&& (string) Capsule::table('cloud_user_isos')
->where('id', $failedIsoDeleteId)->value('state') === 'active'
&& (string) Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $failedIsoDeleteId)
->value('state') === 'active'
&& (string) $failedIsoDeleteParent->effect_resolution === 'not_applied'
&& !(bool) $failedIsoDeleteParent->finalization_incident,
'A zero pointer in ISO delete_failed state was not recovered.'
);
// Two resources claiming the same immutable child are deliberately not
// guessed. The parent remains fenced and the task center gets an identity code.
$ambiguousParentId = $insertParent('template_import');
$insertResolvedChild(
$ambiguousParentId,
'applied',
'template_import_replica'
);
$firstTemplateId = $insertProjectionForParent($ambiguousParentId);
$secondTemplateId = $firstTemplateId . '-duplicate';
$firstTemplate = (array) Capsule::table('cloud_templates')
->where('id', $firstTemplateId)
->first();
$firstTemplate['id'] = $secondTemplateId;
$firstTemplate['state'] = 'importing';
$firstTemplate['active_job_id'] = null;
Capsule::table('cloud_templates')->insert($firstTemplate);
$firstReplica = (array) Capsule::table('cloud_template_replicas')
->where('template_id', $firstTemplateId)
->first();
$firstReplica['template_id'] = $secondTemplateId;
Capsule::table('cloud_template_replicas')->insert($firstReplica);
Capsule::table('cloud_templates')->where('id', $firstTemplateId)->update([
'state' => 'importing',
'active_job_id' => null,
]);
$runLegacySweep($ambiguousParentId);
$ambiguousParent = CloudJob::findOrFail($ambiguousParentId);
$ambiguousRecovery = CloudJobService::recoveryPresentation(
$ambiguousParent,
true
);
$assert(
(string) $ambiguousParent->effect_resolution === 'unknown'
&& (bool) $ambiguousParent->finalization_incident
&& (string) $ambiguousParent->finalization_error_code
=== 'recovery_operation_identity_conflict'
&& ($ambiguousRecovery['code'] ?? null)
=== 'recovery_operation_identity_conflict'
&& Capsule::table('cloud_templates')
->whereIn('id', [$firstTemplateId, $secondTemplateId])
->whereNotNull('active_job_id')
->count() === 0,
'An ambiguous template pointer was guessed or not shown as an identity conflict.'
);
// Already-published parents normally avoid the legacy sweep. A lost pointer
// plus an in-flight catalog state is an explicit exception: repair the unique
// pointer, then keep the contradictory projection fenced and visible.
$publishedTemplateParentId = $insertParent('template_import');
$insertResolvedChild(
$publishedTemplateParentId,
'applied',
'template_import_replica'
);
$publishedTemplateId = $insertProjectionForParent(
$publishedTemplateParentId
);
Capsule::table('cloud_templates')
->where('id', $publishedTemplateId)
->update(['state' => 'importing', 'active_job_id' => null]);
Capsule::table('cloud_jobs')->where('id', $publishedTemplateParentId)->update([
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'normal_completion',
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
$runPublishedSweep($publishedTemplateParentId);
$publishedTemplateParent = CloudJob::findOrFail(
$publishedTemplateParentId
);
$assert(
(int) Capsule::table('cloud_templates')
->where('id', $publishedTemplateId)
->value('active_job_id') === $publishedTemplateParentId
&& (string) Capsule::table('cloud_templates')
->where('id', $publishedTemplateId)
->value('state') === 'importing'
&& (bool) $publishedTemplateParent->finalization_incident
&& (string) $publishedTemplateParent->finalization_error_code
=== 'recovery_instance_state_conflict'
&& (CloudJobService::recoveryPresentation(
$publishedTemplateParent,
true
)['code'] ?? null) === 'recovery_instance_state_conflict',
'A published template with a lost pointer stayed invisible or was falsely unlocked.'
);
$publishedIsoParentId = $insertParent('user_iso_delete');
$insertResolvedChild(
$publishedIsoParentId,
'applied',
'user_iso_delete_replica'
);
$publishedIsoId = $insertProjectionForParent($publishedIsoParentId);
Capsule::table('cloud_user_isos')->where('id', $publishedIsoId)->update([
'state' => 'deleting',
'active_job_id' => 0,
]);
Capsule::table('cloud_jobs')->where('id', $publishedIsoParentId)->update([
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'normal_completion',
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
]);
$runPublishedSweep($publishedIsoParentId);
$publishedIsoParent = CloudJob::findOrFail($publishedIsoParentId);
$assert(
(int) Capsule::table('cloud_user_isos')
->where('id', $publishedIsoId)
->value('active_job_id') === $publishedIsoParentId
&& (string) Capsule::table('cloud_user_isos')
->where('id', $publishedIsoId)
->value('state') === 'deleting'
&& (bool) $publishedIsoParent->finalization_incident
&& (string) $publishedIsoParent->finalization_error_code
=== 'recovery_instance_state_conflict'
&& (CloudJobService::recoveryPresentation(
$publishedIsoParent,
true
)['code'] ?? null) === 'recovery_instance_state_conflict',
'A published ISO with a lost pointer stayed invisible or was falsely unlocked.'
);
// Metadata sync keeps the main template state active, so recovery needs an
// additional in-flight proof. A failed/unknown parent plus the exact current
// child/replica identity is sufficient and must still fence new mutations.
$metadataParentId = $insertParent('template_metadata_sync');
$insertResolvedChild(
$metadataParentId,
'applied',
'template_metadata_replica'
);
$metadataTemplateId = $insertProjectionForParent($metadataParentId);
Capsule::table('cloud_templates')
->where('id', $metadataTemplateId)
->update(['active_job_id' => null]);
$metadataBoundary = new ReflectionMethod(
\App\Services\Cloud\CloudTemplateCatalogService::class,
'activeJobSafelySettled'
);
$assert(
$metadataBoundary->invoke(
null,
\App\Models\CloudTemplate::findOrFail($metadataTemplateId),
true
) === false,
'A missing metadata pointer allowed a new overlapping template mutation.'
);
$assert(
Capsule::table('cloud_templates')
->where('id', $metadataTemplateId)
->whereNull('active_job_id')
->exists(),
'The mutation boundary repaired the pointer in reverse lock order.'
);
$runLegacySweep($metadataParentId);
$metadataParent = CloudJob::findOrFail($metadataParentId);
$assert(
(int) Capsule::table('cloud_templates')
->where('id', $metadataTemplateId)
->value('active_job_id') === $metadataParentId
&& (string) Capsule::table('cloud_templates')
->where('id', $metadataTemplateId)
->value('state') === 'active'
&& (string) $metadataParent->effect_resolution === 'applied'
&& !(bool) $metadataParent->finalization_incident,
'A uniquely proven metadata-sync pointer did not recover.'
);
// A healthy, already-published metadata history with only active replicas is
// not evidence of an in-flight operation and must never be rebound by guess.
$healthyMetadataParentId = $insertParent('template_metadata_sync');
$insertResolvedChild(
$healthyMetadataParentId,
'applied',
'template_metadata_replica'
);
$healthyMetadataTemplateId = $insertProjectionForParent(
$healthyMetadataParentId
);
Capsule::table('cloud_templates')
->where('id', $healthyMetadataTemplateId)
->update(['active_job_id' => null]);
Capsule::table('cloud_jobs')->where('id', $healthyMetadataParentId)->update([
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'normal_completion',
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$parkOtherParents($healthyMetadataParentId);
$assert(
!CloudJobService::reconcileOneLegacyAggregateEffect(),
'A healthy metadata history was incorrectly selected for pointer repair.'
);
$healthyRepairRejected = false;
try {
\App\Services\Cloud\CloudTemplateCatalogService::
repairMissingParentBindingLocked(
CloudJob::findOrFail($healthyMetadataParentId)
);
} catch (\App\Services\Cloud\CloudRecoveryInstanceStateConflict $error) {
$healthyRepairRejected = true;
}
$assert(
$healthyRepairRejected
&& Capsule::table('cloud_templates')
->where('id', $healthyMetadataTemplateId)
->whereNull('active_job_id')
->exists(),
'A healthy metadata history was rebound without in-flight evidence.'
);
$assert(
$metadataBoundary->invoke(
null,
\App\Models\CloudTemplate::findOrFail($healthyMetadataTemplateId),
true
) === true,
'A healthy, fully settled metadata history permanently blocked mutations.'
);
// The same pre-mutation fence applies to an ISO whose visible state was
// already changed back to active while its uniquely linked parent is still
// unresolved. This is the exact window before the parent-first sweep runs.
$isoBoundaryParentId = $insertParent('user_iso_import');
$insertResolvedChild(
$isoBoundaryParentId,
'applied',
'user_iso_import_replica'
);
$isoBoundaryId = $insertProjectionForParent($isoBoundaryParentId);
Capsule::table('cloud_user_isos')->where('id', $isoBoundaryId)->update([
'state' => 'active',
'active_job_id' => null,
]);
$isoBoundary = new ReflectionMethod(
\App\Services\Cloud\CloudUserIsoService::class,
'activeJobSafelySettled'
);
$assert(
$isoBoundary->invoke(
null,
\App\Models\CloudUserIso::findOrFail($isoBoundaryId),
true
) === false
&& Capsule::table('cloud_user_isos')
->where('id', $isoBoundaryId)
->whereNull('active_job_id')
->exists(),
'A lost ISO pointer allowed a second mutation before recovery.'
);
// Even with a failed/unknown parent, two current metadata resources remain an
// identity conflict. Neither pointer may be written.
$ambiguousMetadataParentId = $insertParent('template_metadata_sync');
$insertResolvedChild(
$ambiguousMetadataParentId,
'applied',
'template_metadata_replica'
);
$firstMetadataId = $insertProjectionForParent($ambiguousMetadataParentId);
$secondMetadataId = $firstMetadataId . '-duplicate';
$firstMetadata = (array) Capsule::table('cloud_templates')
->where('id', $firstMetadataId)
->first();
$firstMetadata['id'] = $secondMetadataId;
$firstMetadata['active_job_id'] = null;
Capsule::table('cloud_templates')->insert($firstMetadata);
$firstMetadataReplica = (array) Capsule::table('cloud_template_replicas')
->where('template_id', $firstMetadataId)
->first();
$firstMetadataReplica['template_id'] = $secondMetadataId;
Capsule::table('cloud_template_replicas')->insert($firstMetadataReplica);
Capsule::table('cloud_templates')->where('id', $firstMetadataId)->update([
'active_job_id' => null,
]);
$runLegacySweep($ambiguousMetadataParentId);
$ambiguousMetadataParent = CloudJob::findOrFail(
$ambiguousMetadataParentId
);
$assert(
(string) $ambiguousMetadataParent->effect_resolution === 'unknown'
&& (bool) $ambiguousMetadataParent->finalization_incident
&& (string) $ambiguousMetadataParent->finalization_error_code
=== 'recovery_operation_identity_conflict'
&& Capsule::table('cloud_templates')
->whereIn('id', [$firstMetadataId, $secondMetadataId])
->whereNotNull('active_job_id')
->count() === 0,
'An ambiguous metadata-sync pointer was guessed or falsely released.'
);
$publishParent = static function (int $parentId): void {
Capsule::table('cloud_jobs')->where('id', $parentId)->update([
'outcome' => 'succeeded',
'status' => 'succeeded',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'normal_completion',
'error_code' => null,
'finalization_incident' => 0,
'finalization_error_code' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
};
// A quota-rejected import is a legitimate published failed/applied projection:
// its active replicas remain for explicit cleanup. A lost pointer must be
// repaired without turning that state into a generic incident.
$quotaIsoParentId = $insertParent('user_iso_import');
$insertResolvedChild(
$quotaIsoParentId,
'applied',
'user_iso_import_replica'
);
$quotaIsoId = $insertProjectionForParent($quotaIsoParentId);
Capsule::table('cloud_user_isos')->where('id', $quotaIsoId)->update([
'active_job_id' => null,
]);
Capsule::table('cloud_jobs')->where('id', $quotaIsoParentId)->update([
'outcome' => 'failed',
'status' => 'failed',
'effect_may_have_applied' => 1,
'safe_to_resubmit' => 0,
'effect_resolution' => 'applied',
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'normal_completion',
'error_code' => 'user_iso_quota_exceeded',
'finalization_incident' => 0,
'finalization_error_code' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$runPublishedSweep($quotaIsoParentId);
$quotaIsoParent = CloudJob::findOrFail($quotaIsoParentId);
$assert(
(int) Capsule::table('cloud_user_isos')->where('id', $quotaIsoId)
->value('active_job_id') === $quotaIsoParentId
&& (string) Capsule::table('cloud_user_isos')->where('id', $quotaIsoId)
->value('state') === 'import_failed'
&& (string) $quotaIsoParent->outcome === 'failed'
&& (string) $quotaIsoParent->effect_resolution === 'applied'
&& !(bool) $quotaIsoParent->finalization_incident
&& $isoBoundary->invoke(
null,
\App\Models\CloudUserIso::findOrFail($quotaIsoId),
false
) === true,
'A settled quota cleanup projection was not uniquely rebound.'
);
// A published not-applied delete may legitimately retain an older stale
// replica. It is already safe before pointer repair and remains strictly
// verifiable after the parent-first sweep restores the pointer.
$staleDeleteParentId = $insertParent('template_delete');
$insertResolvedChild(
$staleDeleteParentId,
'not_applied',
'template_delete_replica'
);
$staleDeleteTemplateId = $insertProjectionForParent($staleDeleteParentId);
Capsule::table('cloud_templates')->where('id', $staleDeleteTemplateId)->update([
'state' => 'active',
'active_job_id' => null,
]);
Capsule::table('cloud_template_replicas')
->where('template_id', $staleDeleteTemplateId)->update([
'state' => 'stale',
'revision' => 0,
]);
Capsule::table('cloud_jobs')->where('id', $staleDeleteParentId)->update([
'outcome' => 'failed',
'status' => 'failed',
'effect_may_have_applied' => 0,
'safe_to_resubmit' => 1,
'effect_resolution' => 'not_applied',
'effect_resolved_at' => time(),
'effect_resolution_reason' => 'normal_completion',
'error_code' => 'template_delete_not_applied',
'finalization_incident' => 0,
'finalization_error_code' => null,
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$assert(
$metadataBoundary->invoke(
null,
\App\Models\CloudTemplate::findOrFail($staleDeleteTemplateId),
false
) === true,
'A valid not-applied delete with a stale replica stayed falsely fenced.'
);
$runPublishedSweep($staleDeleteParentId);
$staleDeleteParent = CloudJob::findOrFail($staleDeleteParentId);
$assert(
(int) Capsule::table('cloud_templates')
->where('id', $staleDeleteTemplateId)
->value('active_job_id') === $staleDeleteParentId
&& (string) Capsule::table('cloud_template_replicas')
->where('template_id', $staleDeleteTemplateId)
->value('state') === 'stale'
&& !(bool) $staleDeleteParent->finalization_incident,
'A valid not-applied delete pointer was not repaired safely.'
);
// Published healthy resources with a positive wrong pointer remain eligible
// for diagnosis, but the positive value is never overwritten automatically.
$publishedWrongTemplateParentId = $insertParent('template_import');
$insertResolvedChild(
$publishedWrongTemplateParentId,
'applied',
'template_import_replica'
);
$publishedWrongTemplateId = $insertProjectionForParent(
$publishedWrongTemplateParentId
);
Capsule::table('cloud_templates')
->where('id', $publishedWrongTemplateId)->update([
'state' => 'active',
'active_job_id' => 999999991,
]);
$publishParent($publishedWrongTemplateParentId);
$runPublishedSweep($publishedWrongTemplateParentId);
$publishedWrongTemplateParent = CloudJob::findOrFail(
$publishedWrongTemplateParentId
);
$assert(
(int) Capsule::table('cloud_templates')
->where('id', $publishedWrongTemplateId)
->value('active_job_id') === 999999991
&& (string) $publishedWrongTemplateParent->finalization_error_code
=== 'recovery_operation_identity_conflict'
&& (int) $publishedWrongTemplateParent->available_at >= time() + 290,
'A published template positive pointer was overwritten or not queued.'
);
$publishedWrongIsoParentId = $insertParent('user_iso_import');
$insertResolvedChild(
$publishedWrongIsoParentId,
'applied',
'user_iso_import_replica'
);
$publishedWrongIsoId = $insertProjectionForParent($publishedWrongIsoParentId);
Capsule::table('cloud_user_isos')
->where('id', $publishedWrongIsoId)->update([
'state' => 'active',
'active_job_id' => 999999992,
]);
$publishParent($publishedWrongIsoParentId);
$runPublishedSweep($publishedWrongIsoParentId);
$publishedWrongIsoParent = CloudJob::findOrFail($publishedWrongIsoParentId);
$assert(
(int) Capsule::table('cloud_user_isos')
->where('id', $publishedWrongIsoId)
->value('active_job_id') === 999999992
&& (string) $publishedWrongIsoParent->finalization_error_code
=== 'recovery_operation_identity_conflict'
&& (int) $publishedWrongIsoParent->available_at >= time() + 290,
'A published ISO positive pointer was overwritten or not queued.'
);
// Full content is part of the binding proof. A NULL pointer plus a matching
// identity chain must stay fenced and visible when bytes or hashes diverge.
$corruptTemplateParentId = $insertParent('template_import');
$insertResolvedChild(
$corruptTemplateParentId,
'applied',
'template_import_replica'
);
$corruptTemplateId = $insertProjectionForParent($corruptTemplateParentId);
Capsule::table('cloud_templates')->where('id', $corruptTemplateId)->update([
'state' => 'active',
'active_job_id' => null,
]);
Capsule::table('cloud_template_replicas')
->where('template_id', $corruptTemplateId)->update([
'sha256' => str_repeat('f', 64),
]);
$publishParent($corruptTemplateParentId);
$assert(
$metadataBoundary->invoke(
null,
\App\Models\CloudTemplate::findOrFail($corruptTemplateId),
false
) === false,
'A NULL template pointer with corrupt content falsely unlocked mutation.'
);
$runPublishedSweep($corruptTemplateParentId);
$corruptTemplateParent = CloudJob::findOrFail($corruptTemplateParentId);
$corruptTemplatePointer = Capsule::table('cloud_templates')
->where('id', $corruptTemplateId)->value('active_job_id');
$assert(
(int) $corruptTemplatePointer === $corruptTemplateParentId
&& (string) $corruptTemplateParent->finalization_error_code
=== 'recovery_instance_state_conflict',
'Corrupt template content was unlocked or dropped from recovery: pointer='
. var_export($corruptTemplatePointer, true) . ', code='
. (string) $corruptTemplateParent->finalization_error_code
);
$corruptIsoParentId = $insertParent('user_iso_import');
$insertResolvedChild(
$corruptIsoParentId,
'applied',
'user_iso_import_replica'
);
$corruptIsoId = $insertProjectionForParent($corruptIsoParentId);
Capsule::table('cloud_user_isos')->where('id', $corruptIsoId)->update([
'state' => 'active',
'active_job_id' => 0,
]);
Capsule::table('cloud_user_iso_replicas')
->where('user_iso_id', $corruptIsoId)->update([
'size_bytes' => 8192,
]);
$publishParent($corruptIsoParentId);
$assert(
$isoBoundary->invoke(
null,
\App\Models\CloudUserIso::findOrFail($corruptIsoId),
false
) === false,
'A zero ISO pointer with corrupt content falsely unlocked mutation.'
);
$runPublishedSweep($corruptIsoParentId);
$corruptIsoParent = CloudJob::findOrFail($corruptIsoParentId);
$assert(
(int) Capsule::table('cloud_user_isos')->where('id', $corruptIsoId)
->value('active_job_id') === $corruptIsoParentId
&& (string) $corruptIsoParent->finalization_error_code
=== 'recovery_instance_state_conflict',
'Corrupt ISO content was unlocked or dropped from recovery.'
);
// The same child id cannot belong to both catalog domains. Cross-table reuse
// is an identity incident and may not authorize a NULL-pointer repair.
$crossTableTemplateParentId = $insertParent('template_import');
$crossTableTemplateChildId = $insertResolvedChild(
$crossTableTemplateParentId,
'applied',
'template_import_replica'
);
$crossTableTemplateId = $insertProjectionForParent(
$crossTableTemplateParentId
);
Capsule::table('cloud_templates')
->where('id', $crossTableTemplateId)
->update(['active_job_id' => null]);
$crossTableTemplateChild = CloudJob::findOrFail($crossTableTemplateChildId);
Capsule::table('cloud_user_iso_replicas')->insert([
'host_id' => 800000 + $crossTableTemplateChildId,
'user_iso_id' => md5('cross-table-iso:' . $crossTableTemplateChildId),
'user_id' => 1,
'state' => 'active',
'size_bytes' => 4096,
'sha256' => str_repeat('a', 64),
'operation_id' => (string) $crossTableTemplateChild->operation_id,
'job_id' => $crossTableTemplateChildId,
'error_code' => '',
'created_at' => time(),
'updated_at' => time(),
]);
$runLegacySweep($crossTableTemplateParentId);
$crossTableTemplateParent = CloudJob::findOrFail(
$crossTableTemplateParentId
);
$assert(
Capsule::table('cloud_templates')->where('id', $crossTableTemplateId)
->whereNull('active_job_id')->exists()
&& (string) $crossTableTemplateParent->finalization_error_code
=== 'recovery_operation_identity_conflict',
'A template child duplicated in the ISO ledger repaired its pointer.'
);
$crossTableIsoParentId = $insertParent('user_iso_import');
$crossTableIsoChildId = $insertResolvedChild(
$crossTableIsoParentId,
'applied',
'user_iso_import_replica'
);
$crossTableIsoId = $insertProjectionForParent($crossTableIsoParentId);
Capsule::table('cloud_user_isos')->where('id', $crossTableIsoId)->update([
'active_job_id' => null,
]);
$crossTableIsoChild = CloudJob::findOrFail($crossTableIsoChildId);
Capsule::table('cloud_template_replicas')->insert([
'host_id' => 900000 + $crossTableIsoChildId,
'template_id' => 'cross-table-template-' . $crossTableIsoChildId,
'revision' => 1,
'generation_token' => md5(
'cross-table-template:' . $crossTableIsoChildId
),
'state' => 'active',
'image_bytes' => 4096,
'sha256' => str_repeat('b', 64),
'operation_id' => (string) $crossTableIsoChild->operation_id,
'job_id' => $crossTableIsoChildId,
'error_code' => '',
'created_at' => time(),
'updated_at' => time(),
]);
$runLegacySweep($crossTableIsoParentId);
$crossTableIsoParent = CloudJob::findOrFail($crossTableIsoParentId);
$assert(
Capsule::table('cloud_user_isos')->where('id', $crossTableIsoId)
->whereNull('active_job_id')->exists()
&& (string) $crossTableIsoParent->finalization_error_code
=== 'recovery_operation_identity_conflict',
'An ISO child duplicated in the template ledger repaired its pointer.'
);
// A unique positive pointer is not sufficient: swapping two otherwise valid
// parents must fail closed when their children and replicas belong elsewhere.
$crossTemplateParentA = $insertParent('template_metadata_sync');
$insertResolvedChild(
$crossTemplateParentA,
'applied',
'template_metadata_replica'
);
$crossTemplateA = $insertProjectionForParent($crossTemplateParentA);
$publishParent($crossTemplateParentA);
$crossTemplateParentB = $insertParent('template_metadata_sync');
$insertResolvedChild(
$crossTemplateParentB,
'applied',
'template_metadata_replica'
);
$crossTemplateB = $insertProjectionForParent($crossTemplateParentB);
$publishParent($crossTemplateParentB);
Capsule::table('cloud_templates')->where('id', $crossTemplateA)->update([
'active_job_id' => $crossTemplateParentB,
]);
Capsule::table('cloud_templates')->where('id', $crossTemplateB)->update([
'active_job_id' => $crossTemplateParentA,
]);
$assert(
$metadataBoundary->invoke(
null,
\App\Models\CloudTemplate::findOrFail($crossTemplateA),
false
) === false
&& $metadataBoundary->invoke(
null,
\App\Models\CloudTemplate::findOrFail($crossTemplateB),
false
) === false,
'Cross-wired template parents falsely unlocked both resources.'
);
$runPublishedSweep($crossTemplateParentA);
$runPublishedSweep($crossTemplateParentB);
$assert(
(int) Capsule::table('cloud_templates')->where('id', $crossTemplateA)
->value('active_job_id') === $crossTemplateParentB
&& (int) Capsule::table('cloud_templates')->where('id', $crossTemplateB)
->value('active_job_id') === $crossTemplateParentA
&& (string) CloudJob::findOrFail($crossTemplateParentA)
->finalization_error_code === 'recovery_operation_identity_conflict'
&& (string) CloudJob::findOrFail($crossTemplateParentB)
->finalization_error_code === 'recovery_operation_identity_conflict',
'Cross-wired template pointers were rewritten or removed from recovery.'
);
$crossIsoParentA = $insertParent('user_iso_import');
$insertResolvedChild(
$crossIsoParentA,
'applied',
'user_iso_import_replica'
);
$crossIsoA = $insertProjectionForParent($crossIsoParentA);
$publishParent($crossIsoParentA);
$crossIsoParentB = $insertParent('user_iso_import');
$insertResolvedChild(
$crossIsoParentB,
'applied',
'user_iso_import_replica'
);
$crossIsoB = $insertProjectionForParent($crossIsoParentB);
$publishParent($crossIsoParentB);
Capsule::table('cloud_user_isos')->whereIn(
'id',
[$crossIsoA, $crossIsoB]
)->update(['state' => 'active']);
Capsule::table('cloud_user_isos')->where('id', $crossIsoA)->update([
'active_job_id' => $crossIsoParentB,
]);
Capsule::table('cloud_user_isos')->where('id', $crossIsoB)->update([
'active_job_id' => $crossIsoParentA,
]);
$assert(
$isoBoundary->invoke(
null,
\App\Models\CloudUserIso::findOrFail($crossIsoA),
false
) === false
&& $isoBoundary->invoke(
null,
\App\Models\CloudUserIso::findOrFail($crossIsoB),
false
) === false,
'Cross-wired ISO parents falsely unlocked both resources.'
);
$runPublishedSweep($crossIsoParentA);
$runPublishedSweep($crossIsoParentB);
$assert(
(int) Capsule::table('cloud_user_isos')->where('id', $crossIsoA)
->value('active_job_id') === $crossIsoParentB
&& (int) Capsule::table('cloud_user_isos')->where('id', $crossIsoB)
->value('active_job_id') === $crossIsoParentA
&& (string) CloudJob::findOrFail($crossIsoParentA)
->finalization_error_code === 'recovery_operation_identity_conflict'
&& (string) CloudJob::findOrFail($crossIsoParentB)
->finalization_error_code === 'recovery_operation_identity_conflict',
'Cross-wired ISO pointers were rewritten or removed from recovery.'
);
// Positive damage is never overwritten automatically. Even a unique replica
// chain only proves the expected parent; it cannot authorize replacing a
// positive pointer that may represent a separate damaged generation.
$danglingTemplateParentId = $insertParent('template_metadata_sync');
$insertResolvedChild(
$danglingTemplateParentId,
'applied',
'template_metadata_replica'
);
$danglingTemplateId = $insertProjectionForParent(
$danglingTemplateParentId
);
Capsule::table('cloud_templates')->where('id', $danglingTemplateId)->update([
'active_job_id' => 999999999,
]);
$runLegacySweep($danglingTemplateParentId);
$danglingTemplateParent = CloudJob::findOrFail($danglingTemplateParentId);
$assert(
(int) Capsule::table('cloud_templates')
->where('id', $danglingTemplateId)
->value('active_job_id') === 999999999
&& (string) $danglingTemplateParent->effect_resolution === 'unknown'
&& (bool) $danglingTemplateParent->finalization_incident
&& (string) $danglingTemplateParent->finalization_error_code
=== 'recovery_operation_identity_conflict'
&& (int) $danglingTemplateParent->available_at >= time() + 290,
'A positive dangling template pointer was overwritten or left invisible.'
);
$wrongIsoParentId = $insertParent('user_iso_delete');
$wrongIsoChildId = $insertResolvedChild(
$wrongIsoParentId,
'applied',
'user_iso_delete_replica'
);
$wrongIsoId = $insertProjectionForParent($wrongIsoParentId);
Capsule::table('cloud_user_isos')->where('id', $wrongIsoId)->update([
'state' => 'deleting',
'active_job_id' => $wrongIsoChildId,
]);
$runLegacySweep($wrongIsoParentId);
$wrongIsoParent = CloudJob::findOrFail($wrongIsoParentId);
$assert(
(int) Capsule::table('cloud_user_isos')
->where('id', $wrongIsoId)
->value('active_job_id') === $wrongIsoChildId
&& (string) $wrongIsoParent->effect_resolution === 'unknown'
&& (bool) $wrongIsoParent->finalization_incident
&& (string) $wrongIsoParent->finalization_error_code
=== 'recovery_operation_identity_conflict'
&& (int) $wrongIsoParent->available_at >= time() + 290,
'An ISO pointer to a leaf job was overwritten or left invisible.'
);
// A NULL pointer is intentional after an exactly proven mixed aggregate. The
// background pointer-repair sweep must not rebind that completed parent and
// recreate the lock before an operator starts a fresh cleanup operation.
foreach ([
'template_import' => 'template_import_replica',
'template_delete' => 'template_delete_replica',
'user_iso_import' => 'user_iso_import_replica',
'user_iso_delete' => 'user_iso_delete_replica',
] as $parentKind => $childKind) {
$partialParentId = $insertParent($parentKind);
$appliedChildId = $insertResolvedChild(
$partialParentId,
'applied',
$childKind
);
$insertResolvedChild($partialParentId, 'not_applied', $childKind);
$partialResourceId = $insertProjectionForParent($partialParentId);
CloudJobService::reconcileAggregateParentEffectLocked(
CloudJob::findOrFail($appliedChildId),
null
);
// Earlier corruption fixtures deliberately include aggregate-shaped rows
// with a non-null parent id. Park every other row so this assertion tests
// only whether the released partial itself remains discoverable.
$future = time() + 3600;
Capsule::table('cloud_jobs')->where('id', '<>', $partialParentId)->update([
'available_at' => $future,
'lease_until' => $future,
]);
Capsule::table('cloud_jobs')->where('id', $partialParentId)->update([
'available_at' => 0,
'lease_token' => null,
'lease_until' => 0,
]);
$assert(
!CloudJobService::reconcileOneLegacyAggregateEffect(),
'A settled mixed aggregate re-entered pointer repair: ' . $parentKind
);
$pointer = str_starts_with($parentKind, 'template_')
? Capsule::table('cloud_templates')->where('id', $partialResourceId)
->value('active_job_id')
: Capsule::table('cloud_user_isos')->where('id', $partialResourceId)
->value('active_job_id');
$assert(
$pointer === null
&& !(bool) CloudJob::findOrFail($partialParentId)
->finalization_incident,
'A settled mixed aggregate was rebound or refenced: ' . $parentKind
);
// A later local projection incident must make the released parent
// discoverable again. Recovery may temporarily restore the pointer only
// to validate the immutable mixed ledger; a successful recheck releases
// it again instead of recreating a permanent catalog lock.
Capsule::table('cloud_jobs')->where('id', $partialParentId)->update([
'finalization_incident' => 1,
'finalization_error_code' => 'recovery_instance_state_conflict',
'available_at' => 0,
'lease_until' => 0,
]);
$assert(
CloudJobService::reconcileOneLegacyAggregateEffect(),
'A settled mixed aggregate incident was not rechecked: ' . $parentKind
);
$recheckedPointer = str_starts_with($parentKind, 'template_')
? Capsule::table('cloud_templates')->where('id', $partialResourceId)
->value('active_job_id')
: Capsule::table('cloud_user_isos')->where('id', $partialResourceId)
->value('active_job_id');
$recheckedParent = CloudJob::findOrFail($partialParentId);
$assert(
$recheckedPointer === null
&& !(bool) $recheckedParent->finalization_incident
&& $recheckedParent->finalization_error_code === null
&& CloudJobService::effectScope($recheckedParent) === 'partial',
'A valid settled mixed incident did not release cleanly: ' . $parentKind
);
}
echo "Cloud catalog active-job pointer recovery checks passed.\n";