|
|
|
|
@ -48,6 +48,7 @@ namespace {
|
|
|
|
|
use App\Services\Cloud\CloudAutomaticOperationRecoveryService;
|
|
|
|
|
use App\Services\Cloud\CloudError;
|
|
|
|
|
use App\Services\Cloud\CloudJobService;
|
|
|
|
|
use App\Services\Cloud\CloudOperationCompletionService;
|
|
|
|
|
use App\Services\Cloud\CloudOperationRecoveryIdentity;
|
|
|
|
|
use App\Services\Cloud\CloudRecoveryAggregateStateConflict;
|
|
|
|
|
use App\Services\Cloud\CloudRecoveryInstanceStateConflict;
|
|
|
|
|
@ -3611,5 +3612,176 @@ namespace {
|
|
|
|
|
&& (int) $healthyAfterAudit->available_at >= time() + 290,
|
|
|
|
|
'A healthy published aggregate audit hot-looped or created an incident.');
|
|
|
|
|
|
|
|
|
|
// A success receipt arriving after the immutable deadline failure must
|
|
|
|
|
// converge the original delete generation in place. It may update the
|
|
|
|
|
// effect/catalog projection, but it must not flip either failed audit to
|
|
|
|
|
// succeeded or contact the destructive Agent endpoint again.
|
|
|
|
|
$lateDeleteParentId = $insertParent('template_delete');
|
|
|
|
|
$lateDeleteChildId = $insertResolvedChild(
|
|
|
|
|
$lateDeleteParentId,
|
|
|
|
|
'not_applied',
|
|
|
|
|
'template_delete_replica'
|
|
|
|
|
);
|
|
|
|
|
$lateDeleteTemplateId = $insertProjectionForParent(
|
|
|
|
|
$lateDeleteParentId
|
|
|
|
|
);
|
|
|
|
|
$lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $lateDeleteChildId)->update([
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'DELETE',
|
|
|
|
|
'request_target' => '/v1/templates/'
|
|
|
|
|
. rawurlencode($lateDeleteTemplateId),
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => (string) $lateDeleteChild->operation_id,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'can_manage' => true,
|
|
|
|
|
'template_id' => $lateDeleteTemplateId,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
'effect_may_have_applied' => 1,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'unknown',
|
|
|
|
|
'effect_resolved_at' => null,
|
|
|
|
|
'effect_resolution_reason' => '',
|
|
|
|
|
'completion_event_id' => null,
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_receipt_invalid',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId);
|
|
|
|
|
$lateDeleteIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
$lateDeleteChild,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$beforeLateDeleteRequests = CloudAgentSettingsService::$requests;
|
|
|
|
|
$lateDeleteAccepted = CloudOperationCompletionService::accept([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => 'aggregate-late-template-delete',
|
|
|
|
|
'agent_id' => (string) $lateDeleteChild->host_id,
|
|
|
|
|
'operation_id' => (string) $lateDeleteChild->operation_id,
|
|
|
|
|
'action' => $lateDeleteIdentity['action'],
|
|
|
|
|
'resource_id' => $lateDeleteIdentity['resource_id'],
|
|
|
|
|
'request_fingerprint' =>
|
|
|
|
|
$lateDeleteIdentity['request_fingerprint'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'resource_type' => 'template',
|
|
|
|
|
'id' => $lateDeleteTemplateId,
|
|
|
|
|
'deleted' => true,
|
|
|
|
|
'managed_image_absent' => true,
|
|
|
|
|
'managed_metadata_absent' => true,
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
],
|
|
|
|
|
]);
|
|
|
|
|
$lateDeleteChild = CloudJob::findOrFail($lateDeleteChildId);
|
|
|
|
|
$lateDeleteParent = CloudJob::findOrFail($lateDeleteParentId);
|
|
|
|
|
$assert($lateDeleteAccepted === [
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
'disposition' => 'applied',
|
|
|
|
|
]
|
|
|
|
|
&& (string) $lateDeleteChild->outcome === 'failed'
|
|
|
|
|
&& (string) $lateDeleteChild->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $lateDeleteChild->finalization_incident
|
|
|
|
|
&& (string) $lateDeleteParent->outcome === 'failed'
|
|
|
|
|
&& (string) $lateDeleteParent->error_code
|
|
|
|
|
=== 'aggregate_deadline_exceeded'
|
|
|
|
|
&& (string) $lateDeleteParent->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $lateDeleteParent->finalization_incident
|
|
|
|
|
&& (string) Capsule::table('cloud_templates')
|
|
|
|
|
->where('id', $lateDeleteTemplateId)->value('state') === 'retired'
|
|
|
|
|
&& !Capsule::table('cloud_template_replicas')
|
|
|
|
|
->where('template_id', $lateDeleteTemplateId)->exists()
|
|
|
|
|
&& CloudAgentSettingsService::$requests === $beforeLateDeleteRequests,
|
|
|
|
|
'A late template deletion receipt did not converge the original '
|
|
|
|
|
. 'operation without replaying or rewriting its failure audit.');
|
|
|
|
|
|
|
|
|
|
$lateIsoParentId = $insertParent('user_iso_delete');
|
|
|
|
|
$lateIsoChildId = $insertResolvedChild(
|
|
|
|
|
$lateIsoParentId,
|
|
|
|
|
'not_applied',
|
|
|
|
|
'user_iso_delete_replica'
|
|
|
|
|
);
|
|
|
|
|
$lateIsoId = $insertProjectionForParent($lateIsoParentId);
|
|
|
|
|
$lateIsoChild = CloudJob::findOrFail($lateIsoChildId);
|
|
|
|
|
Capsule::table('cloud_jobs')->where('id', $lateIsoChildId)->update([
|
|
|
|
|
'sealed_payload' => CloudSecrets::encrypt([
|
|
|
|
|
'method' => 'DELETE',
|
|
|
|
|
'request_target' => '/v1/user-isos/'
|
|
|
|
|
. rawurlencode($lateIsoId) . '?owner_id=1',
|
|
|
|
|
'body' => [
|
|
|
|
|
'operation_id' => (string) $lateIsoChild->operation_id,
|
|
|
|
|
],
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'can_manage' => false,
|
|
|
|
|
'user_iso_id' => $lateIsoId,
|
|
|
|
|
], 'job-payload'),
|
|
|
|
|
'effect_may_have_applied' => 1,
|
|
|
|
|
'safe_to_resubmit' => 0,
|
|
|
|
|
'effect_resolution' => 'unknown',
|
|
|
|
|
'effect_resolved_at' => null,
|
|
|
|
|
'effect_resolution_reason' => '',
|
|
|
|
|
'completion_event_id' => null,
|
|
|
|
|
'finalization_incident' => 1,
|
|
|
|
|
'finalization_error_code' => 'recovery_receipt_invalid',
|
|
|
|
|
'available_at' => 0,
|
|
|
|
|
'lease_token' => null,
|
|
|
|
|
'lease_until' => 0,
|
|
|
|
|
]);
|
|
|
|
|
$lateIsoChild = CloudJob::findOrFail($lateIsoChildId);
|
|
|
|
|
$lateIsoIdentity = CloudOperationRecoveryIdentity::expected(
|
|
|
|
|
$lateIsoChild,
|
|
|
|
|
true
|
|
|
|
|
);
|
|
|
|
|
$lateIsoAccepted = CloudOperationCompletionService::accept([
|
|
|
|
|
'type' => 'completion',
|
|
|
|
|
'id' => 'aggregate-late-user-iso-delete',
|
|
|
|
|
'agent_id' => (string) $lateIsoChild->host_id,
|
|
|
|
|
'operation_id' => (string) $lateIsoChild->operation_id,
|
|
|
|
|
'action' => $lateIsoIdentity['action'],
|
|
|
|
|
'resource_id' => $lateIsoIdentity['resource_id'],
|
|
|
|
|
'request_fingerprint' => $lateIsoIdentity['request_fingerprint'],
|
|
|
|
|
'status' => 'succeeded',
|
|
|
|
|
'effect_may_have_applied' => false,
|
|
|
|
|
'safe_to_resubmit' => false,
|
|
|
|
|
'error_code' => '',
|
|
|
|
|
'result' => [
|
|
|
|
|
'resource_type' => 'user_iso',
|
|
|
|
|
'id' => $lateIsoId,
|
|
|
|
|
'owner_id' => 1,
|
|
|
|
|
'deleted' => true,
|
|
|
|
|
'managed_image_absent' => true,
|
|
|
|
|
'managed_metadata_absent' => true,
|
|
|
|
|
'managed_artifacts_absent' => true,
|
|
|
|
|
],
|
|
|
|
|
]);
|
|
|
|
|
$lateIsoChild = CloudJob::findOrFail($lateIsoChildId);
|
|
|
|
|
$lateIsoParent = CloudJob::findOrFail($lateIsoParentId);
|
|
|
|
|
$lateIso = Capsule::table('cloud_user_isos')
|
|
|
|
|
->where('id', $lateIsoId)->first();
|
|
|
|
|
$assert($lateIsoAccepted === [
|
|
|
|
|
'accepted' => true,
|
|
|
|
|
'disposition' => 'applied',
|
|
|
|
|
]
|
|
|
|
|
&& (string) $lateIsoChild->outcome === 'failed'
|
|
|
|
|
&& (string) $lateIsoChild->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $lateIsoChild->finalization_incident
|
|
|
|
|
&& (string) $lateIsoParent->outcome === 'failed'
|
|
|
|
|
&& (string) $lateIsoParent->effect_resolution === 'applied'
|
|
|
|
|
&& !(bool) $lateIsoParent->finalization_incident
|
|
|
|
|
&& $lateIso !== null
|
|
|
|
|
&& (string) $lateIso->state === 'retired'
|
|
|
|
|
&& (string) $lateIso->name === ''
|
|
|
|
|
&& (int) $lateIso->size_bytes === 0
|
|
|
|
|
&& !Capsule::table('cloud_user_iso_replicas')
|
|
|
|
|
->where('user_iso_id', $lateIsoId)->exists()
|
|
|
|
|
&& CloudAgentSettingsService::$requests === $beforeLateDeleteRequests,
|
|
|
|
|
'A late user-ISO deletion receipt did not converge its owner-bound '
|
|
|
|
|
. 'catalog ledgers without replay.');
|
|
|
|
|
|
|
|
|
|
echo "Cloud aggregate recovery convergence tests passed.\n";
|
|
|
|
|
}
|
|
|
|
|
|