|
|
|
|
@ -48,6 +48,7 @@ $schema->create('cloud_traffic_pools', static function (Blueprint $t): void {
|
|
|
|
|
$schema->create('cloud_instances', static function (Blueprint $t): void {
|
|
|
|
|
$t->increments('id');
|
|
|
|
|
$t->integer('user_id');
|
|
|
|
|
$t->integer('current_period_id')->nullable();
|
|
|
|
|
$t->string('service_status');
|
|
|
|
|
$t->string('network_status');
|
|
|
|
|
$t->integer('network_suspended_at')->nullable();
|
|
|
|
|
@ -410,9 +411,273 @@ $attention = $instance->getConnection()->transaction(
|
|
|
|
|
return $result;
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$assert($attention['traffic_decision'] === 'preserved'
|
|
|
|
|
$assert($attention['traffic_decision'] === 'fenced'
|
|
|
|
|
&& in_array('traffic_pool_exhausted',
|
|
|
|
|
$attention['desired_reasons'], true),
|
|
|
|
|
'An attention projection guessed that traffic access could be restored.');
|
|
|
|
|
|
|
|
|
|
$fenceEndsAt = time() + 3600;
|
|
|
|
|
$fencedInstance = new CloudInstance();
|
|
|
|
|
$fencedInstance->id = 11;
|
|
|
|
|
$fencedInstance->user_id = 20;
|
|
|
|
|
$fencedInstance->current_period_id = 31;
|
|
|
|
|
$fencedInstance->service_status = 'active';
|
|
|
|
|
$fencedInstance->network_status = 'blocked_traffic';
|
|
|
|
|
$fencedInstance->network_suspended_at = 900;
|
|
|
|
|
$fencedInstance->expires_at = $fenceEndsAt;
|
|
|
|
|
$fencedInstance->updated_at = 1000;
|
|
|
|
|
$fencedInstance->save();
|
|
|
|
|
$fencedPeriod = new CloudServicePeriod();
|
|
|
|
|
$fencedPeriod->id = 31;
|
|
|
|
|
$fencedPeriod->instance_id = 11;
|
|
|
|
|
$fencedPeriod->user_id = 20;
|
|
|
|
|
$fencedPeriod->traffic_pool_group_id = 2;
|
|
|
|
|
$fencedPeriod->traffic_quota_mode = 'finite';
|
|
|
|
|
$fencedPeriod->status = 'active';
|
|
|
|
|
$fencedPeriod->starts_at = 900;
|
|
|
|
|
$fencedPeriod->ends_at = $fenceEndsAt;
|
|
|
|
|
$fencedPeriod->save();
|
|
|
|
|
$fenced = $fencedInstance->getConnection()->transaction(
|
|
|
|
|
static function () use ($fencedInstance, $fencedPeriod): array {
|
|
|
|
|
$locked = CloudInstance::where('id', $fencedInstance->id)
|
|
|
|
|
->lockForUpdate()->firstOrFail();
|
|
|
|
|
$result = CloudInstanceNetworkPolicy::synchronizeLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
$fencedPeriod,
|
|
|
|
|
1013
|
|
|
|
|
);
|
|
|
|
|
$locked->save();
|
|
|
|
|
return $result;
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$assert($fenced['traffic_decision'] === 'fenced'
|
|
|
|
|
&& $fenced['desired_suspended'] === true
|
|
|
|
|
&& $fenced['applied_suspended'] === true,
|
|
|
|
|
'An attention pool restored an already blocked instance during takeover.');
|
|
|
|
|
|
|
|
|
|
Capsule::table('cloud_traffic_pools')->where('id', 40)->update([
|
|
|
|
|
'state' => 'available',
|
|
|
|
|
'available_bytes' => 100,
|
|
|
|
|
'version' => 5,
|
|
|
|
|
]);
|
|
|
|
|
$fencedResume = str_repeat('f', 32);
|
|
|
|
|
$fencedInstance->getConnection()->transaction(
|
|
|
|
|
static function () use (
|
|
|
|
|
$fencedInstance,
|
|
|
|
|
$fencedPeriod,
|
|
|
|
|
$fencedResume
|
|
|
|
|
): void {
|
|
|
|
|
$locked = CloudInstance::where('id', $fencedInstance->id)
|
|
|
|
|
->lockForUpdate()->firstOrFail();
|
|
|
|
|
CloudInstanceNetworkPolicy::synchronizeLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
$fencedPeriod,
|
|
|
|
|
1014
|
|
|
|
|
);
|
|
|
|
|
CloudInstanceNetworkPolicy::prepareJobLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
'network_resume',
|
|
|
|
|
$fencedResume,
|
|
|
|
|
1014
|
|
|
|
|
);
|
|
|
|
|
$locked->save();
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$assert(CloudInstanceNetworkPolicy::jobStillDesired(
|
|
|
|
|
(int) $fencedInstance->id,
|
|
|
|
|
'network_resume',
|
|
|
|
|
$fencedResume
|
|
|
|
|
) === true,
|
|
|
|
|
'An available current pool unexpectedly fenced a queued resume.');
|
|
|
|
|
Capsule::table('cloud_traffic_pools')->where('id', 40)->update([
|
|
|
|
|
'state' => 'attention',
|
|
|
|
|
'available_bytes' => 0,
|
|
|
|
|
'version' => 6,
|
|
|
|
|
]);
|
|
|
|
|
$assert(CloudInstanceNetworkPolicy::jobStillDesired(
|
|
|
|
|
(int) $fencedInstance->id,
|
|
|
|
|
'network_resume',
|
|
|
|
|
$fencedResume
|
|
|
|
|
) === false,
|
|
|
|
|
'A queued resume crossed an attention pool before instance convergence.');
|
|
|
|
|
$fencedInstance->getConnection()->transaction(
|
|
|
|
|
static function () use (
|
|
|
|
|
$fencedInstance,
|
|
|
|
|
$fencedPeriod,
|
|
|
|
|
$fencedResume
|
|
|
|
|
): void {
|
|
|
|
|
$locked = CloudInstance::where('id', $fencedInstance->id)
|
|
|
|
|
->lockForUpdate()->firstOrFail();
|
|
|
|
|
$result = CloudInstanceNetworkPolicy::synchronizeLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
$fencedPeriod,
|
|
|
|
|
1015
|
|
|
|
|
);
|
|
|
|
|
if (($result['traffic_decision'] ?? null) !== 'fenced'
|
|
|
|
|
|| CloudInstanceNetworkPolicy::jobStillDesired(
|
|
|
|
|
(int) $fencedInstance->id,
|
|
|
|
|
'network_resume',
|
|
|
|
|
$fencedResume
|
|
|
|
|
) !== false) {
|
|
|
|
|
throw new RuntimeException(
|
|
|
|
|
'Attention did not invalidate a queued automatic resume.'
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
CloudInstanceNetworkPolicy::finishJobLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
'network_resume',
|
|
|
|
|
$fencedResume,
|
|
|
|
|
'succeeded',
|
|
|
|
|
true,
|
|
|
|
|
false,
|
|
|
|
|
1015
|
|
|
|
|
);
|
|
|
|
|
$locked->save();
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$assert((string) CloudInstance::find(11)->network_status === 'blocked_traffic'
|
|
|
|
|
&& (int) CloudInstance::find(11)->network_suspended_at === 900,
|
|
|
|
|
'A queued resume crossed an attention fence or rewrote block timing.');
|
|
|
|
|
|
|
|
|
|
// Unknown effect retains the exact operation fence. A fresh generation is
|
|
|
|
|
// forbidden; resolving that same operation may then converge the projection.
|
|
|
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
|
|
|
$locked = CloudInstance::where('id', $instance->id)
|
|
|
|
|
->lockForUpdate()->firstOrFail();
|
|
|
|
|
foreach ([
|
|
|
|
|
CloudInstanceNetworkPolicy::REASON_ADMIN,
|
|
|
|
|
CloudInstanceNetworkPolicy::REASON_TRAFFIC,
|
|
|
|
|
] as $reason) {
|
|
|
|
|
CloudInstanceNetworkPolicy::setReasonLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
$reason,
|
|
|
|
|
false,
|
|
|
|
|
$reason === CloudInstanceNetworkPolicy::REASON_ADMIN
|
|
|
|
|
? 'administrator' : 'traffic_pool',
|
|
|
|
|
$reason === CloudInstanceNetworkPolicy::REASON_ADMIN
|
|
|
|
|
? 'user:1' : 'pool:40:v4',
|
|
|
|
|
1014
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
$locked->save();
|
|
|
|
|
});
|
|
|
|
|
$unknownOperation = str_repeat('d', 32);
|
|
|
|
|
$instance->getConnection()->transaction(
|
|
|
|
|
static function () use ($instance, $unknownOperation): void {
|
|
|
|
|
$locked = CloudInstance::where('id', $instance->id)
|
|
|
|
|
->lockForUpdate()->firstOrFail();
|
|
|
|
|
CloudInstanceNetworkPolicy::prepareJobLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
'network_resume',
|
|
|
|
|
$unknownOperation,
|
|
|
|
|
1015
|
|
|
|
|
);
|
|
|
|
|
CloudInstanceNetworkPolicy::finishJobLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
'network_resume',
|
|
|
|
|
$unknownOperation,
|
|
|
|
|
'failed',
|
|
|
|
|
false,
|
|
|
|
|
true,
|
|
|
|
|
1016
|
|
|
|
|
);
|
|
|
|
|
$locked->save();
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$unknownState = CloudInstanceNetworkPolicyState::findOrFail(10);
|
|
|
|
|
$assert((string) $unknownState->state === 'unknown'
|
|
|
|
|
&& (string) $unknownState->operation_id === $unknownOperation
|
|
|
|
|
&& (string) CloudInstance::find(10)->network_status === 'restore_failed',
|
|
|
|
|
'An ambiguous resume released its exact operation fence.');
|
|
|
|
|
$assert($instance->getConnection()->transaction(
|
|
|
|
|
static function () use ($instance): bool {
|
|
|
|
|
return CloudInstanceNetworkPolicy::allowRetryLocked(
|
|
|
|
|
CloudInstance::where('id', $instance->id)
|
|
|
|
|
->lockForUpdate()->firstOrFail(),
|
|
|
|
|
'network_resume',
|
|
|
|
|
1017
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
) === false, 'An unknown-effect operation was converted into a fresh retry.');
|
|
|
|
|
$instance->getConnection()->transaction(
|
|
|
|
|
static function () use ($instance, $unknownOperation): void {
|
|
|
|
|
$locked = CloudInstance::where('id', $instance->id)
|
|
|
|
|
->lockForUpdate()->firstOrFail();
|
|
|
|
|
CloudInstanceNetworkPolicy::finishJobLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
'network_resume',
|
|
|
|
|
$unknownOperation,
|
|
|
|
|
'succeeded',
|
|
|
|
|
false,
|
|
|
|
|
false,
|
|
|
|
|
1018
|
|
|
|
|
);
|
|
|
|
|
$locked->save();
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$assert((string) CloudInstance::find(10)->network_status === 'active',
|
|
|
|
|
'Resolving the original resume operation did not restore the projection.');
|
|
|
|
|
|
|
|
|
|
// A policy mutation skipped before dispatch changes no remote state.
|
|
|
|
|
$skippedOperation = str_repeat('e', 32);
|
|
|
|
|
$instance->getConnection()->transaction(
|
|
|
|
|
static function () use ($instance, $skippedOperation): void {
|
|
|
|
|
$locked = CloudInstance::where('id', $instance->id)
|
|
|
|
|
->lockForUpdate()->firstOrFail();
|
|
|
|
|
CloudInstanceNetworkPolicy::setReasonLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
CloudInstanceNetworkPolicy::REASON_ADMIN,
|
|
|
|
|
true,
|
|
|
|
|
'administrator',
|
|
|
|
|
'user:1',
|
|
|
|
|
1019
|
|
|
|
|
);
|
|
|
|
|
CloudInstanceNetworkPolicy::prepareJobLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
'network_suspend',
|
|
|
|
|
$skippedOperation,
|
|
|
|
|
1019
|
|
|
|
|
);
|
|
|
|
|
CloudInstanceNetworkPolicy::setReasonLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
CloudInstanceNetworkPolicy::REASON_ADMIN,
|
|
|
|
|
false,
|
|
|
|
|
'administrator',
|
|
|
|
|
'user:1',
|
|
|
|
|
1020
|
|
|
|
|
);
|
|
|
|
|
CloudInstanceNetworkPolicy::finishJobLocked(
|
|
|
|
|
$locked,
|
|
|
|
|
'network_suspend',
|
|
|
|
|
$skippedOperation,
|
|
|
|
|
'succeeded',
|
|
|
|
|
true,
|
|
|
|
|
false,
|
|
|
|
|
1021
|
|
|
|
|
);
|
|
|
|
|
$locked->save();
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$assert((string) CloudInstance::find(10)->network_status === 'active'
|
|
|
|
|
&& (bool) CloudInstanceNetworkPolicyState::findOrFail(10)
|
|
|
|
|
->applied_suspended === false,
|
|
|
|
|
'A pre-dispatch skip was finalized as a real network suspension.');
|
|
|
|
|
$instance->getConnection()->transaction(static function () use ($instance): void {
|
|
|
|
|
CloudInstanceNetworkPolicy::releaseAfterInstanceDeletedLocked(
|
|
|
|
|
CloudInstance::where('id', $instance->id)
|
|
|
|
|
->lockForUpdate()->firstOrFail()
|
|
|
|
|
);
|
|
|
|
|
});
|
|
|
|
|
$fencedInstance->getConnection()->transaction(
|
|
|
|
|
static function () use ($fencedInstance): void {
|
|
|
|
|
CloudInstanceNetworkPolicy::releaseAfterInstanceDeletedLocked(
|
|
|
|
|
CloudInstance::where('id', $fencedInstance->id)
|
|
|
|
|
->lockForUpdate()->firstOrFail()
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
$assert(CloudInstanceNetworkPolicyState::count() === 0
|
|
|
|
|
&& CloudInstanceNetworkRestriction::count() === 0,
|
|
|
|
|
'Deleting an instance left aggregate network-policy state behind.');
|
|
|
|
|
|
|
|
|
|
echo "Cloud instance network policy checks passed.\n";
|
|
|
|
|
|