|
|
|
|
@ -1092,6 +1092,15 @@ final class CloudVpcService
|
|
|
|
|
[$operation, $payload, $runtimeProofRefresh, $previousOutcomePending] = $claim;
|
|
|
|
|
$leaseToken = (string) $operation->lease_token;
|
|
|
|
|
try {
|
|
|
|
|
// A previous apply may have finished while its PHP receipt was
|
|
|
|
|
// lost. Recover that exact proof before requiring a stopped guest
|
|
|
|
|
// for another mutation; a newer manual start must remain intact.
|
|
|
|
|
if (self::recoverPowerConfiguration(
|
|
|
|
|
$operation, $payload, $leaseToken, $actor, $allowFailedRetry
|
|
|
|
|
)) {
|
|
|
|
|
CloudVpcPowerWorkflow::afterApply($operation, $leaseToken, $actor);
|
|
|
|
|
return self::show($vpcKey, $actor);
|
|
|
|
|
}
|
|
|
|
|
if (!CloudVpcPowerWorkflow::beforeApply(
|
|
|
|
|
$operation, $leaseToken, $actor, $allowFailedRetry
|
|
|
|
|
)) {
|
|
|
|
|
@ -1131,53 +1140,7 @@ final class CloudVpcService
|
|
|
|
|
null, self::ENGINE_TIMEOUT_SECONDS
|
|
|
|
|
))->vpc('POST', '/internal/v1/vpcs/apply',
|
|
|
|
|
$actor, $payload, (string) $operation->operation_id);
|
|
|
|
|
$expectedRevision = (int) ($payload['desired_revision'] ?? 0);
|
|
|
|
|
$expectedDigest = (string) ($payload['desired_digest'] ?? '');
|
|
|
|
|
if (!is_array($result)
|
|
|
|
|
|| (string) ($result['operation_id'] ?? '')
|
|
|
|
|
!== (string) $operation->operation_id
|
|
|
|
|
|| (string) ($result['vpc_id'] ?? '')
|
|
|
|
|
!== (string) ($payload['vpc_id'] ?? '')
|
|
|
|
|
|| (int) ($result['desired_revision'] ?? 0) !== $expectedRevision
|
|
|
|
|
|| !hash_equals($expectedDigest, (string) ($result['desired_digest'] ?? ''))) {
|
|
|
|
|
throw new CloudError('引擎 VPC 结果与操作编号不一致,正在核对', 502);
|
|
|
|
|
}
|
|
|
|
|
$observed = is_array($result['observed'] ?? null) ? $result['observed'] : $result;
|
|
|
|
|
// Some engine versions expose a compact `observed` projection and
|
|
|
|
|
// the full typed host readback beside it. Merge the latter into
|
|
|
|
|
// the bounded projection before validating; never discard
|
|
|
|
|
// member_nics/routes/WireGuard evidence merely because the
|
|
|
|
|
// transport envelope used a compact view.
|
|
|
|
|
if (is_array($result['hosts'] ?? null)) {
|
|
|
|
|
if (!is_array($observed['hosts'] ?? null)) {
|
|
|
|
|
$observed['hosts'] = [];
|
|
|
|
|
}
|
|
|
|
|
foreach ($result['hosts'] as $hostKey => $hostItem) {
|
|
|
|
|
if (!is_array($hostItem)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
$hostKey = (string) ($hostItem['host_id'] ?? $hostKey);
|
|
|
|
|
$compact = is_array($observed['hosts'][$hostKey] ?? null)
|
|
|
|
|
? $observed['hosts'][$hostKey] : [];
|
|
|
|
|
$observed['hosts'][$hostKey] = array_replace($compact, $hostItem);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (isset($observed['operation_id'])
|
|
|
|
|
&& (string) $observed['operation_id'] !== (string) $operation->operation_id) {
|
|
|
|
|
throw new CloudError('引擎 VPC 观测与操作编号不一致,正在核对', 502);
|
|
|
|
|
}
|
|
|
|
|
if (isset($observed['vpc_id'])
|
|
|
|
|
&& (string) $observed['vpc_id'] !== (string) ($payload['vpc_id'] ?? '') ) {
|
|
|
|
|
throw new CloudError('引擎 VPC 观测目标不一致,正在核对', 502);
|
|
|
|
|
}
|
|
|
|
|
if (isset($observed['desired_revision'])
|
|
|
|
|
&& (int) $observed['desired_revision'] !== $expectedRevision) {
|
|
|
|
|
throw new CloudError('引擎 VPC 观测版本不一致,正在核对', 502);
|
|
|
|
|
}
|
|
|
|
|
if (isset($observed['desired_digest'])
|
|
|
|
|
&& !hash_equals($expectedDigest, (string) $observed['desired_digest'])) {
|
|
|
|
|
throw new CloudError('引擎 VPC 观测摘要不一致,正在核对', 502);
|
|
|
|
|
}
|
|
|
|
|
$observed = self::applyObservation($operation, $payload, $result);
|
|
|
|
|
self::recordObserved($vpcKey, (int) ($payload['desired_revision'] ?? 0),
|
|
|
|
|
(string) ($payload['desired_digest'] ?? ''), $observed,
|
|
|
|
|
(string) $operation->operation_id, $leaseToken);
|
|
|
|
|
@ -1464,9 +1427,98 @@ final class CloudVpcService
|
|
|
|
|
}, 3);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Read only the immutable target of an interrupted member apply. */
|
|
|
|
|
private static function recoverPowerConfiguration(CloudVpcOperation $operation,
|
|
|
|
|
array $payload, string $lease, array $actor, bool $explicitRetry): bool
|
|
|
|
|
{
|
|
|
|
|
$state = CloudVpcPowerWorkflow::state($operation);
|
|
|
|
|
if ($state === null || !($state['phase'] === 'applying'
|
|
|
|
|
|| ($explicitRetry && $state['phase'] === 'paused'
|
|
|
|
|
&& $state['paused_from'] === 'applying'))) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
$result = (new CloudEngineClient(null, 20))->vpc(
|
|
|
|
|
'GET', '/internal/v1/vpcs/' . $payload['vpc_id'], $actor
|
|
|
|
|
);
|
|
|
|
|
} catch (CloudError $error) {
|
|
|
|
|
if ($error->httpStatus === 404) {
|
|
|
|
|
return false; // No receipt: the original stopped-guest gate applies.
|
|
|
|
|
}
|
|
|
|
|
throw $error;
|
|
|
|
|
}
|
|
|
|
|
if (($result['vpc_id'] ?? null) === $payload['vpc_id']
|
|
|
|
|
&& is_int($result['desired_revision'] ?? null)
|
|
|
|
|
&& $result['desired_revision'] > 0
|
|
|
|
|
&& $result['desired_revision'] < $payload['desired_revision']) {
|
|
|
|
|
return false; // This target has not reached the engine yet.
|
|
|
|
|
}
|
|
|
|
|
$observed = self::applyObservation($operation, $payload, $result);
|
|
|
|
|
self::recordObserved($payload['vpc_id'], $payload['desired_revision'],
|
|
|
|
|
$payload['desired_digest'], $observed, (string) $operation->operation_id,
|
|
|
|
|
$lease, true);
|
|
|
|
|
$operation->refresh();
|
|
|
|
|
return (CloudVpcPowerWorkflow::state($operation)['config_confirmed'] ?? false) === true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** GET recovery and POST apply must consume the same exact evidence. */
|
|
|
|
|
private static function applyObservation(CloudVpcOperation $operation,
|
|
|
|
|
array $payload, array $result): array
|
|
|
|
|
{
|
|
|
|
|
$expectedRevision = (int) ($payload['desired_revision'] ?? 0);
|
|
|
|
|
$expectedDigest = (string) ($payload['desired_digest'] ?? '');
|
|
|
|
|
if (!is_array($result)
|
|
|
|
|
|| (string) ($result['operation_id'] ?? '')
|
|
|
|
|
!== (string) $operation->operation_id
|
|
|
|
|
|| (string) ($result['vpc_id'] ?? '')
|
|
|
|
|
!== (string) ($payload['vpc_id'] ?? '')
|
|
|
|
|
|| ($result['desired_revision'] ?? null) !== $expectedRevision
|
|
|
|
|
|| !hash_equals($expectedDigest, (string) ($result['desired_digest'] ?? ''))) {
|
|
|
|
|
throw new CloudError('引擎 VPC 结果与操作编号不一致,正在核对', 502);
|
|
|
|
|
}
|
|
|
|
|
$observed = is_array($result['observed'] ?? null) ? $result['observed'] : $result;
|
|
|
|
|
// Some engine versions expose a compact `observed` projection and
|
|
|
|
|
// the full typed host readback beside it. Merge the latter into
|
|
|
|
|
// the bounded projection before validating; never discard
|
|
|
|
|
// member_nics/routes/WireGuard evidence merely because the
|
|
|
|
|
// transport envelope used a compact view.
|
|
|
|
|
if (is_array($result['hosts'] ?? null)) {
|
|
|
|
|
if (!is_array($observed['hosts'] ?? null)) {
|
|
|
|
|
$observed['hosts'] = [];
|
|
|
|
|
}
|
|
|
|
|
foreach ($result['hosts'] as $hostKey => $hostItem) {
|
|
|
|
|
if (!is_array($hostItem)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
$hostKey = (string) ($hostItem['host_id'] ?? $hostKey);
|
|
|
|
|
$compact = is_array($observed['hosts'][$hostKey] ?? null)
|
|
|
|
|
? $observed['hosts'][$hostKey] : [];
|
|
|
|
|
$observed['hosts'][$hostKey] = array_replace($compact, $hostItem);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (isset($observed['operation_id'])
|
|
|
|
|
&& (string) $observed['operation_id'] !== (string) $operation->operation_id) {
|
|
|
|
|
throw new CloudError('引擎 VPC 观测与操作编号不一致,正在核对', 502);
|
|
|
|
|
}
|
|
|
|
|
if (isset($observed['vpc_id'])
|
|
|
|
|
&& (string) $observed['vpc_id'] !== (string) ($payload['vpc_id'] ?? '') ) {
|
|
|
|
|
throw new CloudError('引擎 VPC 观测目标不一致,正在核对', 502);
|
|
|
|
|
}
|
|
|
|
|
if (isset($observed['desired_revision'])
|
|
|
|
|
&& $observed['desired_revision'] !== $expectedRevision) {
|
|
|
|
|
throw new CloudError('引擎 VPC 观测版本不一致,正在核对', 502);
|
|
|
|
|
}
|
|
|
|
|
if (isset($observed['desired_digest'])
|
|
|
|
|
&& !hash_equals($expectedDigest, (string) $observed['desired_digest'])) {
|
|
|
|
|
throw new CloudError('引擎 VPC 观测摘要不一致,正在核对', 502);
|
|
|
|
|
}
|
|
|
|
|
return $observed;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Apply an exact engine observation and enforce the all-host gate. */
|
|
|
|
|
public static function recordObserved(string $vpcKey, int $revision, string $digest,
|
|
|
|
|
array $observed, string $operationId, ?string $leaseToken = null): void
|
|
|
|
|
array $observed, string $operationId, ?string $leaseToken = null,
|
|
|
|
|
bool $recoverPowerConfiguration = false): void
|
|
|
|
|
{
|
|
|
|
|
self::assertKey($vpcKey, 'VPC');
|
|
|
|
|
if ($revision < 1 || preg_match('/^[a-f0-9]{64}$/D', $digest) !== 1
|
|
|
|
|
@ -1475,7 +1527,8 @@ final class CloudVpcService
|
|
|
|
|
}
|
|
|
|
|
$connection = (new CloudVpc())->getConnection();
|
|
|
|
|
$connection->transaction(function () use (
|
|
|
|
|
$vpcKey, $revision, $digest, $observed, $operationId, $leaseToken
|
|
|
|
|
$vpcKey, $revision, $digest, $observed, $operationId, $leaseToken,
|
|
|
|
|
$recoverPowerConfiguration
|
|
|
|
|
): void {
|
|
|
|
|
$vpc = CloudVpc::where('vpc_key', $vpcKey)->lockForUpdate()->first();
|
|
|
|
|
if ($vpc === null) {
|
|
|
|
|
@ -2394,7 +2447,9 @@ final class CloudVpcService
|
|
|
|
|
$operation->updated_at = time();
|
|
|
|
|
$operation->finished_at = ($operationSucceeded || $dataPlaneDeferred || $terminalFailure)
|
|
|
|
|
? time() : null;
|
|
|
|
|
$powerPending = CloudVpcPowerWorkflow::configurationObserved($operation, $operationSucceeded);
|
|
|
|
|
$powerPending = CloudVpcPowerWorkflow::configurationObserved(
|
|
|
|
|
$operation, $operationSucceeded, $recoverPowerConfiguration
|
|
|
|
|
);
|
|
|
|
|
if ($powerPending && $leaseToken !== null) {
|
|
|
|
|
$operation->status = 'running';
|
|
|
|
|
$operation->finished_at = null;
|
|
|
|
|
|