fix(cloud): release restored instance placement on delete

go
codex 3 weeks ago
parent 7d7d1bf5bc
commit f491b99d35

@ -827,6 +827,89 @@ final class CloudPlacementService
if ($reservation === null || (string) $reservation->state === 'released') {
return;
}
self::releaseLockedReservation(
$reservation,
$reason,
$includeCommitted
);
}
/** Release the instance's current placement after proven host deletion. */
public static function releaseDeletedInstance(CloudInstance $instance,
string $reason): void
{
if ((string) $instance->status !== 'deleted') {
throw new CloudError('实例尚未确认删除,不能释放资源', 409);
}
$reservationId = (int) ($instance->reservation_id ?? 0);
if ($reservationId < 1) {
if ($instance->order_id === null) {
return;
}
$order = CloudOrder::where('id', (int) $instance->order_id)
->lockForUpdate()->first();
if ($order === null) {
throw new CloudError('已删除实例的开通订单不存在', 409);
}
self::release($order, $reason, true);
return;
}
$host = CloudHost::where('id', (int) $instance->host_id)
->lockForUpdate()->first();
$reservation = CloudResourceReservation::where('id', $reservationId)
->lockForUpdate()->first();
if ($host === null || $reservation === null
|| (int) $reservation->host_id !== (int) $host->id
|| (int) $reservation->instance_id !== (int) $instance->id
|| (int) $reservation->resource_pool_id
!== (int) ($instance->resource_pool_id ?? 0)) {
throw new CloudError('已删除实例的资源预留账本不一致', 409);
}
$orderId = (int) ($reservation->order_id ?? 0);
$restoreId = (int) ($reservation->backup_restore_id ?? 0);
if ($orderId > 0 && $restoreId === 0
&& (int) ($reservation->offer_id ?? 0) > 0) {
$order = CloudOrder::where('id', $orderId)
->lockForUpdate()->first();
if ($order === null
|| (int) ($instance->order_id ?? 0) !== (int) $order->id
|| (int) ($order->reservation_id ?? 0)
!== (int) $reservation->id
|| (int) ($order->host_id ?? 0) !== (int) $instance->host_id) {
throw new CloudError('已删除实例的订单资源账本不一致', 409);
}
} elseif ($orderId === 0 && $restoreId > 0
&& (int) ($reservation->offer_id ?? 0) === 0) {
$restore = CloudBackupRestore::where('id', $restoreId)
->lockForUpdate()->first();
if ($restore === null || $instance->order_id !== null
|| (int) $restore->target_host_id !== (int) $instance->host_id
|| (int) $restore->target_instance_id !== (int) $instance->id
|| (int) $restore->reservation_id !== (int) $reservation->id
|| (string) $restore->state !== 'ready'
|| !hash_equals(
(string) $restore->operation_id,
(string) $reservation->operation_id
)) {
throw new CloudError('已删除恢复实例的资源账本不一致', 409);
}
} else {
throw new CloudError('已删除实例的资源来源无效', 409);
}
self::releaseLockedReservation($reservation, $reason, true);
}
private static function releaseLockedReservation(
CloudResourceReservation $reservation,
string $reason,
bool $includeCommitted
): void {
if ((string) $reservation->state === 'released') {
return;
}
if ((string) $reservation->state === 'committed' && !$includeCommitted) {
throw new CloudError('已交付实例的资源不能按失败任务释放', 409);
}

@ -2861,15 +2861,10 @@ final class CloudWorkerService
['delete', 'expiry_delete', 'delete_force'], true)) {
$instance->status = 'deleted';
$instance->sealed_credential = null;
if ($instance->order_id !== null) {
$provisionOrder = CloudOrder::where('id', $instance->order_id)
->lockForUpdate()->first();
if ($provisionOrder !== null) {
CloudPlacementService::release(
$provisionOrder, 'instance_deleted', true
);
}
}
CloudPlacementService::releaseDeletedInstance(
$instance,
'instance_deleted'
);
} elseif ($job->kind === 'instance_settings') {
// The Agent must acknowledge the controlled seed
// replacement before PHP changes any cached metadata.

@ -91,6 +91,7 @@ namespace {
use App\Models\CloudHostCapacityPolicy;
use App\Models\CloudHostNetworkAttachment;
use App\Models\CloudHostResourcePoolMembership;
use App\Models\CloudBackupRestore;
use App\Models\CloudInstance;
use App\Models\CloudInvoice;
use App\Models\CloudInvoicePayment;
@ -250,14 +251,15 @@ namespace {
$schema->create('cloud_platform_test_policy_state', function (Blueprint $t) { $t->integer('id')->primary(); $t->integer('revision')->default(0); $t->integer('updated_by')->nullable(); $t->integer('updated_at')->default(0); });
$schema->create('cloud_platform_test_policy_rules', function (Blueprint $t) { $t->increments('id'); $t->string('rule_key', 32)->unique(); $t->string('service', 120); $t->string('protocol', 8); $t->integer('port_from'); $t->integer('port_to'); $t->string('direction', 8); $t->string('reason', 500); $t->integer('created_by'); $t->integer('updated_by'); $t->integer('deleted_by')->nullable(); $t->integer('deleted_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_orders', function (Blueprint $t) { $t->increments('id'); foreach (['user_id', 'plan_id', 'offer_id', 'resource_pool_id'] as $v) { $t->integer($v); } $t->integer('host_id')->nullable(); $t->integer('reservation_id')->nullable()->unique(); $t->string('idempotency_key'); $t->string('request_hash'); $t->string('amount'); $t->string('currency')->default('USD'); $t->string('status'); $t->text('spec_json'); $t->text('sealed_payload')->nullable(); $t->integer('instance_id')->nullable(); $t->integer('refunded_at')->nullable(); $t->integer('stock_reserved_at')->nullable(); $t->integer('stock_released_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['user_id', 'idempotency_key']); });
$schema->create('cloud_resource_reservations', function (Blueprint $t) { $t->increments('id'); $t->string('reservation_key')->unique(); $t->string('operation_id')->unique(); foreach (['order_id', 'offer_id', 'resource_pool_id', 'host_id', 'user_id'] as $v) { $t->integer($v); } $t->integer('instance_id')->nullable(); $t->integer('network_attachment_id')->nullable(); $t->integer('public_ip_pool_id')->nullable(); $t->integer('public_ip_id')->nullable(); foreach (['cpu_cores', 'memory_mib', 'disk_gib'] as $v) { $t->integer($v); } $t->string('state'); $t->integer('expires_at')->nullable(); $t->integer('released_at')->nullable(); $t->string('release_reason')->default(''); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); });
$schema->create('cloud_resource_reservations', function (Blueprint $t) { $t->increments('id'); $t->string('reservation_key')->unique(); $t->string('operation_id')->unique(); $t->integer('order_id')->nullable(); $t->integer('backup_restore_id')->nullable(); $t->integer('offer_id')->nullable(); foreach (['resource_pool_id', 'host_id', 'user_id'] as $v) { $t->integer($v); } $t->integer('instance_id')->nullable(); $t->integer('network_attachment_id')->nullable(); $t->integer('public_ip_pool_id')->nullable(); $t->integer('public_ip_id')->nullable(); foreach (['cpu_cores', 'memory_mib', 'disk_gib'] as $v) { $t->integer($v); } $t->string('state'); $t->integer('expires_at')->nullable(); $t->integer('released_at')->nullable(); $t->string('release_reason')->default(''); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); $t->unique('backup_restore_id'); });
$schema->create('cloud_backup_restores', function (Blueprint $t) { $t->increments('id'); $t->string('restore_key')->unique(); $t->integer('backup_id'); $t->integer('target_host_id'); $t->integer('target_instance_id')->nullable(); $t->integer('reservation_id')->nullable()->unique(); $t->integer('job_id')->nullable(); $t->string('operation_id')->unique(); $t->string('new_uuid'); $t->string('new_name'); $t->string('state'); $t->string('error_code')->default(''); $t->integer('ready_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); });
$schema->create('cloud_jobs', function (Blueprint $t) {
$t->increments('id');
$t->string('operation_id')->unique();
$t->string('request_idempotency_key');
$t->string('request_fingerprint');
foreach (['parent_job_id', 'order_id', 'invoice_id', 'instance_id',
'snapshot_id'] as $v) {
'snapshot_id', 'backup_id', 'backup_restore_id'] as $v) {
$t->integer($v)->nullable();
}
$t->integer('user_id');
@ -1562,6 +1564,7 @@ namespace {
string $serviceStatus, string $networkStatus) use ($instance) {
$copy = $instance->replicate();
$copy->order_id = null;
$copy->reservation_id = null;
$copy->current_period_id = null;
$copy->uuid = $uuid;
$copy->name = $name;
@ -2463,6 +2466,127 @@ namespace {
&& (int) $forceDelete->fresh()->dispatch_attempts === 1,
'Successful force-delete task was automatically re-dispatched.');
// Independent backup restores have no purchase order. Their committed
// capacity and fresh IP must still be released after a proven deletion.
$restoreOperationId = str_repeat('e', 32);
$restoredUuid = 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee';
$backupRestore = new CloudBackupRestore();
$backupRestore->restore_key = str_repeat('f', 32);
$backupRestore->backup_id = 9001;
$backupRestore->target_host_id = (int) $host->id;
$backupRestore->target_instance_id = null;
$backupRestore->reservation_id = null;
$backupRestore->job_id = null;
$backupRestore->operation_id = $restoreOperationId;
$backupRestore->new_uuid = $restoredUuid;
$backupRestore->new_name = 'restored-delete-vm';
$backupRestore->state = 'ready';
$backupRestore->error_code = '';
$backupRestore->ready_at = time();
$backupRestore->created_at = $backupRestore->updated_at = time();
$backupRestore->save();
$restoredIp = new CloudPublicIp();
$restoredIp->host_id = (int) $host->id;
$restoredIp->public_ip_pool_id = (int) $targetPublicPool->id;
$restoredIp->address = '203.0.113.36';
$restoredIp->prefix_length = 28;
$restoredIp->state = 'assigned';
$restoredIp->reservation_id = null;
$restoredIp->instance_id = null;
$restoredIp->operation_id = null;
$restoredIp->created_at = $restoredIp->updated_at = time();
$restoredIp->save();
$restoredReservation = new CloudResourceReservation();
$restoredReservation->reservation_key = str_repeat('d', 32);
$restoredReservation->operation_id = $restoreOperationId;
$restoredReservation->order_id = null;
$restoredReservation->backup_restore_id = (int) $backupRestore->id;
$restoredReservation->offer_id = null;
$restoredReservation->resource_pool_id = (int) $pool->id;
$restoredReservation->host_id = (int) $host->id;
$restoredReservation->user_id = 1;
$restoredReservation->instance_id = null;
$restoredReservation->network_attachment_id = (int) $attachment->id;
$restoredReservation->public_ip_pool_id = (int) $targetPublicPool->id;
$restoredReservation->public_ip_id = (int) $restoredIp->id;
$restoredReservation->cpu_cores = 2;
$restoredReservation->memory_mib = 2048;
$restoredReservation->disk_gib = 20;
$restoredReservation->state = 'committed';
$restoredReservation->expires_at = null;
$restoredReservation->released_at = null;
$restoredReservation->release_reason = '';
$restoredReservation->created_at = $restoredReservation->updated_at = time();
$restoredReservation->save();
$restoredInstance = $instance->replicate();
$restoredInstance->uuid = $restoredUuid;
$restoredInstance->host_id = (int) $host->id;
$restoredInstance->user_id = 1;
$restoredInstance->order_id = null;
$restoredInstance->plan_id = null;
$restoredInstance->offer_id = null;
$restoredInstance->resource_pool_id = (int) $pool->id;
$restoredInstance->reservation_id = (int) $restoredReservation->id;
$restoredInstance->current_period_id = null;
$restoredInstance->name = 'restored-delete-vm';
$restoredInstance->status = 'active';
$restoredInstance->service_status = 'exempt';
$restoredInstance->network_status = 'active';
$restoredInstance->expires_at = null;
$restoredInstance->spec_json = json_encode([
'managed' => true,
'vcpu' => 2,
'memory_mib' => 2048,
'disk_gib' => 20,
'public_ip_cidr' => '203.0.113.36/28',
], JSON_THROW_ON_ERROR);
$restoredInstance->sealed_credential = 'sealed-restored-credential';
$restoredInstance->created_at = $restoredInstance->updated_at = time();
$restoredInstance->save();
$restoredReservation->instance_id = (int) $restoredInstance->id;
$restoredReservation->save();
$restoredIp->reservation_id = (int) $restoredReservation->id;
$restoredIp->instance_id = (int) $restoredInstance->id;
$restoredIp->operation_id = $restoreOperationId;
$restoredIp->save();
$backupRestore->target_instance_id = (int) $restoredInstance->id;
$backupRestore->reservation_id = (int) $restoredReservation->id;
$backupRestore->save();
$restoredDelete = CloudJobService::enqueue(
$restoredInstance,
$adminActor,
'delete',
'DELETE',
'',
['remove_managed_disks' => true]
);
$mock->append(new Response(200, [], json_encode([
'operation_id' => $restoredDelete->operation_id,
'status' => 'succeeded',
...CloudOperationRecoveryIdentity::expected($restoredDelete, true),
'result' => [
'uuid' => $restoredUuid,
'deleted' => true,
'managed_artifacts_absent' => true,
'forced' => false,
],
])));
$worker->tick();
$restoredDelete->refresh();
$restoredInstance->refresh();
$restoredReservation->refresh();
$restoredIp->refresh();
$assert($restoredDelete->phase === 'terminal'
&& $restoredDelete->outcome === 'succeeded'
&& $restoredInstance->status === 'deleted'
&& $restoredReservation->state === 'released'
&& $restoredReservation->release_reason === 'instance_deleted'
&& $restoredIp->state === 'available'
&& $restoredIp->reservation_id === null
&& $restoredIp->instance_id === null
&& $restoredIp->operation_id === null,
'Deleting an independent restored instance leaked capacity or its public IP.');
// Provider callbacks credit the wallet before the invoice bridge runs.
// That credit must be consumed once, queue one create job, and remain

Loading…
Cancel
Save