diff --git a/web/app/Services/Cloud/CloudPlacementService.php b/web/app/Services/Cloud/CloudPlacementService.php index 28d8e585..56b63943 100644 --- a/web/app/Services/Cloud/CloudPlacementService.php +++ b/web/app/Services/Cloud/CloudPlacementService.php @@ -827,6 +827,89 @@ final class CloudPlacementService if ($reservation === null || (string) $reservation->state === 'released') { return; } + self::releaseLockedReservation( + $reservation, + $reason, + $includeCommitted + ); + } + + /** Release the instance's current placement after proven host deletion. */ + public static function releaseDeletedInstance(CloudInstance $instance, + string $reason): void + { + if ((string) $instance->status !== 'deleted') { + throw new CloudError('实例尚未确认删除,不能释放资源', 409); + } + $reservationId = (int) ($instance->reservation_id ?? 0); + if ($reservationId < 1) { + if ($instance->order_id === null) { + return; + } + $order = CloudOrder::where('id', (int) $instance->order_id) + ->lockForUpdate()->first(); + if ($order === null) { + throw new CloudError('已删除实例的开通订单不存在', 409); + } + self::release($order, $reason, true); + return; + } + + $host = CloudHost::where('id', (int) $instance->host_id) + ->lockForUpdate()->first(); + $reservation = CloudResourceReservation::where('id', $reservationId) + ->lockForUpdate()->first(); + if ($host === null || $reservation === null + || (int) $reservation->host_id !== (int) $host->id + || (int) $reservation->instance_id !== (int) $instance->id + || (int) $reservation->resource_pool_id + !== (int) ($instance->resource_pool_id ?? 0)) { + throw new CloudError('已删除实例的资源预留账本不一致', 409); + } + + $orderId = (int) ($reservation->order_id ?? 0); + $restoreId = (int) ($reservation->backup_restore_id ?? 0); + if ($orderId > 0 && $restoreId === 0 + && (int) ($reservation->offer_id ?? 0) > 0) { + $order = CloudOrder::where('id', $orderId) + ->lockForUpdate()->first(); + if ($order === null + || (int) ($instance->order_id ?? 0) !== (int) $order->id + || (int) ($order->reservation_id ?? 0) + !== (int) $reservation->id + || (int) ($order->host_id ?? 0) !== (int) $instance->host_id) { + throw new CloudError('已删除实例的订单资源账本不一致', 409); + } + } elseif ($orderId === 0 && $restoreId > 0 + && (int) ($reservation->offer_id ?? 0) === 0) { + $restore = CloudBackupRestore::where('id', $restoreId) + ->lockForUpdate()->first(); + if ($restore === null || $instance->order_id !== null + || (int) $restore->target_host_id !== (int) $instance->host_id + || (int) $restore->target_instance_id !== (int) $instance->id + || (int) $restore->reservation_id !== (int) $reservation->id + || (string) $restore->state !== 'ready' + || !hash_equals( + (string) $restore->operation_id, + (string) $reservation->operation_id + )) { + throw new CloudError('已删除恢复实例的资源账本不一致', 409); + } + } else { + throw new CloudError('已删除实例的资源来源无效', 409); + } + + self::releaseLockedReservation($reservation, $reason, true); + } + + private static function releaseLockedReservation( + CloudResourceReservation $reservation, + string $reason, + bool $includeCommitted + ): void { + if ((string) $reservation->state === 'released') { + return; + } if ((string) $reservation->state === 'committed' && !$includeCommitted) { throw new CloudError('已交付实例的资源不能按失败任务释放', 409); } diff --git a/web/app/Services/Cloud/CloudWorkerService.php b/web/app/Services/Cloud/CloudWorkerService.php index 803c94ca..4c05f0b7 100644 --- a/web/app/Services/Cloud/CloudWorkerService.php +++ b/web/app/Services/Cloud/CloudWorkerService.php @@ -2861,15 +2861,10 @@ final class CloudWorkerService ['delete', 'expiry_delete', 'delete_force'], true)) { $instance->status = 'deleted'; $instance->sealed_credential = null; - if ($instance->order_id !== null) { - $provisionOrder = CloudOrder::where('id', $instance->order_id) - ->lockForUpdate()->first(); - if ($provisionOrder !== null) { - CloudPlacementService::release( - $provisionOrder, 'instance_deleted', true - ); - } - } + CloudPlacementService::releaseDeletedInstance( + $instance, + 'instance_deleted' + ); } elseif ($job->kind === 'instance_settings') { // The Agent must acknowledge the controlled seed // replacement before PHP changes any cached metadata. diff --git a/web/tests/CloudFinancialWorkflowTest.php b/web/tests/CloudFinancialWorkflowTest.php index a17d5281..88f81b9e 100644 --- a/web/tests/CloudFinancialWorkflowTest.php +++ b/web/tests/CloudFinancialWorkflowTest.php @@ -91,6 +91,7 @@ namespace { use App\Models\CloudHostCapacityPolicy; use App\Models\CloudHostNetworkAttachment; use App\Models\CloudHostResourcePoolMembership; + use App\Models\CloudBackupRestore; use App\Models\CloudInstance; use App\Models\CloudInvoice; use App\Models\CloudInvoicePayment; @@ -250,14 +251,15 @@ namespace { $schema->create('cloud_platform_test_policy_state', function (Blueprint $t) { $t->integer('id')->primary(); $t->integer('revision')->default(0); $t->integer('updated_by')->nullable(); $t->integer('updated_at')->default(0); }); $schema->create('cloud_platform_test_policy_rules', function (Blueprint $t) { $t->increments('id'); $t->string('rule_key', 32)->unique(); $t->string('service', 120); $t->string('protocol', 8); $t->integer('port_from'); $t->integer('port_to'); $t->string('direction', 8); $t->string('reason', 500); $t->integer('created_by'); $t->integer('updated_by'); $t->integer('deleted_by')->nullable(); $t->integer('deleted_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_orders', function (Blueprint $t) { $t->increments('id'); foreach (['user_id', 'plan_id', 'offer_id', 'resource_pool_id'] as $v) { $t->integer($v); } $t->integer('host_id')->nullable(); $t->integer('reservation_id')->nullable()->unique(); $t->string('idempotency_key'); $t->string('request_hash'); $t->string('amount'); $t->string('currency')->default('USD'); $t->string('status'); $t->text('spec_json'); $t->text('sealed_payload')->nullable(); $t->integer('instance_id')->nullable(); $t->integer('refunded_at')->nullable(); $t->integer('stock_reserved_at')->nullable(); $t->integer('stock_released_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); $t->unique(['user_id', 'idempotency_key']); }); - $schema->create('cloud_resource_reservations', function (Blueprint $t) { $t->increments('id'); $t->string('reservation_key')->unique(); $t->string('operation_id')->unique(); foreach (['order_id', 'offer_id', 'resource_pool_id', 'host_id', 'user_id'] as $v) { $t->integer($v); } $t->integer('instance_id')->nullable(); $t->integer('network_attachment_id')->nullable(); $t->integer('public_ip_pool_id')->nullable(); $t->integer('public_ip_id')->nullable(); foreach (['cpu_cores', 'memory_mib', 'disk_gib'] as $v) { $t->integer($v); } $t->string('state'); $t->integer('expires_at')->nullable(); $t->integer('released_at')->nullable(); $t->string('release_reason')->default(''); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); }); + $schema->create('cloud_resource_reservations', function (Blueprint $t) { $t->increments('id'); $t->string('reservation_key')->unique(); $t->string('operation_id')->unique(); $t->integer('order_id')->nullable(); $t->integer('backup_restore_id')->nullable(); $t->integer('offer_id')->nullable(); foreach (['resource_pool_id', 'host_id', 'user_id'] as $v) { $t->integer($v); } $t->integer('instance_id')->nullable(); $t->integer('network_attachment_id')->nullable(); $t->integer('public_ip_pool_id')->nullable(); $t->integer('public_ip_id')->nullable(); foreach (['cpu_cores', 'memory_mib', 'disk_gib'] as $v) { $t->integer($v); } $t->string('state'); $t->integer('expires_at')->nullable(); $t->integer('released_at')->nullable(); $t->string('release_reason')->default(''); $t->integer('created_at'); $t->integer('updated_at'); $t->unique('order_id'); $t->unique('backup_restore_id'); }); + $schema->create('cloud_backup_restores', function (Blueprint $t) { $t->increments('id'); $t->string('restore_key')->unique(); $t->integer('backup_id'); $t->integer('target_host_id'); $t->integer('target_instance_id')->nullable(); $t->integer('reservation_id')->nullable()->unique(); $t->integer('job_id')->nullable(); $t->string('operation_id')->unique(); $t->string('new_uuid'); $t->string('new_name'); $t->string('state'); $t->string('error_code')->default(''); $t->integer('ready_at')->nullable(); $t->integer('created_at'); $t->integer('updated_at'); }); $schema->create('cloud_jobs', function (Blueprint $t) { $t->increments('id'); $t->string('operation_id')->unique(); $t->string('request_idempotency_key'); $t->string('request_fingerprint'); foreach (['parent_job_id', 'order_id', 'invoice_id', 'instance_id', - 'snapshot_id'] as $v) { + 'snapshot_id', 'backup_id', 'backup_restore_id'] as $v) { $t->integer($v)->nullable(); } $t->integer('user_id'); @@ -1562,6 +1564,7 @@ namespace { string $serviceStatus, string $networkStatus) use ($instance) { $copy = $instance->replicate(); $copy->order_id = null; + $copy->reservation_id = null; $copy->current_period_id = null; $copy->uuid = $uuid; $copy->name = $name; @@ -2463,6 +2466,127 @@ namespace { && (int) $forceDelete->fresh()->dispatch_attempts === 1, 'Successful force-delete task was automatically re-dispatched.'); + // Independent backup restores have no purchase order. Their committed + // capacity and fresh IP must still be released after a proven deletion. + $restoreOperationId = str_repeat('e', 32); + $restoredUuid = 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee'; + $backupRestore = new CloudBackupRestore(); + $backupRestore->restore_key = str_repeat('f', 32); + $backupRestore->backup_id = 9001; + $backupRestore->target_host_id = (int) $host->id; + $backupRestore->target_instance_id = null; + $backupRestore->reservation_id = null; + $backupRestore->job_id = null; + $backupRestore->operation_id = $restoreOperationId; + $backupRestore->new_uuid = $restoredUuid; + $backupRestore->new_name = 'restored-delete-vm'; + $backupRestore->state = 'ready'; + $backupRestore->error_code = ''; + $backupRestore->ready_at = time(); + $backupRestore->created_at = $backupRestore->updated_at = time(); + $backupRestore->save(); + $restoredIp = new CloudPublicIp(); + $restoredIp->host_id = (int) $host->id; + $restoredIp->public_ip_pool_id = (int) $targetPublicPool->id; + $restoredIp->address = '203.0.113.36'; + $restoredIp->prefix_length = 28; + $restoredIp->state = 'assigned'; + $restoredIp->reservation_id = null; + $restoredIp->instance_id = null; + $restoredIp->operation_id = null; + $restoredIp->created_at = $restoredIp->updated_at = time(); + $restoredIp->save(); + $restoredReservation = new CloudResourceReservation(); + $restoredReservation->reservation_key = str_repeat('d', 32); + $restoredReservation->operation_id = $restoreOperationId; + $restoredReservation->order_id = null; + $restoredReservation->backup_restore_id = (int) $backupRestore->id; + $restoredReservation->offer_id = null; + $restoredReservation->resource_pool_id = (int) $pool->id; + $restoredReservation->host_id = (int) $host->id; + $restoredReservation->user_id = 1; + $restoredReservation->instance_id = null; + $restoredReservation->network_attachment_id = (int) $attachment->id; + $restoredReservation->public_ip_pool_id = (int) $targetPublicPool->id; + $restoredReservation->public_ip_id = (int) $restoredIp->id; + $restoredReservation->cpu_cores = 2; + $restoredReservation->memory_mib = 2048; + $restoredReservation->disk_gib = 20; + $restoredReservation->state = 'committed'; + $restoredReservation->expires_at = null; + $restoredReservation->released_at = null; + $restoredReservation->release_reason = ''; + $restoredReservation->created_at = $restoredReservation->updated_at = time(); + $restoredReservation->save(); + $restoredInstance = $instance->replicate(); + $restoredInstance->uuid = $restoredUuid; + $restoredInstance->host_id = (int) $host->id; + $restoredInstance->user_id = 1; + $restoredInstance->order_id = null; + $restoredInstance->plan_id = null; + $restoredInstance->offer_id = null; + $restoredInstance->resource_pool_id = (int) $pool->id; + $restoredInstance->reservation_id = (int) $restoredReservation->id; + $restoredInstance->current_period_id = null; + $restoredInstance->name = 'restored-delete-vm'; + $restoredInstance->status = 'active'; + $restoredInstance->service_status = 'exempt'; + $restoredInstance->network_status = 'active'; + $restoredInstance->expires_at = null; + $restoredInstance->spec_json = json_encode([ + 'managed' => true, + 'vcpu' => 2, + 'memory_mib' => 2048, + 'disk_gib' => 20, + 'public_ip_cidr' => '203.0.113.36/28', + ], JSON_THROW_ON_ERROR); + $restoredInstance->sealed_credential = 'sealed-restored-credential'; + $restoredInstance->created_at = $restoredInstance->updated_at = time(); + $restoredInstance->save(); + $restoredReservation->instance_id = (int) $restoredInstance->id; + $restoredReservation->save(); + $restoredIp->reservation_id = (int) $restoredReservation->id; + $restoredIp->instance_id = (int) $restoredInstance->id; + $restoredIp->operation_id = $restoreOperationId; + $restoredIp->save(); + $backupRestore->target_instance_id = (int) $restoredInstance->id; + $backupRestore->reservation_id = (int) $restoredReservation->id; + $backupRestore->save(); + $restoredDelete = CloudJobService::enqueue( + $restoredInstance, + $adminActor, + 'delete', + 'DELETE', + '', + ['remove_managed_disks' => true] + ); + $mock->append(new Response(200, [], json_encode([ + 'operation_id' => $restoredDelete->operation_id, + 'status' => 'succeeded', + ...CloudOperationRecoveryIdentity::expected($restoredDelete, true), + 'result' => [ + 'uuid' => $restoredUuid, + 'deleted' => true, + 'managed_artifacts_absent' => true, + 'forced' => false, + ], + ]))); + $worker->tick(); + $restoredDelete->refresh(); + $restoredInstance->refresh(); + $restoredReservation->refresh(); + $restoredIp->refresh(); + $assert($restoredDelete->phase === 'terminal' + && $restoredDelete->outcome === 'succeeded' + && $restoredInstance->status === 'deleted' + && $restoredReservation->state === 'released' + && $restoredReservation->release_reason === 'instance_deleted' + && $restoredIp->state === 'available' + && $restoredIp->reservation_id === null + && $restoredIp->instance_id === null + && $restoredIp->operation_id === null, + 'Deleting an independent restored instance leaked capacity or its public IP.'); + // Provider callbacks credit the wallet before the invoice bridge runs. // That credit must be consumed once, queue one create job, and remain