You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
virt/web/app/Services/Cloud/CloudAbuseAlertDeliveryServ...

507 lines
19 KiB
PHP

<?php
namespace App\Services\Cloud;
use App\Models\User;
use App\Services\Telegram\AdministratorTelegramChannel;
use App\Services\Telegram\AdministratorTelegramDeliveryChannel;
use App\Services\Telegram\AdministratorTelegramDestination;
use App\Services\Telegram\AdministratorTelegramLocalUnavailable;
use App\Services\Telegram\AdministratorTelegramPreflightException;
use App\Services\Telegram\AdministratorTelegramRateBacklog;
use App\Services\Telegram\AdministratorTelegramRemoteEffectUnknown;
/** Durable, at-most-once Telegram delivery for immutable abuse alerts. */
final class CloudAbuseAlertDeliveryService
{
public const DISPATCH_STALE_SECONDS = 30;
private readonly \Closure $clock;
private readonly \Closure $afterRemoteSuccess;
public function __construct(
private readonly ?AdministratorTelegramDeliveryChannel $channel = null,
private readonly ?CloudAbuseAlertMessageFormatter $formatter = null,
?callable $clock = null,
?callable $afterRemoteSuccess = null
) {
$this->clock = \Closure::fromCallable(
$clock ?? static fn (): int => time()
);
$this->afterRemoteSuccess = \Closure::fromCallable(
$afterRemoteSuccess ?? static function (int $messageId): void {}
);
}
/** Process at most one stale fence and one fresh event/delivery. */
public function tick(): bool
{
$now = $this->now();
if ($this->settleStaleDispatch($now)) {
return true;
}
$reserved = $this->reserveNextEvent($now);
if ($reserved === 'local_unavailable') {
throw new \RuntimeException(
'Abuse alert Telegram channel is temporarily unavailable'
);
}
if ($reserved === 'terminal') {
return true;
}
return $this->dispatchNextPending($now)
|| $reserved === 'pending';
}
/** @return 'none'|'pending'|'terminal'|'local_unavailable' */
private function reserveNextEvent(int $now): string
{
$main = self::connection();
$event = $main->table('cloud_network_alert_events as event')
->join(
'cloud_network_findings as finding',
'finding.id',
'=',
'event.finding_id'
)->join(
'cloud_instances as instance',
'instance.id',
'=',
'finding.instance_id'
)
->whereNotExists(static function ($query): void {
$query->selectRaw('1')
->from('cloud_network_alert_deliveries as delivery')
->whereColumn('delivery.event_id', 'event.id');
})->orderBy('event.id')->first([
'event.*',
'finding.instance_id',
'finding.policy_revision',
'finding.rule_code',
'finding.direction',
'finding.ip_family',
'instance.user_id',
'instance.host_id',
]);
if ($event === null) {
return 'none';
}
$formatFailed = false;
try {
$message = $this->messageFormatter()->format($event);
} catch (\Throwable $error) {
$formatFailed = true;
$message = '网络滥用告警事件格式无效,请在管理后台核查事件 #'
. (int) $event->id . '。';
}
self::assertFrozenMessage($message);
$destination = null;
if (!$formatFailed) {
try {
$destination = $this->telegramChannel()->currentDestination();
} catch (AdministratorTelegramLocalUnavailable $error) {
return 'local_unavailable';
}
}
return $main->transaction(function () use (
$main, $event, $message, $destination, $formatFailed, $now
): string {
$effectiveDestination = $destination;
if ($effectiveDestination !== null) {
// Lock recipient before event, matching account deletion's
// user -> delivery/event order. This fences a late pending
// row from appearing after that user's deletion scan.
$administrator = $main->table('user')
->where('id', $effectiveDestination->administratorId)
->lockForUpdate()->first(['id', 'is_admin', 'enable']);
if ($administrator === null
|| (int) $administrator->is_admin !== 1
|| (int) $administrator->enable !== 1) {
$effectiveDestination = null;
}
}
$locked = $main->table('cloud_network_alert_events')
->where('id', (int) $event->id)
->lockForUpdate()->first();
if ($locked === null || $main->table(
'cloud_network_alert_deliveries'
)->where('event_id', (int) $event->id)->exists()) {
return 'none';
}
$state = 'pending';
$errorCode = '';
$skipReason = '';
$terminalAt = null;
if ($formatFailed) {
$state = 'failed';
$errorCode = 'message_format_invalid';
$terminalAt = $now;
} elseif ($effectiveDestination === null) {
$state = 'skipped';
$skipReason = $destination === null
? 'channel_unavailable' : 'destination_ineligible';
$terminalAt = $now;
}
$main->table('cloud_network_alert_deliveries')->insert([
'event_id' => (int) $locked->id,
'binding_generation' =>
$effectiveDestination?->bindingGeneration,
'bound_admin_id' =>
$effectiveDestination?->administratorId,
'chat_id' => $effectiveDestination?->chatId,
'telegram_message_id' => null,
'state' => $state,
'message_text' => $message,
'content_hash' => hash('sha256', $message),
'error_code' => $errorCode,
'skip_reason' => $skipReason,
'dispatch_started_at' => null,
'sent_at' => null,
'terminal_at' => $terminalAt,
'destination_redacted_at' => null,
'created_at' => $now,
'updated_at' => $now,
]);
return $state === 'pending' ? 'pending' : 'terminal';
}, 3);
}
private function dispatchNextPending(int $now): bool
{
$main = self::connection();
$claim = $main->transaction(function () use ($main, $now): ?array {
$candidate = $main->table('cloud_network_alert_deliveries')
->where('state', 'pending')->orderBy('created_at')
->orderBy('id')->first([
'id', 'event_id', 'bound_admin_id',
]);
if ($candidate === null) {
return null;
}
$administrator = $main->table('user')
->where('id', (int) $candidate->bound_admin_id)
->lockForUpdate()->first(['id', 'is_admin', 'enable']);
$event = $main->table('cloud_network_alert_events')
->where('id', (int) $candidate->event_id)
->lockForUpdate()->first(['id']);
if ($event === null) {
return null;
}
$delivery = $main->table('cloud_network_alert_deliveries')
->where('id', (int) $candidate->id)
->where('event_id', (int) $event->id)
->where('state', 'pending')
->lockForUpdate()->first();
if ($delivery === null) {
return null;
}
if ($administrator === null
|| (int) $administrator->is_admin !== 1
|| (int) $administrator->enable !== 1
|| !$this->telegramChannel()->eligibleAdministrator(
(int) $administrator->id
)) {
$updated = $main->table('cloud_network_alert_deliveries')
->where('id', (int) $delivery->id)
->where('state', 'pending')->update([
'state' => 'skipped',
'skip_reason' => 'destination_ineligible',
'terminal_at' => $now,
'updated_at' => $now,
]);
if ($updated !== 1) {
throw new \RuntimeException(
'Abuse alert recipient settlement failed'
);
}
return ['terminal' => true];
}
$message = (string) $delivery->message_text;
if (!self::validDestination($delivery)
|| !self::validFrozenMessage(
$message,
(string) $delivery->content_hash
)) {
$updated = $main->table('cloud_network_alert_deliveries')
->where('id', (int) $delivery->id)
->where('state', 'pending')->update([
'state' => 'failed',
'error_code' => 'delivery_identity_invalid',
'terminal_at' => $now,
'updated_at' => $now,
]);
if ($updated !== 1) {
throw new \RuntimeException(
'Abuse alert invalid delivery finalization failed'
);
}
return ['terminal' => true];
}
$updated = $main->table('cloud_network_alert_deliveries')
->where('id', (int) $delivery->id)
->where('state', 'pending')->update([
'state' => 'dispatching',
'dispatch_started_at' => $now,
'updated_at' => $now,
]);
if ($updated !== 1) {
throw new \RuntimeException('Abuse alert delivery claim failed');
}
return [
'id' => (int) $delivery->id,
'message' => $message,
'destination' => new AdministratorTelegramDestination(
(int) $delivery->bound_admin_id,
(string) $delivery->chat_id,
(string) $delivery->binding_generation
),
];
}, 3);
if ($claim === null) {
return false;
}
if (($claim['terminal'] ?? false) === true) {
return true;
}
return $this->sendClaim($claim, $now);
}
/** @param array{id:int,message:string,destination:AdministratorTelegramDestination} $claim */
private function sendClaim(array $claim, int $now): bool
{
$id = $claim['id'];
try {
$messageId = $this->telegramChannel()->send(
$claim['destination'],
$claim['message'],
['disable_web_page_preview' => 'true']
);
} catch (AdministratorTelegramRateBacklog $error) {
$this->returnToPending($id, $now);
throw new \RuntimeException(
'Abuse alert Telegram delivery backlog is full'
);
} catch (AdministratorTelegramLocalUnavailable $error) {
$this->returnToPending($id, $now);
throw new \RuntimeException(
'Abuse alert Telegram channel is temporarily unavailable'
);
} catch (AdministratorTelegramPreflightException $error) {
if ($error->reason
=== AdministratorTelegramPreflightException::PAYLOAD_INVALID) {
$this->failInvalidPayload($id, $now);
} else {
$this->skipPreflight(
$id,
$now,
$error->reason
=== AdministratorTelegramPreflightException::DESTINATION_CHANGED
? 'destination_changed'
: 'channel_disabled_or_unavailable'
);
}
return true;
} catch (AdministratorTelegramRemoteEffectUnknown $error) {
$this->markUnknown($id, $now, null);
throw new \RuntimeException(
'Abuse alert Telegram remote effect is unknown'
);
} catch (\Throwable $error) {
$this->markUnknown($id, $now, null);
throw new \RuntimeException(
'Abuse alert Telegram remote effect is unknown'
);
}
try {
($this->afterRemoteSuccess)($messageId);
$updated = self::connection()->table(
'cloud_network_alert_deliveries'
)->where('id', $id)->where('state', 'dispatching')->update([
'telegram_message_id' => $messageId,
'state' => 'sent',
'sent_at' => $now,
'terminal_at' => $now,
'updated_at' => $now,
]);
if ($updated !== 1) {
throw new \RuntimeException(
'Abuse alert Telegram delivery finalization failed'
);
}
} catch (\Throwable $error) {
$this->markUnknown($id, $now, $messageId);
throw new \RuntimeException(
'Abuse alert Telegram local finalization is unknown'
);
}
return true;
}
private function settleStaleDispatch(int $now): bool
{
$main = self::connection();
return $main->transaction(function () use ($main, $now): bool {
$row = $main->table('cloud_network_alert_deliveries')
->where('state', 'dispatching')
->where('updated_at', '<=',
$now - self::DISPATCH_STALE_SECONDS)
->orderBy('updated_at')->orderBy('id')
->lockForUpdate()->first();
if ($row === null) {
return false;
}
$updated = $main->table('cloud_network_alert_deliveries')
->where('id', (int) $row->id)
->where('state', 'dispatching')->update([
'state' => 'unknown',
'error_code' => 'worker_interrupted',
'terminal_at' => $now,
'updated_at' => $now,
]);
if ($updated !== 1) {
throw new \RuntimeException(
'Stale abuse alert delivery settlement failed'
);
}
return true;
}, 3);
}
private function returnToPending(int $id, int $now): void
{
$updated = self::connection()->table(
'cloud_network_alert_deliveries'
)->where('id', $id)->where('state', 'dispatching')->update([
'state' => 'pending',
'dispatch_started_at' => null,
'updated_at' => $now,
]);
if ($updated !== 1) {
throw new \RuntimeException(
'Abuse alert local retry release failed'
);
}
}
private function skipPreflight(
int $id,
int $now,
string $reason
): void
{
$updated = self::connection()->table(
'cloud_network_alert_deliveries'
)->where('id', $id)->where('state', 'dispatching')->update([
'state' => 'skipped',
'skip_reason' => $reason,
'dispatch_started_at' => null,
'terminal_at' => $now,
'updated_at' => $now,
]);
if ($updated !== 1) {
throw new \RuntimeException(
'Abuse alert destination-change settlement failed'
);
}
}
private function failInvalidPayload(int $id, int $now): void
{
$updated = self::connection()->table(
'cloud_network_alert_deliveries'
)->where('id', $id)->where('state', 'dispatching')->update([
'state' => 'failed',
'error_code' => 'payload_invalid',
'dispatch_started_at' => null,
'terminal_at' => $now,
'updated_at' => $now,
]);
if ($updated !== 1) {
throw new \RuntimeException(
'Abuse alert payload failure settlement failed'
);
}
}
private function markUnknown(
int $id,
int $now,
?int $messageId
): void {
try {
$updated = self::connection()->table(
'cloud_network_alert_deliveries'
)->where('id', $id)->where('state', 'dispatching')->update([
'telegram_message_id' => $messageId,
'state' => 'unknown',
'error_code' => 'remote_effect_unknown',
'terminal_at' => $now,
'updated_at' => $now,
]);
if ($updated !== 1) {
throw new \RuntimeException(
'Abuse alert unknown-effect settlement failed'
);
}
} catch (\Throwable $error) {
// A retained dispatching row remains fenced and will be converted
// by the stale reconciler. It must never be replayed.
error_log('Abuse alert Telegram reconciliation failed');
}
}
private static function validDestination(object $delivery): bool
{
return (int) $delivery->bound_admin_id > 0
&& preg_match('/^[a-f0-9]{32}$/D',
(string) $delivery->binding_generation) === 1
&& preg_match('/^[1-9][0-9]{0,18}$/D',
(string) $delivery->chat_id) === 1;
}
private static function assertFrozenMessage(string $message): void
{
if (!self::validFrozenMessage($message, hash('sha256', $message))) {
throw new \LogicException('Abuse alert message is invalid');
}
}
private static function validFrozenMessage(
string $message,
string $hash
): bool {
$length = mb_strlen($message, 'UTF-8');
return $length >= 1 && $length <= 4096
&& preg_match('/^[a-f0-9]{64}$/D', $hash) === 1
&& hash_equals($hash, hash('sha256', $message));
}
private function telegramChannel(): AdministratorTelegramDeliveryChannel
{
return $this->channel ?? new AdministratorTelegramChannel();
}
private function messageFormatter(): CloudAbuseAlertMessageFormatter
{
return $this->formatter ?? new CloudAbuseAlertMessageFormatter();
}
private function now(): int
{
$now = ($this->clock)();
if (!is_int($now) || $now < 1) {
throw new \LogicException('Abuse alert delivery clock is invalid');
}
return $now;
}
private static function connection(): object
{
return User::getConnectionResolver()->connection('default');
}
}