You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
507 lines
19 KiB
PHP
507 lines
19 KiB
PHP
<?php
|
|
|
|
namespace App\Services\Cloud;
|
|
|
|
use App\Models\User;
|
|
use App\Services\Telegram\AdministratorTelegramChannel;
|
|
use App\Services\Telegram\AdministratorTelegramDeliveryChannel;
|
|
use App\Services\Telegram\AdministratorTelegramDestination;
|
|
use App\Services\Telegram\AdministratorTelegramLocalUnavailable;
|
|
use App\Services\Telegram\AdministratorTelegramPreflightException;
|
|
use App\Services\Telegram\AdministratorTelegramRateBacklog;
|
|
use App\Services\Telegram\AdministratorTelegramRemoteEffectUnknown;
|
|
|
|
/** Durable, at-most-once Telegram delivery for immutable abuse alerts. */
|
|
final class CloudAbuseAlertDeliveryService
|
|
{
|
|
public const DISPATCH_STALE_SECONDS = 30;
|
|
|
|
private readonly \Closure $clock;
|
|
private readonly \Closure $afterRemoteSuccess;
|
|
|
|
public function __construct(
|
|
private readonly ?AdministratorTelegramDeliveryChannel $channel = null,
|
|
private readonly ?CloudAbuseAlertMessageFormatter $formatter = null,
|
|
?callable $clock = null,
|
|
?callable $afterRemoteSuccess = null
|
|
) {
|
|
$this->clock = \Closure::fromCallable(
|
|
$clock ?? static fn (): int => time()
|
|
);
|
|
$this->afterRemoteSuccess = \Closure::fromCallable(
|
|
$afterRemoteSuccess ?? static function (int $messageId): void {}
|
|
);
|
|
}
|
|
|
|
/** Process at most one stale fence and one fresh event/delivery. */
|
|
public function tick(): bool
|
|
{
|
|
$now = $this->now();
|
|
if ($this->settleStaleDispatch($now)) {
|
|
return true;
|
|
}
|
|
$reserved = $this->reserveNextEvent($now);
|
|
if ($reserved === 'local_unavailable') {
|
|
throw new \RuntimeException(
|
|
'Abuse alert Telegram channel is temporarily unavailable'
|
|
);
|
|
}
|
|
if ($reserved === 'terminal') {
|
|
return true;
|
|
}
|
|
return $this->dispatchNextPending($now)
|
|
|| $reserved === 'pending';
|
|
}
|
|
|
|
/** @return 'none'|'pending'|'terminal'|'local_unavailable' */
|
|
private function reserveNextEvent(int $now): string
|
|
{
|
|
$main = self::connection();
|
|
$event = $main->table('cloud_network_alert_events as event')
|
|
->join(
|
|
'cloud_network_findings as finding',
|
|
'finding.id',
|
|
'=',
|
|
'event.finding_id'
|
|
)->join(
|
|
'cloud_instances as instance',
|
|
'instance.id',
|
|
'=',
|
|
'finding.instance_id'
|
|
)
|
|
->whereNotExists(static function ($query): void {
|
|
$query->selectRaw('1')
|
|
->from('cloud_network_alert_deliveries as delivery')
|
|
->whereColumn('delivery.event_id', 'event.id');
|
|
})->orderBy('event.id')->first([
|
|
'event.*',
|
|
'finding.instance_id',
|
|
'finding.policy_revision',
|
|
'finding.rule_code',
|
|
'finding.direction',
|
|
'finding.ip_family',
|
|
'instance.user_id',
|
|
'instance.host_id',
|
|
]);
|
|
if ($event === null) {
|
|
return 'none';
|
|
}
|
|
$formatFailed = false;
|
|
try {
|
|
$message = $this->messageFormatter()->format($event);
|
|
} catch (\Throwable $error) {
|
|
$formatFailed = true;
|
|
$message = '网络滥用告警事件格式无效,请在管理后台核查事件 #'
|
|
. (int) $event->id . '。';
|
|
}
|
|
self::assertFrozenMessage($message);
|
|
$destination = null;
|
|
if (!$formatFailed) {
|
|
try {
|
|
$destination = $this->telegramChannel()->currentDestination();
|
|
} catch (AdministratorTelegramLocalUnavailable $error) {
|
|
return 'local_unavailable';
|
|
}
|
|
}
|
|
|
|
return $main->transaction(function () use (
|
|
$main, $event, $message, $destination, $formatFailed, $now
|
|
): string {
|
|
$effectiveDestination = $destination;
|
|
if ($effectiveDestination !== null) {
|
|
// Lock recipient before event, matching account deletion's
|
|
// user -> delivery/event order. This fences a late pending
|
|
// row from appearing after that user's deletion scan.
|
|
$administrator = $main->table('user')
|
|
->where('id', $effectiveDestination->administratorId)
|
|
->lockForUpdate()->first(['id', 'is_admin', 'enable']);
|
|
if ($administrator === null
|
|
|| (int) $administrator->is_admin !== 1
|
|
|| (int) $administrator->enable !== 1) {
|
|
$effectiveDestination = null;
|
|
}
|
|
}
|
|
$locked = $main->table('cloud_network_alert_events')
|
|
->where('id', (int) $event->id)
|
|
->lockForUpdate()->first();
|
|
if ($locked === null || $main->table(
|
|
'cloud_network_alert_deliveries'
|
|
)->where('event_id', (int) $event->id)->exists()) {
|
|
return 'none';
|
|
}
|
|
|
|
$state = 'pending';
|
|
$errorCode = '';
|
|
$skipReason = '';
|
|
$terminalAt = null;
|
|
if ($formatFailed) {
|
|
$state = 'failed';
|
|
$errorCode = 'message_format_invalid';
|
|
$terminalAt = $now;
|
|
} elseif ($effectiveDestination === null) {
|
|
$state = 'skipped';
|
|
$skipReason = $destination === null
|
|
? 'channel_unavailable' : 'destination_ineligible';
|
|
$terminalAt = $now;
|
|
}
|
|
$main->table('cloud_network_alert_deliveries')->insert([
|
|
'event_id' => (int) $locked->id,
|
|
'binding_generation' =>
|
|
$effectiveDestination?->bindingGeneration,
|
|
'bound_admin_id' =>
|
|
$effectiveDestination?->administratorId,
|
|
'chat_id' => $effectiveDestination?->chatId,
|
|
'telegram_message_id' => null,
|
|
'state' => $state,
|
|
'message_text' => $message,
|
|
'content_hash' => hash('sha256', $message),
|
|
'error_code' => $errorCode,
|
|
'skip_reason' => $skipReason,
|
|
'dispatch_started_at' => null,
|
|
'sent_at' => null,
|
|
'terminal_at' => $terminalAt,
|
|
'destination_redacted_at' => null,
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
return $state === 'pending' ? 'pending' : 'terminal';
|
|
}, 3);
|
|
}
|
|
|
|
private function dispatchNextPending(int $now): bool
|
|
{
|
|
$main = self::connection();
|
|
$claim = $main->transaction(function () use ($main, $now): ?array {
|
|
$candidate = $main->table('cloud_network_alert_deliveries')
|
|
->where('state', 'pending')->orderBy('created_at')
|
|
->orderBy('id')->first([
|
|
'id', 'event_id', 'bound_admin_id',
|
|
]);
|
|
if ($candidate === null) {
|
|
return null;
|
|
}
|
|
$administrator = $main->table('user')
|
|
->where('id', (int) $candidate->bound_admin_id)
|
|
->lockForUpdate()->first(['id', 'is_admin', 'enable']);
|
|
$event = $main->table('cloud_network_alert_events')
|
|
->where('id', (int) $candidate->event_id)
|
|
->lockForUpdate()->first(['id']);
|
|
if ($event === null) {
|
|
return null;
|
|
}
|
|
$delivery = $main->table('cloud_network_alert_deliveries')
|
|
->where('id', (int) $candidate->id)
|
|
->where('event_id', (int) $event->id)
|
|
->where('state', 'pending')
|
|
->lockForUpdate()->first();
|
|
if ($delivery === null) {
|
|
return null;
|
|
}
|
|
if ($administrator === null
|
|
|| (int) $administrator->is_admin !== 1
|
|
|| (int) $administrator->enable !== 1
|
|
|| !$this->telegramChannel()->eligibleAdministrator(
|
|
(int) $administrator->id
|
|
)) {
|
|
$updated = $main->table('cloud_network_alert_deliveries')
|
|
->where('id', (int) $delivery->id)
|
|
->where('state', 'pending')->update([
|
|
'state' => 'skipped',
|
|
'skip_reason' => 'destination_ineligible',
|
|
'terminal_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse alert recipient settlement failed'
|
|
);
|
|
}
|
|
return ['terminal' => true];
|
|
}
|
|
$message = (string) $delivery->message_text;
|
|
if (!self::validDestination($delivery)
|
|
|| !self::validFrozenMessage(
|
|
$message,
|
|
(string) $delivery->content_hash
|
|
)) {
|
|
$updated = $main->table('cloud_network_alert_deliveries')
|
|
->where('id', (int) $delivery->id)
|
|
->where('state', 'pending')->update([
|
|
'state' => 'failed',
|
|
'error_code' => 'delivery_identity_invalid',
|
|
'terminal_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse alert invalid delivery finalization failed'
|
|
);
|
|
}
|
|
return ['terminal' => true];
|
|
}
|
|
$updated = $main->table('cloud_network_alert_deliveries')
|
|
->where('id', (int) $delivery->id)
|
|
->where('state', 'pending')->update([
|
|
'state' => 'dispatching',
|
|
'dispatch_started_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException('Abuse alert delivery claim failed');
|
|
}
|
|
return [
|
|
'id' => (int) $delivery->id,
|
|
'message' => $message,
|
|
'destination' => new AdministratorTelegramDestination(
|
|
(int) $delivery->bound_admin_id,
|
|
(string) $delivery->chat_id,
|
|
(string) $delivery->binding_generation
|
|
),
|
|
];
|
|
}, 3);
|
|
|
|
if ($claim === null) {
|
|
return false;
|
|
}
|
|
if (($claim['terminal'] ?? false) === true) {
|
|
return true;
|
|
}
|
|
return $this->sendClaim($claim, $now);
|
|
}
|
|
|
|
/** @param array{id:int,message:string,destination:AdministratorTelegramDestination} $claim */
|
|
private function sendClaim(array $claim, int $now): bool
|
|
{
|
|
$id = $claim['id'];
|
|
try {
|
|
$messageId = $this->telegramChannel()->send(
|
|
$claim['destination'],
|
|
$claim['message'],
|
|
['disable_web_page_preview' => 'true']
|
|
);
|
|
} catch (AdministratorTelegramRateBacklog $error) {
|
|
$this->returnToPending($id, $now);
|
|
throw new \RuntimeException(
|
|
'Abuse alert Telegram delivery backlog is full'
|
|
);
|
|
} catch (AdministratorTelegramLocalUnavailable $error) {
|
|
$this->returnToPending($id, $now);
|
|
throw new \RuntimeException(
|
|
'Abuse alert Telegram channel is temporarily unavailable'
|
|
);
|
|
} catch (AdministratorTelegramPreflightException $error) {
|
|
if ($error->reason
|
|
=== AdministratorTelegramPreflightException::PAYLOAD_INVALID) {
|
|
$this->failInvalidPayload($id, $now);
|
|
} else {
|
|
$this->skipPreflight(
|
|
$id,
|
|
$now,
|
|
$error->reason
|
|
=== AdministratorTelegramPreflightException::DESTINATION_CHANGED
|
|
? 'destination_changed'
|
|
: 'channel_disabled_or_unavailable'
|
|
);
|
|
}
|
|
return true;
|
|
} catch (AdministratorTelegramRemoteEffectUnknown $error) {
|
|
$this->markUnknown($id, $now, null);
|
|
throw new \RuntimeException(
|
|
'Abuse alert Telegram remote effect is unknown'
|
|
);
|
|
} catch (\Throwable $error) {
|
|
$this->markUnknown($id, $now, null);
|
|
throw new \RuntimeException(
|
|
'Abuse alert Telegram remote effect is unknown'
|
|
);
|
|
}
|
|
|
|
try {
|
|
($this->afterRemoteSuccess)($messageId);
|
|
$updated = self::connection()->table(
|
|
'cloud_network_alert_deliveries'
|
|
)->where('id', $id)->where('state', 'dispatching')->update([
|
|
'telegram_message_id' => $messageId,
|
|
'state' => 'sent',
|
|
'sent_at' => $now,
|
|
'terminal_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse alert Telegram delivery finalization failed'
|
|
);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
$this->markUnknown($id, $now, $messageId);
|
|
throw new \RuntimeException(
|
|
'Abuse alert Telegram local finalization is unknown'
|
|
);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
private function settleStaleDispatch(int $now): bool
|
|
{
|
|
$main = self::connection();
|
|
return $main->transaction(function () use ($main, $now): bool {
|
|
$row = $main->table('cloud_network_alert_deliveries')
|
|
->where('state', 'dispatching')
|
|
->where('updated_at', '<=',
|
|
$now - self::DISPATCH_STALE_SECONDS)
|
|
->orderBy('updated_at')->orderBy('id')
|
|
->lockForUpdate()->first();
|
|
if ($row === null) {
|
|
return false;
|
|
}
|
|
$updated = $main->table('cloud_network_alert_deliveries')
|
|
->where('id', (int) $row->id)
|
|
->where('state', 'dispatching')->update([
|
|
'state' => 'unknown',
|
|
'error_code' => 'worker_interrupted',
|
|
'terminal_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Stale abuse alert delivery settlement failed'
|
|
);
|
|
}
|
|
return true;
|
|
}, 3);
|
|
}
|
|
|
|
private function returnToPending(int $id, int $now): void
|
|
{
|
|
$updated = self::connection()->table(
|
|
'cloud_network_alert_deliveries'
|
|
)->where('id', $id)->where('state', 'dispatching')->update([
|
|
'state' => 'pending',
|
|
'dispatch_started_at' => null,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse alert local retry release failed'
|
|
);
|
|
}
|
|
}
|
|
|
|
private function skipPreflight(
|
|
int $id,
|
|
int $now,
|
|
string $reason
|
|
): void
|
|
{
|
|
$updated = self::connection()->table(
|
|
'cloud_network_alert_deliveries'
|
|
)->where('id', $id)->where('state', 'dispatching')->update([
|
|
'state' => 'skipped',
|
|
'skip_reason' => $reason,
|
|
'dispatch_started_at' => null,
|
|
'terminal_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse alert destination-change settlement failed'
|
|
);
|
|
}
|
|
}
|
|
|
|
private function failInvalidPayload(int $id, int $now): void
|
|
{
|
|
$updated = self::connection()->table(
|
|
'cloud_network_alert_deliveries'
|
|
)->where('id', $id)->where('state', 'dispatching')->update([
|
|
'state' => 'failed',
|
|
'error_code' => 'payload_invalid',
|
|
'dispatch_started_at' => null,
|
|
'terminal_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse alert payload failure settlement failed'
|
|
);
|
|
}
|
|
}
|
|
|
|
private function markUnknown(
|
|
int $id,
|
|
int $now,
|
|
?int $messageId
|
|
): void {
|
|
try {
|
|
$updated = self::connection()->table(
|
|
'cloud_network_alert_deliveries'
|
|
)->where('id', $id)->where('state', 'dispatching')->update([
|
|
'telegram_message_id' => $messageId,
|
|
'state' => 'unknown',
|
|
'error_code' => 'remote_effect_unknown',
|
|
'terminal_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
if ($updated !== 1) {
|
|
throw new \RuntimeException(
|
|
'Abuse alert unknown-effect settlement failed'
|
|
);
|
|
}
|
|
} catch (\Throwable $error) {
|
|
// A retained dispatching row remains fenced and will be converted
|
|
// by the stale reconciler. It must never be replayed.
|
|
error_log('Abuse alert Telegram reconciliation failed');
|
|
}
|
|
}
|
|
|
|
private static function validDestination(object $delivery): bool
|
|
{
|
|
return (int) $delivery->bound_admin_id > 0
|
|
&& preg_match('/^[a-f0-9]{32}$/D',
|
|
(string) $delivery->binding_generation) === 1
|
|
&& preg_match('/^[1-9][0-9]{0,18}$/D',
|
|
(string) $delivery->chat_id) === 1;
|
|
}
|
|
|
|
private static function assertFrozenMessage(string $message): void
|
|
{
|
|
if (!self::validFrozenMessage($message, hash('sha256', $message))) {
|
|
throw new \LogicException('Abuse alert message is invalid');
|
|
}
|
|
}
|
|
|
|
private static function validFrozenMessage(
|
|
string $message,
|
|
string $hash
|
|
): bool {
|
|
$length = mb_strlen($message, 'UTF-8');
|
|
return $length >= 1 && $length <= 4096
|
|
&& preg_match('/^[a-f0-9]{64}$/D', $hash) === 1
|
|
&& hash_equals($hash, hash('sha256', $message));
|
|
}
|
|
|
|
private function telegramChannel(): AdministratorTelegramDeliveryChannel
|
|
{
|
|
return $this->channel ?? new AdministratorTelegramChannel();
|
|
}
|
|
|
|
private function messageFormatter(): CloudAbuseAlertMessageFormatter
|
|
{
|
|
return $this->formatter ?? new CloudAbuseAlertMessageFormatter();
|
|
}
|
|
|
|
private function now(): int
|
|
{
|
|
$now = ($this->clock)();
|
|
if (!is_int($now) || $now < 1) {
|
|
throw new \LogicException('Abuse alert delivery clock is invalid');
|
|
}
|
|
return $now;
|
|
}
|
|
|
|
private static function connection(): object
|
|
{
|
|
return User::getConnectionResolver()->connection('default');
|
|
}
|
|
}
|