|
|
|
|
@ -9,6 +9,7 @@ use App\Models\CloudTrafficInstanceCursor;
|
|
|
|
|
use App\Models\CloudTrafficLedgerEvent;
|
|
|
|
|
use App\Models\CloudTrafficPool;
|
|
|
|
|
use App\Models\CloudTrafficPoolGroup;
|
|
|
|
|
use App\Models\CloudTrafficScanFailure;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Shared prepaid traffic accounting.
|
|
|
|
|
@ -645,14 +646,26 @@ final class CloudTrafficPoolAccounting
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Discover missing and due period projections without locking instances. */
|
|
|
|
|
public static function reconcileDue(int $limit = 200, ?int $now = null): array
|
|
|
|
|
public static function reconcileDue(
|
|
|
|
|
int $limit = 200,
|
|
|
|
|
?int $now = null,
|
|
|
|
|
int $afterPeriodId = 0
|
|
|
|
|
): array
|
|
|
|
|
{
|
|
|
|
|
$now ??= time();
|
|
|
|
|
$limit = max(1, $limit);
|
|
|
|
|
$afterPeriodId = max(0, $afterPeriodId);
|
|
|
|
|
$periodIds = [];
|
|
|
|
|
$missing = CloudServicePeriod::query()
|
|
|
|
|
->leftJoin('cloud_traffic_entitlements as traffic_entitlement',
|
|
|
|
|
'traffic_entitlement.service_period_id', '=',
|
|
|
|
|
'cloud_service_periods.id')
|
|
|
|
|
->leftJoin('cloud_traffic_scan_failures as missing_failure',
|
|
|
|
|
static function ($join): void {
|
|
|
|
|
$join->on('missing_failure.subject_id', '=',
|
|
|
|
|
'cloud_service_periods.id')
|
|
|
|
|
->where('missing_failure.scan_kind', '=', 'period');
|
|
|
|
|
})
|
|
|
|
|
->join('cloud_traffic_pool_groups as traffic_group',
|
|
|
|
|
'traffic_group.id', '=',
|
|
|
|
|
'cloud_service_periods.traffic_pool_group_id')
|
|
|
|
|
@ -660,13 +673,20 @@ final class CloudTrafficPoolAccounting
|
|
|
|
|
->whereIn('cloud_service_periods.status',
|
|
|
|
|
['scheduled', 'active', 'exhausted', 'finished'])
|
|
|
|
|
->whereNull('traffic_entitlement.id')
|
|
|
|
|
->where(function ($retry) use ($now): void {
|
|
|
|
|
$retry->whereNull('missing_failure.id')
|
|
|
|
|
->orWhere('missing_failure.retry_at', '<=', $now);
|
|
|
|
|
})
|
|
|
|
|
->where('cloud_service_periods.id', '>', $afterPeriodId)
|
|
|
|
|
->orderBy('cloud_service_periods.id')->limit($limit)
|
|
|
|
|
->pluck('cloud_service_periods.id')->all();
|
|
|
|
|
foreach ($missing as $id) {
|
|
|
|
|
$periodIds[(int) $id] = true;
|
|
|
|
|
}
|
|
|
|
|
if (count($periodIds) < $limit) {
|
|
|
|
|
$due = CloudTrafficEntitlement::query()
|
|
|
|
|
$due = CloudTrafficEntitlement::query()
|
|
|
|
|
->leftJoin('cloud_traffic_scan_failures as due_failure',
|
|
|
|
|
static function ($join): void {
|
|
|
|
|
$join->on('due_failure.subject_id', '=',
|
|
|
|
|
'cloud_traffic_entitlements.service_period_id')
|
|
|
|
|
->where('due_failure.scan_kind', '=', 'period');
|
|
|
|
|
})
|
|
|
|
|
->join('cloud_traffic_pools as due_pool',
|
|
|
|
|
'due_pool.id', '=',
|
|
|
|
|
'cloud_traffic_entitlements.pool_id')
|
|
|
|
|
@ -674,6 +694,12 @@ final class CloudTrafficPoolAccounting
|
|
|
|
|
'due_group.id', '=',
|
|
|
|
|
'due_pool.traffic_pool_group_id')
|
|
|
|
|
->whereIn('due_group.mode', ['shadow', 'enforced'])
|
|
|
|
|
->where(function ($retry) use ($now): void {
|
|
|
|
|
$retry->whereNull('due_failure.id')
|
|
|
|
|
->orWhere('due_failure.retry_at', '<=', $now);
|
|
|
|
|
})
|
|
|
|
|
->where('cloud_traffic_entitlements.service_period_id', '>',
|
|
|
|
|
$afterPeriodId)
|
|
|
|
|
->where(function ($query) use ($now): void {
|
|
|
|
|
$query->where(function ($scheduled) use ($now): void {
|
|
|
|
|
$scheduled->where(
|
|
|
|
|
@ -692,15 +718,16 @@ final class CloudTrafficPoolAccounting
|
|
|
|
|
)->where('cloud_traffic_entitlements.closed_at',
|
|
|
|
|
'<=', $now);
|
|
|
|
|
});
|
|
|
|
|
})->orderBy('cloud_traffic_entitlements.id')
|
|
|
|
|
->limit($limit - count($periodIds))
|
|
|
|
|
})->orderBy('cloud_traffic_entitlements.service_period_id')
|
|
|
|
|
->limit($limit)
|
|
|
|
|
->pluck('cloud_traffic_entitlements.service_period_id')->all();
|
|
|
|
|
foreach ($due as $id) {
|
|
|
|
|
$periodIds[(int) $id] = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (count($periodIds) < $limit) {
|
|
|
|
|
$changed = CloudTrafficEntitlement::query()
|
|
|
|
|
$changed = CloudTrafficEntitlement::query()
|
|
|
|
|
->leftJoin('cloud_traffic_scan_failures as changed_failure',
|
|
|
|
|
static function ($join): void {
|
|
|
|
|
$join->on('changed_failure.subject_id', '=',
|
|
|
|
|
'cloud_traffic_entitlements.service_period_id')
|
|
|
|
|
->where('changed_failure.scan_kind', '=', 'period');
|
|
|
|
|
})
|
|
|
|
|
->join('cloud_service_periods as traffic_period',
|
|
|
|
|
'traffic_period.id', '=',
|
|
|
|
|
'cloud_traffic_entitlements.service_period_id')
|
|
|
|
|
@ -711,18 +738,25 @@ final class CloudTrafficPoolAccounting
|
|
|
|
|
'changed_group.id', '=',
|
|
|
|
|
'changed_pool.traffic_pool_group_id')
|
|
|
|
|
->whereIn('changed_group.mode', ['shadow', 'enforced'])
|
|
|
|
|
->where(function ($retry) use ($now): void {
|
|
|
|
|
$retry->whereNull('changed_failure.id')
|
|
|
|
|
->orWhere('changed_failure.retry_at', '<=', $now);
|
|
|
|
|
})
|
|
|
|
|
->where('cloud_traffic_entitlements.service_period_id', '>',
|
|
|
|
|
$afterPeriodId)
|
|
|
|
|
->whereColumn(
|
|
|
|
|
'traffic_period.traffic_quota_bytes',
|
|
|
|
|
'>',
|
|
|
|
|
'cloud_traffic_entitlements.granted_bytes'
|
|
|
|
|
)->orderBy('cloud_traffic_entitlements.id')
|
|
|
|
|
->limit($limit - count($periodIds))
|
|
|
|
|
)->orderBy('cloud_traffic_entitlements.service_period_id')
|
|
|
|
|
->limit($limit)
|
|
|
|
|
->pluck('cloud_traffic_entitlements.service_period_id')
|
|
|
|
|
->all();
|
|
|
|
|
foreach ($changed as $id) {
|
|
|
|
|
$periodIds[(int) $id] = true;
|
|
|
|
|
}
|
|
|
|
|
foreach (array_merge($missing, $due, $changed) as $id) {
|
|
|
|
|
$periodIds[(int) $id] = true;
|
|
|
|
|
}
|
|
|
|
|
ksort($periodIds, SORT_NUMERIC);
|
|
|
|
|
$periodIds = array_slice($periodIds, 0, $limit, true);
|
|
|
|
|
$processed = 0;
|
|
|
|
|
$errors = [];
|
|
|
|
|
foreach (array_keys($periodIds) as $periodId) {
|
|
|
|
|
@ -732,23 +766,44 @@ final class CloudTrafficPoolAccounting
|
|
|
|
|
'period-scan:' . $periodId,
|
|
|
|
|
$now
|
|
|
|
|
);
|
|
|
|
|
self::clearScanFailure('period', $periodId);
|
|
|
|
|
$processed++;
|
|
|
|
|
} catch (\Throwable $error) {
|
|
|
|
|
self::recordScanFailure('period', $periodId, $error, $now);
|
|
|
|
|
$errors[] = [
|
|
|
|
|
'period_id' => $periodId,
|
|
|
|
|
'error' => get_class($error),
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return ['processed' => $processed, 'errors' => $errors];
|
|
|
|
|
$selectedIds = array_keys($periodIds);
|
|
|
|
|
$nextCursor = count($selectedIds) >= $limit
|
|
|
|
|
? (int) max($selectedIds) : 0;
|
|
|
|
|
return [
|
|
|
|
|
'processed' => $processed,
|
|
|
|
|
'errors' => $errors,
|
|
|
|
|
'next_cursor' => $nextCursor,
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Discover persisted counters that have not yet crossed the accounting seam. */
|
|
|
|
|
public static function settlePendingUsage(int $limit = 200): array
|
|
|
|
|
public static function settlePendingUsage(
|
|
|
|
|
int $limit = 200,
|
|
|
|
|
int $afterInstanceId = 0
|
|
|
|
|
): array
|
|
|
|
|
{
|
|
|
|
|
$now = time();
|
|
|
|
|
$limit = max(1, $limit);
|
|
|
|
|
$afterInstanceId = max(0, $afterInstanceId);
|
|
|
|
|
$rows = CloudInstance::query()
|
|
|
|
|
->leftJoin('cloud_traffic_instance_cursors as traffic_cursor',
|
|
|
|
|
'traffic_cursor.instance_id', '=', 'cloud_instances.id')
|
|
|
|
|
->leftJoin('cloud_traffic_scan_failures as instance_failure',
|
|
|
|
|
static function ($join): void {
|
|
|
|
|
$join->on('instance_failure.subject_id', '=',
|
|
|
|
|
'cloud_instances.id')
|
|
|
|
|
->where('instance_failure.scan_kind', '=', 'instance');
|
|
|
|
|
})
|
|
|
|
|
->whereExists(function ($eligible): void {
|
|
|
|
|
$eligible->selectRaw('1')
|
|
|
|
|
->from('cloud_service_periods as eligible_period')
|
|
|
|
|
@ -776,7 +831,11 @@ final class CloudTrafficPoolAccounting
|
|
|
|
|
'<', 'traffic_cursor.accounted_download_bytes')
|
|
|
|
|
->orWhereColumn('cloud_instances.lifetime_upload_bytes',
|
|
|
|
|
'<', 'traffic_cursor.accounted_upload_bytes');
|
|
|
|
|
})->orderBy('cloud_instances.id')->limit($limit)->distinct()
|
|
|
|
|
})->where(function ($retry) use ($now): void {
|
|
|
|
|
$retry->whereNull('instance_failure.id')
|
|
|
|
|
->orWhere('instance_failure.retry_at', '<=', $now);
|
|
|
|
|
})->where('cloud_instances.id', '>', $afterInstanceId)
|
|
|
|
|
->orderBy('cloud_instances.id')->limit($limit)->distinct()
|
|
|
|
|
->get([
|
|
|
|
|
'cloud_instances.id',
|
|
|
|
|
'cloud_instances.traffic_synced_at',
|
|
|
|
|
@ -789,15 +848,93 @@ final class CloudTrafficPoolAccounting
|
|
|
|
|
(int) $row->id,
|
|
|
|
|
(int) $row->traffic_synced_at
|
|
|
|
|
);
|
|
|
|
|
self::clearScanFailure('instance', (int) $row->id);
|
|
|
|
|
$processed++;
|
|
|
|
|
} catch (\Throwable $error) {
|
|
|
|
|
self::recordScanFailure(
|
|
|
|
|
'instance',
|
|
|
|
|
(int) $row->id,
|
|
|
|
|
$error,
|
|
|
|
|
$now
|
|
|
|
|
);
|
|
|
|
|
$errors[] = [
|
|
|
|
|
'instance_id' => (int) $row->id,
|
|
|
|
|
'error' => get_class($error),
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return ['processed' => $processed, 'errors' => $errors];
|
|
|
|
|
$nextCursor = count($rows) >= $limit
|
|
|
|
|
? (int) $rows->last()->id : 0;
|
|
|
|
|
return [
|
|
|
|
|
'processed' => $processed,
|
|
|
|
|
'errors' => $errors,
|
|
|
|
|
'next_cursor' => $nextCursor,
|
|
|
|
|
];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function recordScanFailure(
|
|
|
|
|
string $scanKind,
|
|
|
|
|
int $subjectId,
|
|
|
|
|
\Throwable $error,
|
|
|
|
|
int $now
|
|
|
|
|
): void {
|
|
|
|
|
if (!in_array($scanKind, ['period', 'instance'], true)
|
|
|
|
|
|| $subjectId < 1) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
$model = new CloudTrafficScanFailure();
|
|
|
|
|
$connection = $model->getConnection();
|
|
|
|
|
$connection->transaction(function () use (
|
|
|
|
|
$connection,
|
|
|
|
|
$scanKind,
|
|
|
|
|
$subjectId,
|
|
|
|
|
$error,
|
|
|
|
|
$now
|
|
|
|
|
): void {
|
|
|
|
|
$subjectExists = $scanKind === 'period'
|
|
|
|
|
? CloudServicePeriod::where('id', $subjectId)
|
|
|
|
|
->select('id')->lockForUpdate()->first() !== null
|
|
|
|
|
: CloudInstance::where('id', $subjectId)
|
|
|
|
|
->select('id')->lockForUpdate()->first() !== null;
|
|
|
|
|
if (!$subjectExists) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
$connection->table('cloud_traffic_scan_failures')
|
|
|
|
|
->insertOrIgnore([
|
|
|
|
|
'scan_kind' => $scanKind,
|
|
|
|
|
'subject_id' => $subjectId,
|
|
|
|
|
'attempt_count' => 0,
|
|
|
|
|
'last_error_code' => '',
|
|
|
|
|
'retry_at' => $now,
|
|
|
|
|
'first_failed_at' => $now,
|
|
|
|
|
'last_failed_at' => $now,
|
|
|
|
|
]);
|
|
|
|
|
$failure = CloudTrafficScanFailure::where(
|
|
|
|
|
'scan_kind', $scanKind
|
|
|
|
|
)->where('subject_id', $subjectId)->lockForUpdate()->first();
|
|
|
|
|
if ($failure === null) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
$attempt = min(65535, (int) $failure->attempt_count + 1);
|
|
|
|
|
$class = get_class($error);
|
|
|
|
|
$separator = strrpos($class, '\\');
|
|
|
|
|
$code = $separator === false
|
|
|
|
|
? $class : substr($class, $separator + 1);
|
|
|
|
|
$backoff = min(3600, 30 * (2 ** min(7, $attempt - 1)));
|
|
|
|
|
$failure->attempt_count = $attempt;
|
|
|
|
|
$failure->last_error_code = substr($code, 0, 128);
|
|
|
|
|
$failure->retry_at = $now + $backoff;
|
|
|
|
|
$failure->last_failed_at = $now;
|
|
|
|
|
$failure->save();
|
|
|
|
|
}, 3);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function clearScanFailure(
|
|
|
|
|
string $scanKind,
|
|
|
|
|
int $subjectId
|
|
|
|
|
): void {
|
|
|
|
|
CloudTrafficScanFailure::where('scan_kind', $scanKind)
|
|
|
|
|
->where('subject_id', $subjectId)->delete();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static function assertAccountingAttentionResolvedLocked(
|
|
|
|
|
|