Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
10 changes: 5 additions & 5 deletions .env.ci
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ LOG_LEVEL=debug

FLARE_KEY=

DB_CONNECTION=mysql
DB_CONNECTION=pgsql
DB_HOST=127.0.0.1
DB_PORT=3306
DB_DATABASE=laravel
DB_USERNAME=root
DB_PASSWORD=root
DB_PORT=5432
DB_DATABASE=testing
DB_USERNAME=postgres
DB_PASSWORD=postgres

FPH_ENABLED=true
CSP_ENABLED=true
Expand Down
5 changes: 4 additions & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -57,4 +57,7 @@ CLOUDINARY_CLOUD_NAME=my-cloud-name
CLOUDINARY_API_KEY=my-api-key
CLOUDINARY_API_SECRET=my-api-secret

LARAVEL_DEFAULT_FATHOM_SITE_ID=WECFOADW
LARAVEL_DEFAULT_FATHOM_SITE_ID=WECFOADW
LITELLM_URL=https://llm.codebar.net
LITELLM_MASTER_KEY=
GITHUB_TOKEN=
2 changes: 1 addition & 1 deletion .github/workflows/assets_production.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
name: production-assets | ci

env:
PHP_VERSION: 8.4
PHP_VERSION: 8.5
NODE_VERSION: 24

steps:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/larastan_pull_request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ jobs:
name: pull-request | ci larastan

env:
PHP_VERSION: 8.4
PHP_VERSION: 8.5
NODE_VERSION: 24

steps:
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/pest_coverage_pull_request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ jobs:
name: pull-request | ci pest coverage

env:
PHP_VERSION: 8.4
PHP_VERSION: 8.5
NODE_VERSION: 24
PEST_MIN_COVERAGE: 1
PEST_MIN_TYPE_COVERAGE: 1
Expand All @@ -21,7 +21,7 @@ jobs:
uses: shivammathur/setup-php@v2
with:
php-version: ${{ env.PHP_VERSION }}
extensions: dom, curl, fileinfo, mysql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, xdebug
extensions: dom, curl, fileinfo, pgsql, pdo_pgsql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick, xdebug
coverage: xdebug

- name: Prepare the .env file
Expand All @@ -45,9 +45,9 @@ jobs:

- name: Setup Database
run: |
sudo systemctl start mysql
mysql --user="root" --password="root" -e "CREATE DATABASE laravel character set UTF8mb4 collate utf8mb4_bin;"
mysql --user="root" --password="root" -e "CREATE DATABASE logs character set UTF8mb4 collate utf8mb4_bin;"
sudo systemctl start postgresql.service
sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
sudo -u postgres createdb testing

- name: Generate App Key & Run Migrations
run: |
Expand Down
9 changes: 5 additions & 4 deletions .github/workflows/pest_pull_request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ jobs:
name: pull-request | ci pest

env:
PHP_VERSION: 8.4
PHP_VERSION: 8.5
NODE_VERSION: 24

steps:
Expand All @@ -19,7 +19,7 @@ jobs:
uses: shivammathur/setup-php@v2
with:
php-version: ${{ env.PHP_VERSION }}
extensions: dom, curl, fileinfo, mysql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick
extensions: dom, curl, fileinfo, pgsql, pdo_pgsql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick

- name: Prepare the .env file
run: cp .env.ci .env
Expand All @@ -42,8 +42,9 @@ jobs:

- name: Setup Database
run: |
sudo systemctl start mysql
mysql --user="root" --password="root" -e "CREATE DATABASE laravel character set UTF8mb4 collate utf8mb4_bin;"
sudo systemctl start postgresql.service
sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
sudo -u postgres createdb testing

- name: Generate App Key & Run Migrations
run: |
Expand Down
40 changes: 40 additions & 0 deletions .github/workflows/pint_pull_request.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
name: Pull Request | CI Pint

on: pull_request

jobs:
pint:
runs-on: ubuntu-latest
name: pull-request | ci pint

env:
PHP_VERSION: 8.5

steps:
- name: Checkout code
uses: actions/checkout@v7

- name: Setup PHP Environment
uses: shivammathur/setup-php@v2
with:
php-version: ${{ env.PHP_VERSION }}
extensions: dom, curl, fileinfo, mysql, libxml, mbstring, zip, pcntl, pdo, sqlite, pdo_sqlite, bcmath, soap, intl, gd, exif, iconv, imagick

- name: Prepare the .env file
run: cp .env.ci .env

- name: Cache Composer Dependencies
uses: actions/cache@v6
with:
path: |
~/.composer/cache/files
~/.composer/cache/repo
key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.json') }}
restore-keys: |
${{ runner.os }}-composer-

- name: Install Composer Dependencies
run: composer install --no-progress --prefer-dist --no-interaction --optimize-autoloader

- name: Execute code style check
run: ./vendor/bin/pint --test
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,7 @@ auth.json

# Coverage
/coverage/

# Lighthouse audit reports (regenerate via tests/lighthouse/run.sh)
/tests/lighthouse/reports/
--cache-directory
84 changes: 84 additions & 0 deletions app/Actions/FetchLlmUsageAction.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
<?php

namespace App\Actions;

use Carbon\CarbonImmutable;
use Illuminate\Http\Client\PendingRequest;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Http;

class FetchLlmUsageAction
{
/**
* Fetch the per-model usage aggregates for a single day from the LiteLLM proxy.
*
* @return Collection<int, array{date: string, model: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int, spend: float}>
*/
public function fetchDay(CarbonImmutable $date): Collection
{
$response = $this->client()->get('/spend/logs', [
'start_date' => $date->toDateString(),
'end_date' => $date->addDay()->toDateString(),
'summarize' => 'false',
])->throw();

$logs = $response->json();

return $this->parseSpendLogs($date, is_array($logs) ? $logs : []);
}

private function client(): PendingRequest
{
return Http::baseUrl(config()->string('services.litellm.url'))
->withToken(config()->string('services.litellm.master_key'))
->acceptJson()
->connectTimeout(10)
->timeout(120)
->retry(times: 2, sleepMilliseconds: 1000, throw: false);
}

/**
* @param array<mixed> $logs
* @return Collection<int, array{date: string, model: string, prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int, spend: float}>
*/
private function parseSpendLogs(CarbonImmutable $date, array $logs): Collection
{
// The end_date bound is inclusive, so the response can contain rows of
// the following day — keep only rows that started on the requested day.
return collect($logs)
->filter(fn (mixed $log): bool => filled($this->stringValue($log, 'model_group')))
->filter(fn (mixed $log): bool => str_starts_with($this->stringValue($log, 'startTime'), $date->toDateString()))
->groupBy(fn (mixed $log): string => $this->stringValue($log, 'model_group'))
->map(fn (Collection $rows, string $model): array => [
'date' => $date->toDateString(),
'model' => $model,
'prompt_tokens' => $rows->sum(fn (mixed $log): int => $this->intValue($log, 'prompt_tokens')),
'completion_tokens' => $rows->sum(fn (mixed $log): int => $this->intValue($log, 'completion_tokens')),
'total_tokens' => $rows->sum(fn (mixed $log): int => $this->intValue($log, 'total_tokens')),
'requests' => $rows->count(),
'spend' => round($rows->sum(fn (mixed $log): float => $this->floatValue($log, 'spend')), 6),
])
->values();
}

private function stringValue(mixed $log, string $key): string
{
$value = data_get($log, $key);

return is_string($value) ? $value : '';
}

private function intValue(mixed $log, string $key): int
{
$value = data_get($log, $key);

return is_numeric($value) ? (int) $value : 0;
}

private function floatValue(mixed $log, string $key): float
{
$value = data_get($log, $key);

return is_numeric($value) ? (float) $value : 0.0;
}
}
146 changes: 146 additions & 0 deletions app/Actions/LlmUsageStatsAction.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
<?php

namespace App\Actions;

use App\Models\AiModelDailyUsage;
use Carbon\CarbonImmutable;
use Closure;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Collection as EloquentCollection;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Str;

class LlmUsageStatsAction
{
public const string VERSION_CACHE_KEY = 'llm_usage_version';

public const string OTHER_MODEL = 'other';

/**
* @return Collection<int, string>
*/
public function models(): Collection
{
return $this->remember('models', function () {
return AiModelDailyUsage::query()
->whereNotNull('ai_model_id')
->distinct()
->orderBy('model')
->get(['model'])
->map(fn (AiModelDailyUsage $row): string => $row->model);
});
}

public function hasOtherModels(): bool
{
return $this->remember('has_other_models', function () {
return AiModelDailyUsage::query()->whereNull('ai_model_id')->exists();
});
}

/**
* @return Collection<int, numeric-string>
*/
public function years(): Collection
{
return $this->remember('years', function () {
return AiModelDailyUsage::query()
->orderBy('date')
->get(['date'])
->map(fn (AiModelDailyUsage $row): string => $row->date->format('Y'))
->unique()
->values();
});
}

/**
* @return Collection<int, array{label: string, prompt_tokens: int<0, max>, completion_tokens: int<0, max>, total_tokens: int<0, max>, requests: int<0, max>}>
*/
public function monthlyBreakdown(?string $year, ?string $month, ?string $model): Collection
{
$suffix = 'breakdown_'.($year ?? 'all').'_'.($month ?? 'all').'_'.($model ?? 'all');

return $this->remember($suffix, function () use ($year, $month, $model) {
return AiModelDailyUsage::query()
->when($model === self::OTHER_MODEL, fn (Builder $query) => $query->whereNull('ai_model_id'))
->when($model && $model !== self::OTHER_MODEL, fn (Builder $query) => $query->where('model', $model))
->when($year, fn (Builder $query) => $query->whereYear('date', $year))
->when($month, fn (Builder $query) => $query->whereMonth('date', $month))
->orderBy('date')
->get()
->groupBy(fn (AiModelDailyUsage $row) => $row->date->format('Y-m'))
->map(
/** @param EloquentCollection<int, AiModelDailyUsage> $rows */
fn (EloquentCollection $rows, string $label) => [
'label' => $label,
'prompt_tokens' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->prompt_tokens),
'completion_tokens' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->completion_tokens),
'total_tokens' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->total_tokens),
'requests' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->requests),
]
)
->values();
});
}

/**
* @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
*/
public function currentMonthSummary(?string $model = null): array
{
return $this->summary('month_summary', CarbonImmutable::now()->startOfMonth(), $model);
}

/**
* @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
*/
public function currentYearSummary(?string $model = null): array
{
return $this->summary('year_summary', CarbonImmutable::now()->startOfYear(), $model);
}

/**
* @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
*/
public function totalSummary(?string $model = null): array
{
return $this->summary('total_summary', null, $model);
}

/**
* @return array{prompt_tokens: int, completion_tokens: int, total_tokens: int, requests: int}
*/
private function summary(string $suffix, ?CarbonImmutable $from, ?string $model = null): array
{
return $this->remember($suffix.'_'.($model ?? 'all'), function () use ($from, $model) {
$rows = AiModelDailyUsage::query()
->when($from, fn (Builder $query) => $query->where('date', '>=', $from))
->when($model === self::OTHER_MODEL, fn (Builder $query) => $query->whereNull('ai_model_id'))
->when($model && $model !== self::OTHER_MODEL, fn (Builder $query) => $query->where('model', $model))
->get();

return [
'prompt_tokens' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->prompt_tokens),
'completion_tokens' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->completion_tokens),
'total_tokens' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->total_tokens),
'requests' => $rows->sum(fn (AiModelDailyUsage $row): int => $row->requests),
];
});
}

/**
* @template TValue
*
* @param Closure(): TValue $callback
* @return TValue
*/
private function remember(string $suffix, Closure $callback): mixed
{
$version = Cache::get(self::VERSION_CACHE_KEY, 0);
$version = is_int($version) ? $version : 0;
$key = Str::slug("llm_usage_{$version}_{$suffix}", '_');

return Cache::remember($key, now()->addHour(), $callback);
}
}
Loading