From dc36701c9d7da28985c7eb711b6c553cc72f1dc8 Mon Sep 17 00:00:00 2001 From: Meghdad Date: Sun, 10 May 2026 13:21:24 +0330 Subject: [PATCH] init project --- .env.example | 5 + .../Embeddings/RecordEmbeddingUsageEvent.php | 38 ++ app/Models/EmbeddingEntry.php | 37 ++ app/Models/EmbeddingUsageEvent.php | 46 ++ app/Models/User.php | 9 + app/Services/DashboardUsageSummary.php | 153 +++++++ app/Services/EmbeddingVector.php | 119 ++++++ app/Services/EmbeddingWorkbench.php | 404 ++++++++++++++++++ config/ai.php | 12 +- database/factories/EmbeddingEntryFactory.php | 38 ++ .../factories/EmbeddingUsageEventFactory.php | 35 ++ ..._075912_create_embedding_entries_table.php | 54 +++ ...58_create_embedding_usage_events_table.php | 39 ++ resources/views/components/app-logo.blade.php | 4 +- .../components/desktop-user-menu.blade.php | 9 +- resources/views/dashboard.blade.php | 154 ++++++- resources/views/layouts/app.blade.php | 4 +- resources/views/layouts/app/header.blade.php | 56 +-- resources/views/layouts/app/sidebar.blade.php | 14 +- .../embeddings/partials/navigation.blade.php | 31 ++ .../pages/embeddings/⚡compare.blade.php | 333 +++++++++++++++ .../views/pages/embeddings/⚡embed.blade.php | 168 ++++++++ .../views/pages/embeddings/⚡search.blade.php | 159 +++++++ routes/web.php | 13 +- tests/Feature/DashboardTest.php | 49 ++- tests/Feature/EmbeddingToolsTest.php | 235 ++++++++++ 26 files changed, 2149 insertions(+), 69 deletions(-) create mode 100644 app/Actions/Embeddings/RecordEmbeddingUsageEvent.php create mode 100644 app/Models/EmbeddingEntry.php create mode 100644 app/Models/EmbeddingUsageEvent.php create mode 100644 app/Services/DashboardUsageSummary.php create mode 100644 app/Services/EmbeddingVector.php create mode 100644 app/Services/EmbeddingWorkbench.php create mode 100644 database/factories/EmbeddingEntryFactory.php create mode 100644 database/factories/EmbeddingUsageEventFactory.php create mode 100644 database/migrations/2026_05_10_075912_create_embedding_entries_table.php create mode 100644 database/migrations/2026_05_10_092458_create_embedding_usage_events_table.php create mode 100644 resources/views/pages/embeddings/partials/navigation.blade.php create mode 100644 resources/views/pages/embeddings/⚡compare.blade.php create mode 100644 resources/views/pages/embeddings/⚡embed.blade.php create mode 100644 resources/views/pages/embeddings/⚡search.blade.php create mode 100644 tests/Feature/EmbeddingToolsTest.php diff --git a/.env.example b/.env.example index 859adc6..788b88b 100644 --- a/.env.example +++ b/.env.example @@ -40,6 +40,11 @@ QUEUE_CONNECTION=database CACHE_STORE=database # CACHE_PREFIX= +OLLAMA_URL=http://localhost:11434 +OLLAMA_EMBEDDINGS_MODEL=nomic-embed-text +OLLAMA_EMBEDDINGS_MODELS=nomic-embed-text +OLLAMA_EMBEDDINGS_DIMENSIONS=768 + MEMCACHED_HOST=127.0.0.1 REDIS_CLIENT=phpredis diff --git a/app/Actions/Embeddings/RecordEmbeddingUsageEvent.php b/app/Actions/Embeddings/RecordEmbeddingUsageEvent.php new file mode 100644 index 0000000..370b8e1 --- /dev/null +++ b/app/Actions/Embeddings/RecordEmbeddingUsageEvent.php @@ -0,0 +1,38 @@ +modelName($selectedModel); + + return EmbeddingUsageEvent::query()->create([ + 'user_id' => $user->id, + 'tool' => $tool, + 'provider' => $workbench->providerName(), + 'model' => $model, + 'embedding_dimensions' => $workbench->dimensions($model), + 'input_count' => max(0, $inputCount), + 'result_count' => max(0, $resultCount), + 'tokens' => $tokens, + ]); + } +} diff --git a/app/Models/EmbeddingEntry.php b/app/Models/EmbeddingEntry.php new file mode 100644 index 0000000..7f7f588 --- /dev/null +++ b/app/Models/EmbeddingEntry.php @@ -0,0 +1,37 @@ + */ + use HasFactory; + + /** + * Get the attributes that should be cast. + * + * @return array + */ + protected function casts(): array + { + return [ + 'embedding' => 'array', + 'embedding_dimensions' => 'integer', + 'tokens' => 'integer', + ]; + } +} diff --git a/app/Models/EmbeddingUsageEvent.php b/app/Models/EmbeddingUsageEvent.php new file mode 100644 index 0000000..2b4472d --- /dev/null +++ b/app/Models/EmbeddingUsageEvent.php @@ -0,0 +1,46 @@ + */ + use HasFactory; + + /** + * @return BelongsTo + */ + public function user(): BelongsTo + { + return $this->belongsTo(User::class); + } + + /** + * @return array + */ + protected function casts(): array + { + return [ + 'embedding_dimensions' => 'integer', + 'input_count' => 'integer', + 'result_count' => 'integer', + 'tokens' => 'integer', + ]; + } +} diff --git a/app/Models/User.php b/app/Models/User.php index b5b0918..c479fd3 100644 --- a/app/Models/User.php +++ b/app/Models/User.php @@ -7,6 +7,7 @@ use Database\Factories\UserFactory; use Illuminate\Database\Eloquent\Attributes\Fillable; use Illuminate\Database\Eloquent\Attributes\Hidden; use Illuminate\Database\Eloquent\Factories\HasFactory; +use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Foundation\Auth\User as Authenticatable; use Illuminate\Notifications\Notifiable; use Illuminate\Support\Str; @@ -43,4 +44,12 @@ class User extends Authenticatable ->map(fn ($word) => Str::substr($word, 0, 1)) ->implode(''); } + + /** + * @return HasMany + */ + public function embeddingUsageEvents(): HasMany + { + return $this->hasMany(EmbeddingUsageEvent::class); + } } diff --git a/app/Services/DashboardUsageSummary.php b/app/Services/DashboardUsageSummary.php new file mode 100644 index 0000000..9204c55 --- /dev/null +++ b/app/Services/DashboardUsageSummary.php @@ -0,0 +1,153 @@ +, + * tools: array, + * models: array, + * recent: array + * } + */ + public function forUser(User $user, int $days = 14): array + { + $days = max(7, min(30, $days)); + $startDate = CarbonImmutable::today()->subDays($days - 1); + + $events = EmbeddingUsageEvent::query() + ->whereBelongsTo($user) + ->where('created_at', '>=', $startDate->startOfDay()) + ->latest() + ->get(['tool', 'model', 'input_count', 'result_count', 'created_at']); + + return [ + 'days' => $days, + 'totals' => $this->totals($events), + 'daily' => $this->dailyActivity($events, $startDate, $days), + 'tools' => $this->toolUsage($events), + 'models' => $this->modelUsage($events), + 'recent' => $this->recentActivity($events), + ]; + } + + /** + * @param Collection $events + * @return array{actions: int, inputs: int, results: int, models: int} + */ + protected function totals(Collection $events): array + { + return [ + 'actions' => $events->count(), + 'inputs' => (int) $events->sum('input_count'), + 'results' => (int) $events->sum('result_count'), + 'models' => $events->pluck('model')->unique()->count(), + ]; + } + + /** + * @param Collection $events + * @return array + */ + protected function dailyActivity(Collection $events, CarbonImmutable $startDate, int $days): array + { + $daily = collect(range(0, $days - 1)) + ->map(function (int $offset) use ($events, $startDate): array { + $date = $startDate->addDays($offset); + $dayEvents = $events->filter(fn (EmbeddingUsageEvent $event): bool => $event->created_at->isSameDay($date)); + + return [ + 'date' => $date->toDateString(), + 'label' => $date->format('M j'), + 'actions' => $dayEvents->count(), + 'inputs' => (int) $dayEvents->sum('input_count'), + 'height' => 0, + ]; + }); + + $maxActions = max(1, (int) $daily->max('actions')); + + return $daily + ->map(fn (array $day): array => [ + ...$day, + 'height' => $day['actions'] > 0 ? max(8, (int) round(($day['actions'] / $maxActions) * 100)) : 0, + ]) + ->values() + ->all(); + } + + /** + * @param Collection $events + * @return array + */ + protected function toolUsage(Collection $events): array + { + $maxActions = max(1, $events->groupBy('tool')->map->count()->max() ?? 0); + + return $events + ->groupBy('tool') + ->map(fn (Collection $toolEvents, string $tool): array => [ + 'key' => $tool, + 'label' => Str::of($tool)->replace('_', ' ')->headline()->value(), + 'actions' => $toolEvents->count(), + 'inputs' => (int) $toolEvents->sum('input_count'), + 'results' => (int) $toolEvents->sum('result_count'), + 'percentage' => (int) round(($toolEvents->count() / $maxActions) * 100), + ]) + ->sortByDesc('actions') + ->values() + ->all(); + } + + /** + * @param Collection $events + * @return array + */ + protected function modelUsage(Collection $events): array + { + $maxActions = max(1, $events->groupBy('model')->map->count()->max() ?? 0); + + return $events + ->groupBy('model') + ->map(fn (Collection $modelEvents, string $model): array => [ + 'label' => $model, + 'actions' => $modelEvents->count(), + 'inputs' => (int) $modelEvents->sum('input_count'), + 'percentage' => (int) round(($modelEvents->count() / $maxActions) * 100), + ]) + ->sortByDesc('actions') + ->take(5) + ->values() + ->all(); + } + + /** + * @param Collection $events + * @return array + */ + protected function recentActivity(Collection $events): array + { + return $events + ->take(6) + ->map(fn (EmbeddingUsageEvent $event): array => [ + 'tool' => $event->tool, + 'tool_label' => Str::of($event->tool)->replace('_', ' ')->headline()->value(), + 'model' => $event->model, + 'inputs' => $event->input_count, + 'results' => $event->result_count, + 'created_at' => $event->created_at->diffForHumans(short: true), + ]) + ->values() + ->all(); + } +} diff --git a/app/Services/EmbeddingVector.php b/app/Services/EmbeddingVector.php new file mode 100644 index 0000000..fda6d5b --- /dev/null +++ b/app/Services/EmbeddingVector.php @@ -0,0 +1,119 @@ + + */ + public static function phrasesFromText(string $text, int $limit = 50): array + { + $lines = preg_split('/\R/u', $text) ?: []; + $phrases = []; + $seen = []; + + foreach ($lines as $line) { + $phrase = self::cleanPhrase($line); + + if ($phrase === '') { + continue; + } + + $hash = self::hashPhrase($phrase); + + if (isset($seen[$hash])) { + continue; + } + + $seen[$hash] = true; + $phrases[] = $phrase; + + if (count($phrases) >= $limit) { + break; + } + } + + return $phrases; + } + + public static function cleanPhrase(string $phrase): string + { + return Str::of($phrase)->squish()->value(); + } + + public static function hashPhrase(string $phrase): string + { + return hash('sha256', Str::of($phrase)->squish()->lower()->value()); + } + + /** + * @param array $vector + * @return array + */ + public static function normalize(array $vector): array + { + $magnitude = sqrt(array_sum(array_map( + fn (int|float $value): float => (float) $value * (float) $value, + $vector + ))); + + if ($magnitude == 0.0) { + return array_map(fn (): float => 0.0, $vector); + } + + return array_map(fn (int|float $value): float => (float) $value / $magnitude, $vector); + } + + /** + * @param array $first + * @param array $second + */ + public static function cosineSimilarity(array $first, array $second): float + { + if (count($first) !== count($second)) { + throw new InvalidArgumentException('Vectors must have the same number of dimensions.'); + } + + $dotProduct = 0.0; + $firstMagnitude = 0.0; + $secondMagnitude = 0.0; + + foreach ($first as $index => $firstValue) { + $secondValue = $second[$index]; + + $dotProduct += (float) $firstValue * (float) $secondValue; + $firstMagnitude += (float) $firstValue * (float) $firstValue; + $secondMagnitude += (float) $secondValue * (float) $secondValue; + } + + if ($firstMagnitude == 0.0 || $secondMagnitude == 0.0) { + return 0.0; + } + + return max(-1.0, min(1.0, $dotProduct / (sqrt($firstMagnitude) * sqrt($secondMagnitude)))); + } + + /** + * @param array $vector + * @return array + */ + public static function rounded(array $vector, int $precision = 6): array + { + return array_map(fn (int|float $value): float => round((float) $value, $precision), $vector); + } + + /** + * @param array $vector + * @return array + */ + public static function preview(array $vector, int $limit = 12): array + { + return array_slice(self::rounded($vector), 0, $limit); + } +} diff --git a/app/Services/EmbeddingWorkbench.php b/app/Services/EmbeddingWorkbench.php new file mode 100644 index 0000000..5cfcbff --- /dev/null +++ b/app/Services/EmbeddingWorkbench.php @@ -0,0 +1,404 @@ +|null + */ + protected ?Collection $ollamaModelNames = null; + + public function providerName(): string + { + $provider = config('ai.default_for_embeddings', 'ollama'); + + return $provider instanceof Lab ? $provider->value : (string) $provider; + } + + public function modelName(?string $model = null): string + { + $model = trim((string) $model); + + if ($model !== '') { + return $this->validateModel($model); + } + + $default = Ai::embeddingProvider($this->providerName())->defaultEmbeddingsModel(); + $availableModels = $this->availableModelNames(); + + if ($availableModels->contains($default)) { + return $default; + } + + $firstAvailableModel = $availableModels->first(); + + if ($firstAvailableModel === null) { + throw new InvalidArgumentException('No embedding models are available.'); + } + + return $firstAvailableModel; + } + + public function dimensions(?string $model = null): int + { + $this->modelName($model); + + return Ai::embeddingProvider($this->providerName())->defaultEmbeddingsDimensions(); + } + + /** + * @return array + */ + public function modelOptions(): array + { + $default = Ai::embeddingProvider($this->providerName())->defaultEmbeddingsModel(); + + return $this->availableModelNames() + ->map(fn (string $model): array => [ + 'name' => $model, + 'dimensions' => $this->dimensions($model), + 'default' => $model === $default, + ]) + ->all(); + } + + /** + * @param list $phrases + * @return Collection + */ + public function embed(array $phrases, ?string $model = null): Collection + { + $phrases = $this->uniquePhrases($phrases); + + if ($phrases === []) { + return collect(); + } + + $provider = $this->providerName(); + $model = $this->modelName($model); + $dimensions = $this->dimensions($model); + $hashes = collect($phrases)->mapWithKeys(fn (string $phrase): array => [ + EmbeddingVector::hashPhrase($phrase) => $phrase, + ]); + + $entries = EmbeddingEntry::query() + ->where('provider', $provider) + ->where('model', $model) + ->where('embedding_dimensions', $dimensions) + ->whereIn('content_hash', $hashes->keys()) + ->get() + ->keyBy('content_hash'); + + $missingPhrases = $hashes + ->reject(fn (string $phrase, string $hash): bool => $entries->has($hash)) + ->values() + ->all(); + + if ($missingPhrases !== []) { + $response = Embeddings::for($missingPhrases) + ->dimensions($dimensions) + ->cache() + ->timeout(60) + ->generate($provider, $model); + + foreach ($missingPhrases as $index => $phrase) { + $embedding = array_map('floatval', $response->embeddings[$index] ?? []); + + $this->ensureExpectedDimensions($embedding, $dimensions); + + $entry = EmbeddingEntry::query()->updateOrCreate( + [ + 'content_hash' => EmbeddingVector::hashPhrase($phrase), + 'provider' => $response->meta->provider, + 'model' => $response->meta->model, + 'embedding_dimensions' => $dimensions, + ], + [ + 'source_text' => $phrase, + 'embedding' => $embedding, + 'tokens' => $response->tokens, + ] + ); + + $entries->put($entry->content_hash, $entry); + } + } + + return collect($phrases) + ->map(fn (string $phrase): ?EmbeddingEntry => $entries->get(EmbeddingVector::hashPhrase($phrase))) + ->filter() + ->values(); + } + + /** + * @return Collection + */ + public function search(string $query, int $limit = 10, float $minimumSimilarity = 0.3, ?string $model = null): Collection + { + $query = EmbeddingVector::cleanPhrase($query); + + if ($query === '') { + return collect(); + } + + $limit = max(1, min(50, $limit)); + $minimumSimilarity = max(0.0, min(1.0, $minimumSimilarity)); + $provider = $this->providerName(); + $model = $this->modelName($model); + $dimensions = $this->dimensions($model); + $queryEmbedding = $this->generateEmbedding($query, $model); + + if ($this->canUseVectorQueries()) { + return EmbeddingEntry::query() + ->select('embedding_entries.*') + ->selectVectorDistance('embedding', $queryEmbedding, as: 'distance') + ->where('provider', $provider) + ->where('model', $model) + ->where('embedding_dimensions', $dimensions) + ->whereVectorSimilarTo('embedding', $queryEmbedding, minSimilarity: $minimumSimilarity) + ->limit($limit) + ->get() + ->map(fn (EmbeddingEntry $entry): array => [ + 'entry' => $entry, + 'similarity' => round(1 - (float) $entry->distance, 6), + 'distance' => round((float) $entry->distance, 6), + ]); + } + + return EmbeddingEntry::query() + ->where('provider', $provider) + ->where('model', $model) + ->where('embedding_dimensions', $dimensions) + ->get() + ->map(function (EmbeddingEntry $entry) use ($queryEmbedding): array { + $similarity = EmbeddingVector::cosineSimilarity($queryEmbedding, $entry->embedding); + + return [ + 'entry' => $entry, + 'similarity' => round($similarity, 6), + 'distance' => round(1 - $similarity, 6), + ]; + }) + ->filter(fn (array $result): bool => $result['similarity'] >= $minimumSimilarity) + ->sortByDesc('similarity') + ->take($limit) + ->values(); + } + + /** + * @param list $phrases + * @return array{entries: Collection, matrix: array}>, closest_pair: array{first: EmbeddingEntry, second: EmbeddingEntry, similarity: float}|null} + */ + public function compare(array $phrases, ?string $model = null): array + { + $entries = $this->embed($phrases, $model)->values(); + $matrix = []; + $closestPair = null; + + foreach ($entries as $rowIndex => $entry) { + $scores = []; + + foreach ($entries as $columnIndex => $comparedEntry) { + $similarity = round(EmbeddingVector::cosineSimilarity($entry->embedding, $comparedEntry->embedding), 6); + $scores[] = $similarity; + + if ($rowIndex < $columnIndex && ( + $closestPair === null || $similarity > $closestPair['similarity'] + )) { + $closestPair = [ + 'first' => $entry, + 'second' => $comparedEntry, + 'similarity' => $similarity, + ]; + } + } + + $matrix[] = [ + 'entry' => $entry, + 'scores' => $scores, + ]; + } + + return [ + 'entries' => $entries, + 'matrix' => $matrix, + 'closest_pair' => $closestPair, + ]; + } + + /** + * @return array{id: int, source_text: string, provider: string, model: string, dimensions: int, tokens: int|null, vector_preview: array, vector_json: string} + */ + public function presentEntry(EmbeddingEntry $entry): array + { + return [ + 'id' => $entry->id, + 'source_text' => $entry->source_text, + 'provider' => $entry->provider, + 'model' => $entry->model, + 'dimensions' => $entry->embedding_dimensions, + 'tokens' => $entry->tokens, + 'vector_preview' => EmbeddingVector::preview($entry->embedding), + 'vector_json' => json_encode(EmbeddingVector::rounded($entry->embedding), JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR), + ]; + } + + /** + * @param array $phrases + * @return list + */ + protected function uniquePhrases(array $phrases): array + { + $unique = []; + + foreach ($phrases as $phrase) { + $phrase = EmbeddingVector::cleanPhrase($phrase); + + if ($phrase === '') { + continue; + } + + $unique[EmbeddingVector::hashPhrase($phrase)] = $phrase; + } + + return array_values($unique); + } + + /** + * @return array + */ + protected function generateEmbedding(string $text, ?string $model = null): array + { + $model = $this->modelName($model); + $dimensions = $this->dimensions($model); + $embedding = array_map( + 'floatval', + Embeddings::for([$text]) + ->dimensions($dimensions) + ->cache() + ->timeout(60) + ->generate($this->providerName(), $model) + ->first() + ); + + $this->ensureExpectedDimensions($embedding, $dimensions); + + return $embedding; + } + + /** + * @param array $embedding + */ + protected function ensureExpectedDimensions(array $embedding, int $dimensions): void + { + if (count($embedding) !== $dimensions) { + throw new RuntimeException('The embedding model returned '.count($embedding)." dimensions, but the database is configured for {$dimensions}."); + } + } + + protected function canUseVectorQueries(): bool + { + return DB::connection()->getDriverName() === 'pgsql'; + } + + /** + * @return Collection + */ + protected function availableModelNames(): Collection + { + if ($this->providerName() === 'ollama') { + $ollamaModels = $this->ollamaModelNames(); + + if ($ollamaModels->isNotEmpty()) { + return $ollamaModels; + } + } + + return $this->configuredModelNames(); + } + + /** + * @return Collection + */ + protected function configuredModelNames(): Collection + { + $models = config('ai.providers.'.$this->providerName().'.models.embeddings.available', []); + + if (! is_array($models)) { + $models = []; + } + + return collect($models) + ->push(Ai::embeddingProvider($this->providerName())->defaultEmbeddingsModel()) + ->map(fn (mixed $model): string => trim((string) $model)) + ->filter() + ->unique() + ->values(); + } + + /** + * @return Collection + */ + protected function ollamaModelNames(): Collection + { + if ($this->ollamaModelNames !== null) { + return $this->ollamaModelNames; + } + + try { + $models = Http::baseUrl($this->ollamaUrl()) + ->acceptJson() + ->connectTimeout(2) + ->timeout(5) + ->get('api/tags') + ->throw() + ->json('models', []); + } catch (Throwable) { + return $this->ollamaModelNames = collect(); + } + + if (! is_array($models)) { + return $this->ollamaModelNames = collect(); + } + + return $this->ollamaModelNames = collect($models) + ->map(fn (mixed $model): string => is_array($model) ? trim((string) ($model['name'] ?? '')) : '') + ->filter() + ->unique() + ->values(); + } + + protected function ollamaUrl(): string + { + return rtrim((string) config('ai.providers.ollama.url', 'http://localhost:11434'), '/'); + } + + protected function validateModel(string $model): string + { + $model = trim($model); + + if ($model === '') { + throw new InvalidArgumentException('Choose an embedding model.'); + } + + $availableModels = $this->availableModelNames(); + + if ($availableModels->doesntContain($model)) { + throw new InvalidArgumentException("The embedding model [{$model}] is not available."); + } + + return $model; + } +} diff --git a/config/ai.php b/config/ai.php index f722b28..d23f174 100644 --- a/config/ai.php +++ b/config/ai.php @@ -17,7 +17,7 @@ return [ 'default_for_images' => 'gemini', 'default_for_audio' => 'openai', 'default_for_transcription' => 'openai', - 'default_for_embeddings' => 'openai', + 'default_for_embeddings' => 'ollama', 'default_for_reranking' => 'cohere', /* @@ -116,6 +116,16 @@ return [ 'driver' => 'ollama', 'key' => env('OLLAMA_API_KEY', ''), 'url' => env('OLLAMA_URL', 'http://localhost:11434'), + 'models' => [ + 'embeddings' => [ + 'default' => env('OLLAMA_EMBEDDINGS_MODEL', 'nomic-embed-text'), + 'available' => array_values(array_filter(array_map( + 'trim', + explode(',', env('OLLAMA_EMBEDDINGS_MODELS', env('OLLAMA_EMBEDDINGS_MODEL', 'nomic-embed-text'))) + ))), + 'dimensions' => (int) env('OLLAMA_EMBEDDINGS_DIMENSIONS', 768), + ], + ], ], 'openai' => [ diff --git a/database/factories/EmbeddingEntryFactory.php b/database/factories/EmbeddingEntryFactory.php new file mode 100644 index 0000000..5593221 --- /dev/null +++ b/database/factories/EmbeddingEntryFactory.php @@ -0,0 +1,38 @@ + + */ +class EmbeddingEntryFactory extends Factory +{ + /** + * Define the model's default state. + * + * @return array + */ + public function definition(): array + { + $sourceText = fake()->sentence(); + $dimensions = (int) config('ai.providers.ollama.models.embeddings.dimensions', 768); + $embedding = EmbeddingVector::normalize( + array_map(fn () => fake()->randomFloat(6, -1, 1), range(1, $dimensions)) + ); + + return [ + 'source_text' => $sourceText, + 'content_hash' => hash('sha256', Str::of($sourceText)->squish()->lower()->value()), + 'embedding' => $embedding, + 'embedding_dimensions' => $dimensions, + 'provider' => 'ollama', + 'model' => config('ai.providers.ollama.models.embeddings.default', 'nomic-embed-text'), + 'tokens' => fake()->numberBetween(1, 100), + ]; + } +} diff --git a/database/factories/EmbeddingUsageEventFactory.php b/database/factories/EmbeddingUsageEventFactory.php new file mode 100644 index 0000000..a0a5440 --- /dev/null +++ b/database/factories/EmbeddingUsageEventFactory.php @@ -0,0 +1,35 @@ + + */ +class EmbeddingUsageEventFactory extends Factory +{ + /** + * Define the model's default state. + * + * @return array + */ + public function definition(): array + { + $tool = fake()->randomElement(['embed', 'search', 'compare']); + $inputCount = fake()->numberBetween(1, 8); + + return [ + 'user_id' => User::factory(), + 'tool' => $tool, + 'provider' => 'ollama', + 'model' => config('ai.providers.ollama.models.embeddings.default', 'nomic-embed-text'), + 'embedding_dimensions' => (int) config('ai.providers.ollama.models.embeddings.dimensions', 768), + 'input_count' => $inputCount, + 'result_count' => $tool === 'search' ? fake()->numberBetween(0, 10) : $inputCount, + 'tokens' => fake()->optional()->numberBetween(5, 500), + ]; + } +} diff --git a/database/migrations/2026_05_10_075912_create_embedding_entries_table.php b/database/migrations/2026_05_10_075912_create_embedding_entries_table.php new file mode 100644 index 0000000..6890851 --- /dev/null +++ b/database/migrations/2026_05_10_075912_create_embedding_entries_table.php @@ -0,0 +1,54 @@ +getDriverName(); + $dimensions = (int) config('ai.providers.ollama.models.embeddings.dimensions', 768); + + if ($driver === 'pgsql') { + Schema::ensureVectorExtensionExists(); + } + + Schema::create('embedding_entries', function (Blueprint $table) use ($dimensions, $driver) { + $table->id(); + $table->text('source_text'); + $table->string('content_hash', 64); + $table->string('provider'); + $table->string('model'); + $table->unsignedSmallInteger('embedding_dimensions')->default($dimensions); + $table->unsignedInteger('tokens')->nullable(); + + if ($driver === 'pgsql') { + $table->vector('embedding', dimensions: $dimensions); + $table->vectorIndex('embedding'); + } else { + $table->json('embedding'); + } + + $table->timestamps(); + + $table->unique( + ['content_hash', 'provider', 'model', 'embedding_dimensions'], + 'embedding_entries_unique_embedding' + ); + $table->index(['provider', 'model', 'embedding_dimensions']); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('embedding_entries'); + } +}; diff --git a/database/migrations/2026_05_10_092458_create_embedding_usage_events_table.php b/database/migrations/2026_05_10_092458_create_embedding_usage_events_table.php new file mode 100644 index 0000000..2dc1a23 --- /dev/null +++ b/database/migrations/2026_05_10_092458_create_embedding_usage_events_table.php @@ -0,0 +1,39 @@ +id(); + $table->foreignId('user_id')->constrained()->cascadeOnDelete(); + $table->string('tool', 32); + $table->string('provider'); + $table->string('model'); + $table->unsignedSmallInteger('embedding_dimensions'); + $table->unsignedInteger('input_count')->default(0); + $table->unsignedInteger('result_count')->default(0); + $table->unsignedInteger('tokens')->nullable(); + $table->timestamps(); + + $table->index(['user_id', 'created_at']); + $table->index(['user_id', 'tool', 'created_at']); + $table->index(['user_id', 'model']); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('embedding_usage_events'); + } +}; diff --git a/resources/views/components/app-logo.blade.php b/resources/views/components/app-logo.blade.php index 26e8f68..d4319fd 100644 --- a/resources/views/components/app-logo.blade.php +++ b/resources/views/components/app-logo.blade.php @@ -3,13 +3,13 @@ ]) @if($sidebar) - + @else - + diff --git a/resources/views/components/desktop-user-menu.blade.php b/resources/views/components/desktop-user-menu.blade.php index 7fb3706..967c936 100644 --- a/resources/views/components/desktop-user-menu.blade.php +++ b/resources/views/components/desktop-user-menu.blade.php @@ -1,9 +1,8 @@ - - + diff --git a/resources/views/dashboard.blade.php b/resources/views/dashboard.blade.php index 8f08c05..f1d4283 100644 --- a/resources/views/dashboard.blade.php +++ b/resources/views/dashboard.blade.php @@ -1,18 +1,146 @@ -
-
-
- -
-
- -
-
- + @php + $stats = [ + ['label' => __('Actions'), 'value' => number_format($usage['totals']['actions']), 'meta' => __('Last :days days', ['days' => $usage['days']]), 'color' => 'bg-emerald-500'], + ['label' => __('Phrases'), 'value' => number_format($usage['totals']['inputs']), 'meta' => __('Submitted inputs'), 'color' => 'bg-sky-500'], + ['label' => __('Results'), 'value' => number_format($usage['totals']['results']), 'meta' => __('Returned rows'), 'color' => 'bg-amber-500'], + ['label' => __('Models'), 'value' => number_format($usage['totals']['models']), 'meta' => __('Used locally'), 'color' => 'bg-zinc-500'], + ]; + @endphp + +
+
+
+ {{ __('Usage overview') }} + {{ __('Last :days days', ['days' => $usage['days']]) }}
+ + + {{ __('New embedding') }} +
-
- + +
+ @foreach ($stats as $stat) +
+
+
+ {{ $stat['label'] }} +
{{ $stat['value'] }}
+
+ +
+
{{ $stat['meta'] }}
+
+ @endforeach
-
+ +
+
+
+ {{ __('Activity') }} + {{ __('Actions') }} +
+ +
+ @foreach ($usage['daily'] as $day) +
+
+
+
+
{{ $day['label'] }}
+
+ @endforeach +
+
+ +
+ {{ __('Tool mix') }} + +
+ @forelse ($usage['tools'] as $tool) +
+
+ {{ $tool['label'] }} + {{ number_format($tool['actions']) }} +
+
+
+
+
+ {{ __(':inputs inputs, :results results', ['inputs' => number_format($tool['inputs']), 'results' => number_format($tool['results'])]) }} +
+
+ @empty +
+ {{ __('No usage yet.') }} +
+ @endforelse +
+
+
+ +
+
+ {{ __('Models') }} + +
+ @forelse ($usage['models'] as $model) +
+
+ {{ $model['label'] }} + {{ number_format($model['actions']) }} +
+
+
+
+
+ {{ __(':inputs inputs', ['inputs' => number_format($model['inputs'])]) }} +
+
+ @empty +
+ {{ __('No model usage yet.') }} +
+ @endforelse +
+
+ +
+ {{ __('Recent activity') }} + +
+ + + + + + + + + + + + @forelse ($usage['recent'] as $event) + + + + + + + + @empty + + + + @endforelse + +
{{ __('Tool') }}{{ __('Model') }}{{ __('Inputs') }}{{ __('Results') }}{{ __('When') }}
{{ $event['tool_label'] }}{{ $event['model'] }}{{ number_format($event['inputs']) }}{{ number_format($event['results']) }}{{ $event['created_at'] }}
{{ __('No recent activity.') }}
+
+
+
+
diff --git a/resources/views/layouts/app.blade.php b/resources/views/layouts/app.blade.php index 037dd1b..b7f50df 100644 --- a/resources/views/layouts/app.blade.php +++ b/resources/views/layouts/app.blade.php @@ -1,5 +1,5 @@ - + {{ $slot }} - + diff --git a/resources/views/layouts/app/header.blade.php b/resources/views/layouts/app/header.blade.php index 989b8e4..c3f85b2 100644 --- a/resources/views/layouts/app/header.blade.php +++ b/resources/views/layouts/app/header.blade.php @@ -13,34 +13,22 @@ {{ __('Dashboard') }} + + + {{ __('Embed') }} + + + + {{ __('Search') }} + + + + {{ __('Compare') }} + - - - - - - - - - - - - @@ -56,19 +44,21 @@ {{ __('Dashboard') }} + + {{ __('Embed') }} + + + + {{ __('Search') }} + + + + {{ __('Compare') }} + - - - - {{ __('Repository') }} - - - {{ __('Documentation') }} - - {{ $slot }} diff --git a/resources/views/layouts/app/sidebar.blade.php b/resources/views/layouts/app/sidebar.blade.php index 6db2170..26e6a81 100644 --- a/resources/views/layouts/app/sidebar.blade.php +++ b/resources/views/layouts/app/sidebar.blade.php @@ -15,21 +15,15 @@ {{ __('Dashboard') }} + + + {{ __('Embeddings') }} + - - - {{ __('Repository') }} - - - - {{ __('Documentation') }} - - -