feat: add log collection and viewing system

Log Collectors (backend/internal/collectors/logs/):
- LogEntry model with level, source, message, fields
- Manager for coordinating multiple collectors
- JournalCollector: systemd journal via journalctl CLI
- FileCollector: tail log files with format parsing (plain, json, nginx)
- DockerCollector: docker container logs via docker CLI
- All collectors are pure Go (no CGO dependencies)

Database Storage:
- Add logs table with indexes for efficient querying
- StoreLogs: batch insert log entries
- QueryLogs: filter by agent, source, level, time, full-text search
- DeleteOldLogs: retention cleanup
- Implementations for both SQLite and PostgreSQL

Frontend Log Viewer:
- Log types and level color definitions
- Logs API client with streaming support
- /logs route with search, level filters, source filters
- Live streaming mode for real-time log tailing
- Paginated loading with load more

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2025-12-28 08:52:31 +01:00
parent c0dbf80521
commit 014bc9bbb5
10 changed files with 2453 additions and 0 deletions

View File

@@ -0,0 +1,111 @@
// Logs API client
import { browser } from '$app/environment';
import { authStore } from '$lib/stores/auth';
import { get } from 'svelte/store';
import type { LogFilter, LogResponse, LogEntry } from '$lib/types/logs';
const API_BASE = '/api/v1';
// Get auth token from store
function getToken(): string | null {
if (!browser) return null;
const auth = get(authStore);
return auth.token;
}
// Make authenticated API request
async function authFetch(path: string, options: RequestInit = {}): Promise<Response> {
const token = getToken();
const headers: HeadersInit = {
'Content-Type': 'application/json',
...(options.headers || {})
};
if (token) {
(headers as Record<string, string>)['Authorization'] = `Bearer ${token}`;
}
return fetch(`${API_BASE}${path}`, {
...options,
headers
});
}
// Build query string from filter
function buildQueryString(filter: LogFilter): string {
const params = new URLSearchParams();
if (filter.agentId) params.set('agent_id', filter.agentId);
if (filter.source) params.set('source', filter.source);
if (filter.sourceName) params.set('source_name', filter.sourceName);
if (filter.query) params.set('q', filter.query);
if (filter.from) params.set('from', filter.from);
if (filter.to) params.set('to', filter.to);
if (filter.limit) params.set('limit', filter.limit.toString());
if (filter.offset) params.set('offset', filter.offset.toString());
if (filter.level && filter.level.length > 0) {
params.set('level', filter.level.join(','));
}
const qs = params.toString();
return qs ? `?${qs}` : '';
}
export const logsApi = {
// Query logs with filters
async query(filter: LogFilter = {}): Promise<LogResponse> {
const queryString = buildQueryString(filter);
const response = await authFetch(`/logs${queryString}`);
if (!response.ok) {
throw new Error('Failed to fetch logs');
}
return response.json();
},
// Get available sources
async getSources(agentId?: string): Promise<{ source: string; sourceName: string }[]> {
const params = agentId ? `?agent_id=${agentId}` : '';
const response = await authFetch(`/logs/sources${params}`);
if (!response.ok) {
throw new Error('Failed to fetch log sources');
}
return response.json();
},
// Stream logs via SSE (for live tailing)
streamLogs(
filter: LogFilter,
onEntry: (entry: LogEntry) => void,
onError: (error: Error) => void
): () => void {
const queryString = buildQueryString(filter);
const token = getToken();
const url = `${API_BASE}/logs/stream${queryString}`;
const eventSource = new EventSource(url + (token ? `&token=${token}` : ''));
eventSource.onmessage = (event) => {
try {
const entry = JSON.parse(event.data) as LogEntry;
onEntry(entry);
} catch {
// Ignore parse errors
}
};
eventSource.onerror = () => {
onError(new Error('Log stream disconnected'));
};
// Return cleanup function
return () => {
eventSource.close();
};
}
};

View File

@@ -0,0 +1,47 @@
// Log entry types
export type LogLevel = 'debug' | 'info' | 'warning' | 'error' | 'fatal';
export interface LogEntry {
id: number;
agentId: string;
timestamp: string;
source: string; // 'journal', 'file', 'docker'
sourceName: string; // Unit name, filename, container name
level: LogLevel;
message: string;
fields?: Record<string, string>;
}
export interface LogFilter {
agentId?: string;
source?: string;
sourceName?: string;
level?: LogLevel[];
query?: string;
from?: string;
to?: string;
limit?: number;
offset?: number;
}
export interface LogResponse {
entries: LogEntry[];
total: number;
}
// Level colors for display
export const levelColors: Record<LogLevel, { bg: string; text: string; border: string }> = {
debug: { bg: 'bg-slate-500/20', text: 'text-slate-400', border: 'border-slate-500/30' },
info: { bg: 'bg-blue-500/20', text: 'text-blue-400', border: 'border-blue-500/30' },
warning: { bg: 'bg-yellow-500/20', text: 'text-yellow-400', border: 'border-yellow-500/30' },
error: { bg: 'bg-red-500/20', text: 'text-red-400', border: 'border-red-500/30' },
fatal: { bg: 'bg-purple-500/20', text: 'text-purple-400', border: 'border-purple-500/30' }
};
// Source icons
export const sourceIcons: Record<string, string> = {
journal: '📜',
file: '📄',
docker: '🐳'
};

View File

@@ -0,0 +1,556 @@
<script lang="ts">
import { onMount, onDestroy } from 'svelte';
import { logsApi } from '$lib/api/logs';
import type { LogEntry, LogLevel, LogFilter } from '$lib/types/logs';
import { levelColors, sourceIcons } from '$lib/types/logs';
import { theme } from '$lib/stores/theme';
// State
let entries: LogEntry[] = [];
let total = 0;
let isLoading = true;
let error = '';
let isLive = false;
let stopStream: (() => void) | null = null;
// Filters
let filter: LogFilter = {
limit: 100,
offset: 0
};
let searchQuery = '';
let selectedLevels: LogLevel[] = [];
let selectedSource = '';
// Available options
const allLevels: LogLevel[] = ['debug', 'info', 'warning', 'error', 'fatal'];
onMount(async () => {
await loadLogs();
});
onDestroy(() => {
if (stopStream) {
stopStream();
}
});
async function loadLogs() {
isLoading = true;
error = '';
try {
const currentFilter: LogFilter = {
...filter,
query: searchQuery || undefined,
level: selectedLevels.length > 0 ? selectedLevels : undefined,
source: selectedSource || undefined
};
const response = await logsApi.query(currentFilter);
entries = response.entries || [];
total = response.total;
} catch (err) {
error = err instanceof Error ? err.message : 'Failed to load logs';
entries = [];
} finally {
isLoading = false;
}
}
function handleSearch() {
filter.offset = 0;
loadLogs();
}
function toggleLevel(level: LogLevel) {
if (selectedLevels.includes(level)) {
selectedLevels = selectedLevels.filter((l) => l !== level);
} else {
selectedLevels = [...selectedLevels, level];
}
filter.offset = 0;
loadLogs();
}
function handleSourceChange(event: Event) {
const select = event.target as HTMLSelectElement;
selectedSource = select.value;
filter.offset = 0;
loadLogs();
}
function toggleLive() {
if (isLive) {
// Stop streaming
if (stopStream) {
stopStream();
stopStream = null;
}
isLive = false;
} else {
// Start streaming
isLive = true;
const currentFilter: LogFilter = {
...filter,
query: searchQuery || undefined,
level: selectedLevels.length > 0 ? selectedLevels : undefined,
source: selectedSource || undefined
};
stopStream = logsApi.streamLogs(
currentFilter,
(entry) => {
// Add new entry at the top
entries = [entry, ...entries.slice(0, 99)];
},
(err) => {
error = err.message;
isLive = false;
}
);
}
}
function loadMore() {
filter.offset = (filter.offset || 0) + (filter.limit || 100);
loadLogs();
}
function formatTimestamp(ts: string): string {
const date = new Date(ts);
return date.toLocaleString();
}
function formatRelativeTime(ts: string): string {
const now = new Date();
const date = new Date(ts);
const diff = now.getTime() - date.getTime();
if (diff < 60000) return 'just now';
if (diff < 3600000) return `${Math.floor(diff / 60000)}m ago`;
if (diff < 86400000) return `${Math.floor(diff / 3600000)}h ago`;
return `${Math.floor(diff / 86400000)}d ago`;
}
</script>
<svelte:head>
<title>Logs - Tyto</title>
</svelte:head>
<div class="logs-page {$theme === 'light' ? 'light' : 'dark'}">
<header class="page-header">
<div class="header-left">
<a href="/" class="back-link">
<svg viewBox="0 0 20 20" fill="currentColor">
<path
fill-rule="evenodd"
d="M9.707 16.707a1 1 0 01-1.414 0l-6-6a1 1 0 010-1.414l6-6a1 1 0 011.414 1.414L5.414 9H17a1 1 0 110 2H5.414l4.293 4.293a1 1 0 010 1.414z"
clip-rule="evenodd"
/>
</svg>
</a>
<h1>Logs</h1>
<span class="log-count">{total} entries</span>
</div>
<div class="header-right">
<button class="live-button" class:active={isLive} onclick={toggleLive}>
<span class="live-dot"></span>
{isLive ? 'Stop Live' : 'Go Live'}
</button>
</div>
</header>
<!-- Filters -->
<div class="filters">
<div class="search-box">
<svg class="search-icon" viewBox="0 0 20 20" fill="currentColor">
<path
fill-rule="evenodd"
d="M8 4a4 4 0 100 8 4 4 0 000-8zM2 8a6 6 0 1110.89 3.476l4.817 4.817a1 1 0 01-1.414 1.414l-4.816-4.816A6 6 0 012 8z"
clip-rule="evenodd"
/>
</svg>
<input
type="text"
placeholder="Search logs..."
bind:value={searchQuery}
onkeydown={(e) => e.key === 'Enter' && handleSearch()}
/>
<button class="search-btn" onclick={handleSearch}>Search</button>
</div>
<div class="filter-group">
<span class="filter-label">Level:</span>
<div class="level-filters">
{#each allLevels as level}
<button
class="level-btn {levelColors[level].bg}"
class:active={selectedLevels.includes(level)}
onclick={() => toggleLevel(level)}
>
{level}
</button>
{/each}
</div>
</div>
<div class="filter-group">
<span class="filter-label">Source:</span>
<select class="source-select" value={selectedSource} onchange={handleSourceChange}>
<option value="">All sources</option>
<option value="journal">Journal</option>
<option value="file">File</option>
<option value="docker">Docker</option>
</select>
</div>
</div>
{#if error}
<div class="error-banner">{error}</div>
{/if}
{#if isLoading && entries.length === 0}
<div class="loading">Loading logs...</div>
{:else if entries.length === 0}
<div class="empty-state">
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor">
<path
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z"
/>
</svg>
<p>No logs found</p>
<span>Adjust your filters or wait for new log entries</span>
</div>
{:else}
<div class="logs-container">
{#each entries as entry (entry.id)}
<div class="log-entry {levelColors[entry.level].border}">
<div class="log-meta">
<span class="log-level {levelColors[entry.level].bg} {levelColors[entry.level].text}">
{entry.level}
</span>
<span class="log-source" title={entry.sourceName}>
{sourceIcons[entry.source] || '📋'} {entry.sourceName}
</span>
<span class="log-time" title={formatTimestamp(entry.timestamp)}>
{formatRelativeTime(entry.timestamp)}
</span>
</div>
<div class="log-message">{entry.message}</div>
{#if entry.fields && Object.keys(entry.fields).length > 0}
<div class="log-fields">
{#each Object.entries(entry.fields) as [key, value]}
<span class="field"><strong>{key}:</strong> {value}</span>
{/each}
</div>
{/if}
</div>
{/each}
{#if entries.length < total && !isLive}
<button class="load-more" onclick={loadMore} disabled={isLoading}>
{isLoading ? 'Loading...' : `Load more (${total - entries.length} remaining)`}
</button>
{/if}
</div>
{/if}
</div>
<style>
.logs-page {
min-height: 100vh;
padding: 1rem;
}
.page-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 1.5rem;
}
.header-left {
display: flex;
align-items: center;
gap: 1rem;
}
.back-link {
display: flex;
align-items: center;
color: inherit;
opacity: 0.6;
transition: opacity 0.2s;
}
.back-link:hover {
opacity: 1;
}
.back-link svg {
width: 1.25rem;
height: 1.25rem;
}
.page-header h1 {
font-size: 1.5rem;
font-weight: 600;
margin: 0;
}
.log-count {
font-size: 0.875rem;
opacity: 0.6;
}
.live-button {
display: flex;
align-items: center;
gap: 0.5rem;
padding: 0.5rem 1rem;
background: rgba(128, 128, 128, 0.2);
border: 1px solid rgba(128, 128, 128, 0.3);
border-radius: 9999px;
color: inherit;
font-size: 0.875rem;
cursor: pointer;
transition: all 0.2s;
}
.live-button.active {
background: rgba(239, 68, 68, 0.2);
border-color: rgba(239, 68, 68, 0.3);
color: #ef4444;
}
.live-dot {
width: 0.5rem;
height: 0.5rem;
border-radius: 50%;
background: currentColor;
}
.live-button.active .live-dot {
animation: pulse 1s infinite;
}
@keyframes pulse {
0%,
100% {
opacity: 1;
}
50% {
opacity: 0.5;
}
}
.filters {
display: flex;
flex-wrap: wrap;
gap: 1rem;
margin-bottom: 1.5rem;
padding: 1rem;
background: rgba(128, 128, 128, 0.1);
border-radius: 0.75rem;
}
.search-box {
display: flex;
align-items: center;
gap: 0.5rem;
flex: 1;
min-width: 200px;
}
.search-icon {
width: 1.25rem;
height: 1.25rem;
opacity: 0.5;
}
.search-box input {
flex: 1;
padding: 0.5rem 0.75rem;
background: rgba(128, 128, 128, 0.1);
border: 1px solid rgba(128, 128, 128, 0.3);
border-radius: 0.5rem;
color: inherit;
font-size: 0.875rem;
}
.search-box input:focus {
outline: none;
border-color: #3b82f6;
}
.search-btn {
padding: 0.5rem 1rem;
background: #3b82f6;
border: none;
border-radius: 0.5rem;
color: white;
font-size: 0.875rem;
cursor: pointer;
}
.filter-group {
display: flex;
align-items: center;
gap: 0.5rem;
}
.filter-label {
font-size: 0.75rem;
text-transform: uppercase;
letter-spacing: 0.05em;
opacity: 0.6;
}
.level-filters {
display: flex;
gap: 0.25rem;
}
.level-btn {
padding: 0.25rem 0.5rem;
border: 1px solid transparent;
border-radius: 0.25rem;
font-size: 0.75rem;
text-transform: uppercase;
cursor: pointer;
opacity: 0.5;
transition: opacity 0.2s;
color: inherit;
background: rgba(128, 128, 128, 0.2);
}
.level-btn.active {
opacity: 1;
border-color: currentColor;
}
.source-select {
padding: 0.375rem 0.75rem;
background: rgba(128, 128, 128, 0.1);
border: 1px solid rgba(128, 128, 128, 0.3);
border-radius: 0.5rem;
color: inherit;
font-size: 0.875rem;
}
.error-banner {
padding: 0.75rem 1rem;
background: rgba(239, 68, 68, 0.1);
border: 1px solid rgba(239, 68, 68, 0.3);
border-radius: 0.5rem;
color: #ef4444;
margin-bottom: 1rem;
}
.loading,
.empty-state {
text-align: center;
padding: 3rem 1rem;
opacity: 0.6;
}
.empty-state svg {
width: 3rem;
height: 3rem;
margin-bottom: 1rem;
}
.empty-state p {
font-size: 1.125rem;
margin: 0 0 0.5rem 0;
}
.empty-state span {
font-size: 0.875rem;
}
.logs-container {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.log-entry {
padding: 0.75rem 1rem;
background: rgba(255, 255, 255, 0.03);
border-left: 3px solid;
border-radius: 0.25rem;
}
.log-meta {
display: flex;
align-items: center;
gap: 0.75rem;
margin-bottom: 0.375rem;
font-size: 0.75rem;
}
.log-level {
padding: 0.125rem 0.5rem;
border-radius: 0.25rem;
text-transform: uppercase;
font-weight: 600;
}
.log-source {
opacity: 0.6;
}
.log-time {
opacity: 0.5;
margin-left: auto;
}
.log-message {
font-family: ui-monospace, monospace;
font-size: 0.875rem;
white-space: pre-wrap;
word-break: break-word;
}
.log-fields {
display: flex;
flex-wrap: wrap;
gap: 0.5rem;
margin-top: 0.5rem;
font-size: 0.75rem;
opacity: 0.7;
}
.field {
padding: 0.125rem 0.375rem;
background: rgba(128, 128, 128, 0.1);
border-radius: 0.25rem;
}
.load-more {
padding: 0.75rem 1rem;
background: rgba(128, 128, 128, 0.2);
border: 1px solid rgba(128, 128, 128, 0.3);
border-radius: 0.5rem;
color: inherit;
font-size: 0.875rem;
cursor: pointer;
margin-top: 1rem;
}
.load-more:hover:not(:disabled) {
background: rgba(128, 128, 128, 0.3);
}
.load-more:disabled {
opacity: 0.5;
cursor: not-allowed;
}
</style>