Files
tyto/backend/internal/database/database.go
vikingowl 014bc9bbb5 feat: add log collection and viewing system
Log Collectors (backend/internal/collectors/logs/):
- LogEntry model with level, source, message, fields
- Manager for coordinating multiple collectors
- JournalCollector: systemd journal via journalctl CLI
- FileCollector: tail log files with format parsing (plain, json, nginx)
- DockerCollector: docker container logs via docker CLI
- All collectors are pure Go (no CGO dependencies)

Database Storage:
- Add logs table with indexes for efficient querying
- StoreLogs: batch insert log entries
- QueryLogs: filter by agent, source, level, time, full-text search
- DeleteOldLogs: retention cleanup
- Implementations for both SQLite and PostgreSQL

Frontend Log Viewer:
- Log types and level color definitions
- Logs API client with streaming support
- /logs route with search, level filters, source filters
- Live streaming mode for real-time log tailing
- Paginated loading with load more

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-28 08:52:31 +01:00

254 lines
8.3 KiB
Go

// Package database provides database abstraction for Tyto.
// Supports both SQLite (default) and PostgreSQL backends.
package database
import (
"context"
"time"
"tyto/internal/models"
)
// Database defines the interface for all database operations.
type Database interface {
// Lifecycle
Close() error
Migrate() error
// Metrics storage
StoreMetrics(ctx context.Context, agentID string, metrics *models.AllMetrics) error
QueryMetrics(ctx context.Context, agentID string, from, to time.Time, resolution string) ([]MetricPoint, error)
GetLatestMetrics(ctx context.Context, agentID string) (*models.AllMetrics, error)
// Agents (extends registry with persistence)
StoreAgent(ctx context.Context, agent *Agent) error
GetAgent(ctx context.Context, id string) (*Agent, error)
ListAgents(ctx context.Context) ([]*Agent, error)
UpdateAgentStatus(ctx context.Context, id string, status AgentStatus, lastSeen time.Time) error
DeleteAgent(ctx context.Context, id string) error
// Users
CreateUser(ctx context.Context, user *User) error
GetUser(ctx context.Context, id string) (*User, error)
GetUserByUsername(ctx context.Context, username string) (*User, error)
UpdateUser(ctx context.Context, user *User) error
DeleteUser(ctx context.Context, id string) error
ListUsers(ctx context.Context) ([]*User, error)
// Roles
CreateRole(ctx context.Context, role *Role) error
GetRole(ctx context.Context, id string) (*Role, error)
ListRoles(ctx context.Context) ([]*Role, error)
UpdateRole(ctx context.Context, role *Role) error
DeleteRole(ctx context.Context, id string) error
GetUserRoles(ctx context.Context, userID string) ([]*Role, error)
AssignRole(ctx context.Context, userID, roleID string) error
RemoveRole(ctx context.Context, userID, roleID string) error
// Sessions
CreateSession(ctx context.Context, session *Session) error
GetSession(ctx context.Context, token string) (*Session, error)
DeleteSession(ctx context.Context, token string) error
DeleteUserSessions(ctx context.Context, userID string) error
CleanupExpiredSessions(ctx context.Context) error
// Alerts
StoreAlert(ctx context.Context, alert *Alert) error
GetAlert(ctx context.Context, id string) (*Alert, error)
QueryAlerts(ctx context.Context, filter AlertFilter) ([]*Alert, error)
AcknowledgeAlert(ctx context.Context, id string) error
// Retention
RunRetention(ctx context.Context) error
// Logs
StoreLogs(ctx context.Context, entries []LogEntry) error
QueryLogs(ctx context.Context, filter LogFilter) ([]LogEntry, int, error)
DeleteOldLogs(ctx context.Context, before time.Time) (int, error)
}
// MetricPoint represents a single metric data point.
type MetricPoint struct {
Timestamp time.Time `json:"timestamp"`
AgentID string `json:"agentId"`
// Aggregated values
CPUAvg float64 `json:"cpuAvg"`
CPUMin float64 `json:"cpuMin"`
CPUMax float64 `json:"cpuMax"`
MemoryAvg float64 `json:"memoryAvg"`
MemoryMin float64 `json:"memoryMin"`
MemoryMax float64 `json:"memoryMax"`
DiskAvg float64 `json:"diskAvg,omitempty"`
GPUAvg float64 `json:"gpuAvg,omitempty"`
}
// AgentStatus represents agent connection state.
type AgentStatus string
const (
AgentStatusPending AgentStatus = "pending"
AgentStatusApproved AgentStatus = "approved"
AgentStatusConnected AgentStatus = "connected"
AgentStatusOffline AgentStatus = "offline"
AgentStatusRevoked AgentStatus = "revoked"
)
// Agent represents a registered monitoring agent.
type Agent struct {
ID string `json:"id"`
Name string `json:"name,omitempty"`
Hostname string `json:"hostname"`
OS string `json:"os"`
Architecture string `json:"architecture"`
Version string `json:"version"`
Capabilities []string `json:"capabilities,omitempty"`
Status AgentStatus `json:"status"`
CertSerial string `json:"certSerial,omitempty"`
CertExpiry time.Time `json:"certExpiry,omitempty"`
LastSeen time.Time `json:"lastSeen,omitempty"`
RegisteredAt time.Time `json:"registeredAt"`
Tags []string `json:"tags,omitempty"`
}
// AuthProvider indicates how a user authenticates.
type AuthProvider string
const (
AuthProviderLocal AuthProvider = "local"
AuthProviderLDAP AuthProvider = "ldap"
)
// User represents a system user.
type User struct {
ID string `json:"id"`
Username string `json:"username"`
Email string `json:"email,omitempty"`
PasswordHash []byte `json:"-"`
AuthProvider AuthProvider `json:"authProvider"`
LDAPDN string `json:"ldapDn,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
LastLogin time.Time `json:"lastLogin,omitempty"`
Disabled bool `json:"disabled"`
}
// Role represents a set of permissions.
type Role struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description,omitempty"`
Permissions []string `json:"permissions"`
IsSystem bool `json:"isSystem"`
CreatedAt time.Time `json:"createdAt"`
}
// Session represents an authenticated user session.
type Session struct {
Token string `json:"token"`
UserID string `json:"userId"`
CreatedAt time.Time `json:"createdAt"`
ExpiresAt time.Time `json:"expiresAt"`
IPAddress string `json:"ipAddress,omitempty"`
UserAgent string `json:"userAgent,omitempty"`
}
// AlertSeverity indicates alert severity level.
type AlertSeverity string
const (
AlertSeverityWarning AlertSeverity = "warning"
AlertSeverityCritical AlertSeverity = "critical"
)
// Alert represents a triggered alert.
type Alert struct {
ID string `json:"id"`
AgentID string `json:"agentId"`
Type string `json:"type"`
Severity AlertSeverity `json:"severity"`
Message string `json:"message"`
Value float64 `json:"value"`
Threshold float64 `json:"threshold"`
TriggeredAt time.Time `json:"triggeredAt"`
ResolvedAt *time.Time `json:"resolvedAt,omitempty"`
Acknowledged bool `json:"acknowledged"`
}
// AlertFilter specifies criteria for querying alerts.
type AlertFilter struct {
AgentID string
Type string
Severity AlertSeverity
Acknowledged *bool
From time.Time
To time.Time
Limit int
Offset int
}
// LogLevel represents log severity.
type LogLevel string
const (
LogLevelDebug LogLevel = "debug"
LogLevelInfo LogLevel = "info"
LogLevelWarning LogLevel = "warning"
LogLevelError LogLevel = "error"
LogLevelFatal LogLevel = "fatal"
)
// LogEntry represents a stored log entry.
type LogEntry struct {
ID int64 `json:"id"`
AgentID string `json:"agentId"`
Timestamp time.Time `json:"timestamp"`
Source string `json:"source"` // "journal", "file", "docker"
SourceName string `json:"sourceName"` // Unit name, filename, container
Level LogLevel `json:"level"`
Message string `json:"message"`
Fields map[string]string `json:"fields,omitempty"`
}
// LogFilter specifies criteria for querying logs.
type LogFilter struct {
AgentID string // Filter by agent
Source string // Filter by source type (journal, file, docker)
SourceName string // Filter by source name
Level []LogLevel // Filter by levels
Query string // Full-text search query
From time.Time
To time.Time
Limit int
Offset int
}
// RetentionConfig defines data retention policies.
type RetentionConfig struct {
// Raw metrics retention (default: 24 hours)
RawRetention time.Duration
// 1-minute aggregation retention (default: 7 days)
OneMinuteRetention time.Duration
// 5-minute aggregation retention (default: 30 days)
FiveMinuteRetention time.Duration
// Hourly aggregation retention (default: 1 year)
HourlyRetention time.Duration
// Log retention (default: 7 days)
LogRetention time.Duration
}
// DefaultRetentionConfig returns default retention settings.
func DefaultRetentionConfig() RetentionConfig {
return RetentionConfig{
RawRetention: 24 * time.Hour,
OneMinuteRetention: 7 * 24 * time.Hour,
FiveMinuteRetention: 30 * 24 * time.Hour,
HourlyRetention: 365 * 24 * time.Hour,
LogRetention: 7 * 24 * time.Hour,
}
}