From aa7a982caf6e2622a4621f751e712f06b29ef5b0 Mon Sep 17 00:00:00 2001 From: vikingowl Date: Thu, 5 Mar 2026 15:18:44 +0100 Subject: [PATCH] feat: add AI research, geocoding, duplicate detection, enriched market submit - AI research via Mistral API for admin market editing - Auto-geocoding via Nominatim OSM with rate limiting - Public geocode endpoint (POST /api/v1/geocode) - Duplicate detection using pg_trgm trigram similarity - Extended SubmitMarketRequest with street, opening_hours, admission_info - pg_trgm migration for fuzzy name matching --- backend/internal/config/config.go | 10 + .../internal/domain/market/admin_handler.go | 27 +++ backend/internal/domain/market/dto.go | 81 +++++-- .../internal/domain/market/geocode_handler.go | 41 ++++ backend/internal/domain/market/repository.go | 44 ++++ backend/internal/domain/market/research.go | 214 +++++++++++++++++ backend/internal/domain/market/routes.go | 8 +- backend/internal/domain/market/service.go | 44 +++- backend/internal/pkg/ai/client.go | 219 ++++++++++++++++++ backend/internal/pkg/geocode/nominatim.go | 93 ++++++++ backend/internal/server/routes.go | 15 +- .../migrations/000009_add_pg_trgm.down.sql | 1 + backend/migrations/000009_add_pg_trgm.up.sql | 3 + 13 files changed, 778 insertions(+), 22 deletions(-) create mode 100644 backend/internal/domain/market/geocode_handler.go create mode 100644 backend/internal/domain/market/research.go create mode 100644 backend/internal/pkg/ai/client.go create mode 100644 backend/internal/pkg/geocode/nominatim.go create mode 100644 backend/migrations/000009_add_pg_trgm.down.sql create mode 100644 backend/migrations/000009_add_pg_trgm.up.sql diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 93c5b5b..25943b9 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -20,6 +20,12 @@ type Config struct { SMTP SMTPConfig Turnstile TurnstileConfig Notification NotificationConfig + AI AIConfig +} + +type AIConfig struct { + APIKey string + Model string } type AppConfig struct { @@ -240,6 +246,10 @@ func Load() (*Config, error) { AdminEmail: envStr("ADMIN_EMAIL", ""), FrontendURL: envStr("FRONTEND_URL", "http://localhost:5173"), }, + AI: AIConfig{ + APIKey: envStr("AI_API_KEY", ""), + Model: envStr("AI_MODEL", "mistral-medium-latest"), + }, }, nil } diff --git a/backend/internal/domain/market/admin_handler.go b/backend/internal/domain/market/admin_handler.go index e53afa8..d444423 100644 --- a/backend/internal/domain/market/admin_handler.go +++ b/backend/internal/domain/market/admin_handler.go @@ -178,3 +178,30 @@ func (h *AdminHandler) UpdateStatus(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"data": gin.H{"message": "status updated"}}) } + +func (h *AdminHandler) FindDuplicates(c *gin.Context) { + id, err := uuid.Parse(c.Param("id")) + if err != nil { + apiErr := apierror.BadRequest("invalid_id", "invalid market ID") + c.JSON(apiErr.Status, apierror.NewResponse(apiErr)) + return + } + + dupes, err := h.service.FindDuplicates(c.Request.Context(), id) + if err != nil { + if errors.Is(err, ErrMarketNotFound) { + apiErr := apierror.NotFound("market") + c.JSON(apiErr.Status, apierror.NewResponse(apiErr)) + return + } + apiErr := apierror.Internal("failed to find duplicates") + c.JSON(apiErr.Status, apierror.NewResponse(apiErr)) + return + } + + if dupes == nil { + dupes = []DuplicateCandidate{} + } + + c.JSON(http.StatusOK, DuplicatesResponse{Data: dupes}) +} diff --git a/backend/internal/domain/market/dto.go b/backend/internal/domain/market/dto.go index f318323..53caf9b 100644 --- a/backend/internal/domain/market/dto.go +++ b/backend/internal/domain/market/dto.go @@ -314,24 +314,75 @@ type UpdateMarketRequest struct { Website *string `json:"website" validate:"omitempty,max=500"` OrganizerName *string `json:"organizer_name" validate:"omitempty,max=200"` ImageURL *string `json:"image_url" validate:"omitempty,max=500"` + AdminNotes *string `json:"admin_notes" validate:"omitempty,max=2000"` } type SubmitMarketRequest struct { - Name string `json:"name" validate:"required,min=3,max=200"` - Description string `json:"description" validate:"max=5000"` - Latitude *float64 `json:"latitude" validate:"omitempty,gte=-90,lte=90"` - Longitude *float64 `json:"longitude" validate:"omitempty,gte=-180,lte=180"` - City string `json:"city" validate:"required,max=100"` - State string `json:"state" validate:"max=100"` - Zip string `json:"zip" validate:"max=20"` - Country string `json:"country" validate:"required,len=2"` - StartDate string `json:"start_date" validate:"required"` - EndDate string `json:"end_date" validate:"required"` - Website string `json:"website" validate:"max=500"` - OrganizerName string `json:"organizer_name" validate:"max=200"` - SubmitterEmail string `json:"submitter_email" validate:"required,email"` - SubmitterName string `json:"submitter_name" validate:"required,min=2,max=100"` - TurnstileToken string `json:"turnstile_token" validate:"required"` + Name string `json:"name" validate:"required,min=3,max=200"` + Description string `json:"description" validate:"max=5000"` + Latitude *float64 `json:"latitude" validate:"omitempty,gte=-90,lte=90"` + Longitude *float64 `json:"longitude" validate:"omitempty,gte=-180,lte=180"` + Street string `json:"street" validate:"max=200"` + City string `json:"city" validate:"required,max=100"` + State string `json:"state" validate:"max=100"` + Zip string `json:"zip" validate:"max=20"` + Country string `json:"country" validate:"required,len=2"` + StartDate string `json:"start_date" validate:"required"` + EndDate string `json:"end_date" validate:"required"` + OpeningHours json.RawMessage `json:"opening_hours"` + AdmissionInfo json.RawMessage `json:"admission_info"` + Website string `json:"website" validate:"max=500"` + OrganizerName string `json:"organizer_name" validate:"max=200"` + ImageURL string `json:"image_url" validate:"max=500"` + SubmitterEmail string `json:"submitter_email" validate:"required,email"` + SubmitterName string `json:"submitter_name" validate:"required,min=2,max=100"` + TurnstileToken string `json:"turnstile_token" validate:"required"` +} + +type GeocodeRequest struct { + Street string `json:"street"` + City string `json:"city" validate:"required"` + Zip string `json:"zip"` + Country string `json:"country"` +} + +type GeocodeResponse struct { + Data GeocodeResult `json:"data"` +} + +type GeocodeResult struct { + Latitude *float64 `json:"latitude"` + Longitude *float64 `json:"longitude"` +} + +type ResearchResult struct { + Suggestions []FieldSuggestion `json:"suggestions"` + Sources []string `json:"sources"` +} + +type FieldSuggestion struct { + Field string `json:"field"` + CurrentValue any `json:"current_value"` + SuggestedValue any `json:"suggested_value"` + Confidence string `json:"confidence"` + Reason string `json:"reason"` +} + +type ResearchResponse struct { + Data ResearchResult `json:"data"` +} + +type DuplicateCandidate struct { + ID uuid.UUID `json:"id"` + Name string `json:"name"` + City string `json:"city"` + StartDate string `json:"start_date"` + EndDate string `json:"end_date"` + Similarity float64 `json:"similarity"` +} + +type DuplicatesResponse struct { + Data []DuplicateCandidate `json:"data"` } type UpdateStatusRequest struct { diff --git a/backend/internal/domain/market/geocode_handler.go b/backend/internal/domain/market/geocode_handler.go new file mode 100644 index 0000000..766379b --- /dev/null +++ b/backend/internal/domain/market/geocode_handler.go @@ -0,0 +1,41 @@ +package market + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "marktvogt.de/backend/internal/pkg/apierror" + "marktvogt.de/backend/internal/pkg/geocode" + "marktvogt.de/backend/internal/pkg/validate" +) + +type GeocodeHandler struct { + geocoder *geocode.Geocoder +} + +func NewGeocodeHandler(geocoder *geocode.Geocoder) *GeocodeHandler { + return &GeocodeHandler{geocoder: geocoder} +} + +func (h *GeocodeHandler) Geocode(c *gin.Context) { + var req GeocodeRequest + if apiErr := validate.BindJSON(c, &req); apiErr != nil { + c.JSON(apiErr.Status, apierror.NewResponse(apiErr)) + return + } + + lat, lon, err := h.geocoder.Geocode(c.Request.Context(), req.Street, req.City, req.Zip, req.Country) + if err != nil { + apiErr := apierror.Internal("geocoding failed") + c.JSON(apiErr.Status, apierror.NewResponse(apiErr)) + return + } + + c.JSON(http.StatusOK, GeocodeResponse{ + Data: GeocodeResult{ + Latitude: lat, + Longitude: lon, + }, + }) +} diff --git a/backend/internal/domain/market/repository.go b/backend/internal/domain/market/repository.go index ce2e0e9..e063271 100644 --- a/backend/internal/domain/market/repository.go +++ b/backend/internal/domain/market/repository.go @@ -23,6 +23,7 @@ type Repository interface { AdminSearch(ctx context.Context, params AdminSearchParams) ([]Market, int, error) UpdateStatus(ctx context.Context, id uuid.UUID, status string, reviewedBy uuid.UUID, adminNotes string) error SlugExists(ctx context.Context, slug string) (bool, error) + FindSimilar(ctx context.Context, id uuid.UUID, name, city string, startDate, endDate time.Time) ([]DuplicateCandidate, error) } type pgRepository struct { @@ -380,12 +381,14 @@ func (r *pgRepository) Update(ctx context.Context, id uuid.UUID, m Market) (Mark p6, p7, p8, p9, p10 := nextArg(), nextArg(), nextArg(), nextArg(), nextArg() p11, p12, p13, p14 := nextArg(), nextArg(), nextArg(), nextArg() p15, p16, p17 := nextArg(), nextArg(), nextArg() + p18 := nextArg() args = append(args, m.Street, m.City, m.State, m.Zip, m.Country, m.StartDate.Format(time.DateOnly), m.EndDate.Format(time.DateOnly), m.OpeningHours, m.AdmissionInfo, m.Website, m.OrganizerName, m.ImageURL, + m.AdminNotes, ) query := fmt.Sprintf(` @@ -396,6 +399,7 @@ func (r *pgRepository) Update(ctx context.Context, id uuid.UUID, m Market) (Mark start_date = %s::date, end_date = %s::date, opening_hours = %s::jsonb, admission_info = %s::jsonb, website = %s, organizer_name = %s, image_url = %s, + admin_notes = %s, updated_at = NOW() WHERE id = $1 RETURNING `+returningColumns, @@ -403,6 +407,7 @@ func (r *pgRepository) Update(ctx context.Context, id uuid.UUID, m Market) (Mark p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, + p18, ) updated, err := scanMarketFull(r.db.QueryRow(ctx, query, args...)) @@ -515,6 +520,45 @@ func (r *pgRepository) UpdateStatus(ctx context.Context, id uuid.UUID, status st return nil } +func (r *pgRepository) FindSimilar(ctx context.Context, id uuid.UUID, name, city string, startDate, endDate time.Time) ([]DuplicateCandidate, error) { + query := ` + SELECT m.id, m.name, m.city, + m.start_date, m.end_date, + similarity(m.name, $2) AS sim + FROM markets m + WHERE m.id != $1 + AND ( + m.city ILIKE $3 + OR similarity(m.name, $2) > 0.3 + ) + AND m.start_date <= $5::date + AND m.end_date >= $4::date + ORDER BY sim DESC + LIMIT 10 + ` + + rows, err := r.db.Query(ctx, query, id, name, city, + startDate.Format(time.DateOnly), endDate.Format(time.DateOnly)) + if err != nil { + return nil, fmt.Errorf("finding similar markets: %w", err) + } + defer rows.Close() + + var results []DuplicateCandidate + for rows.Next() { + var d DuplicateCandidate + var start, end time.Time + if err := rows.Scan(&d.ID, &d.Name, &d.City, &start, &end, &d.Similarity); err != nil { + return nil, fmt.Errorf("scanning duplicate: %w", err) + } + d.StartDate = start.Format(time.DateOnly) + d.EndDate = end.Format(time.DateOnly) + results = append(results, d) + } + + return results, rows.Err() +} + func (r *pgRepository) SlugExists(ctx context.Context, slug string) (bool, error) { var exists bool err := r.db.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM markets WHERE slug = $1)", slug).Scan(&exists) diff --git a/backend/internal/domain/market/research.go b/backend/internal/domain/market/research.go new file mode 100644 index 0000000..c559fd0 --- /dev/null +++ b/backend/internal/domain/market/research.go @@ -0,0 +1,214 @@ +package market + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "sync" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + "marktvogt.de/backend/internal/pkg/ai" + "marktvogt.de/backend/internal/pkg/apierror" +) + +type ResearchHandler struct { + service *Service + aiClient *ai.Client + mu sync.Mutex + cooldown map[uuid.UUID]time.Time +} + +func NewResearchHandler(service *Service, aiClient *ai.Client) *ResearchHandler { + return &ResearchHandler{ + service: service, + aiClient: aiClient, + cooldown: make(map[uuid.UUID]time.Time), + } +} + +func (h *ResearchHandler) Research(c *gin.Context) { + if !h.aiClient.Enabled() { + apiErr := apierror.BadRequest("ai_disabled", "AI research is not configured") + c.JSON(apiErr.Status, apierror.NewResponse(apiErr)) + return + } + + id, err := uuid.Parse(c.Param("id")) + if err != nil { + apiErr := apierror.BadRequest("invalid_id", "invalid market ID") + c.JSON(apiErr.Status, apierror.NewResponse(apiErr)) + return + } + + // Rate limit: 1 per market per 5 minutes + h.mu.Lock() + if last, ok := h.cooldown[id]; ok && time.Since(last) < 5*time.Minute { + h.mu.Unlock() + apiErr := apierror.BadRequest("rate_limited", "Bitte warte 5 Minuten zwischen Recherche-Aufrufen") + c.JSON(apiErr.Status, apierror.NewResponse(apiErr)) + return + } + h.cooldown[id] = time.Now() + h.mu.Unlock() + + m, err := h.service.GetByID(c.Request.Context(), id) + if err != nil { + if errors.Is(err, ErrMarketNotFound) { + apiErr := apierror.NotFound("market") + c.JSON(apiErr.Status, apierror.NewResponse(apiErr)) + return + } + apiErr := apierror.Internal("failed to get market") + c.JSON(apiErr.Status, apierror.NewResponse(apiErr)) + return + } + + prompt := buildResearchPrompt(m) + + rawResponse, err := h.aiClient.Research(c.Request.Context(), prompt) + if err != nil { + slog.ErrorContext(c.Request.Context(), "AI research failed", "market_id", id, "error", err) + apiErr := apierror.Internal("AI research failed") + c.JSON(apiErr.Status, apierror.NewResponse(apiErr)) + return + } + + result := parseResearchResponse(rawResponse, m) + c.JSON(http.StatusOK, ResearchResponse{Data: result}) +} + +func buildResearchPrompt(m Market) string { + dateRange := "" + if !m.StartDate.IsZero() && !m.EndDate.IsZero() { + dateRange = fmt.Sprintf("Zeitraum: %s bis %s\n", m.StartDate.Format("02.01.2006"), m.EndDate.Format("02.01.2006")) + } + + website := "" + if m.Website != "" { + website = fmt.Sprintf("Website: %s\n", m.Website) + } + + return fmt.Sprintf(`Recherchiere den folgenden Mittelaltermarkt und finde aktuelle Informationen. + +Name: %s +Stadt: %s +%s%s +Suche nach: +- Genaue Adresse/Standort +- Öffnungszeiten +- Eintrittspreise (Erwachsene, Kinder, ermäßigt) +- Website-URL +- Veranstalter-Name +- Kurzbeschreibung des Marktes + +Antworte ausschließlich mit einem validen JSON-Objekt (keine Markdown-Codeblöcke, keine Kommentare): +{ + "suggestions": [ + { + "field": "feldname", + "current_value": "aktueller wert oder null", + "suggested_value": "vorgeschlagener wert", + "confidence": "high|medium|low", + "reason": "kurze begründung" + } + ], + "sources": ["url1", "url2"] +} + +Mögliche Feldnamen: description, street, city, zip, website, organizer_name, opening_hours, admission_info +Für opening_hours: suggested_value als Array von {day, open, close}. Verwende für "day" NUR den deutschen Wochentag (z.B. "Donnerstag"), NICHT das Datum. Erlaubte Werte: Montag, Dienstag, Mittwoch, Donnerstag, Freitag, Samstag, Sonntag. +Für admission_info: suggested_value als {adult_cents, child_cents, reduced_cents, free_under_age, notes} (Preise in Cent) +`, m.Name, m.City, dateRange, website) +} + +func parseResearchResponse(raw string, _ Market) ResearchResult { + // Try to extract JSON from the response (may be wrapped in markdown code blocks) + cleaned := raw + if idx := findJSONStart(raw); idx >= 0 { + cleaned = raw[idx:] + if end := findJSONEnd(cleaned); end > 0 { + cleaned = cleaned[:end+1] + } + } + + // Strip single-line comments (// ...) that LLMs sometimes add to JSON + cleaned = stripJSONComments(cleaned) + + var result ResearchResult + if err := json.Unmarshal([]byte(cleaned), &result); err != nil { + slog.Warn("failed to parse AI research response as JSON", "error", err, "cleaned", cleaned) + return ResearchResult{ + Suggestions: []FieldSuggestion{{ + Field: "description", + SuggestedValue: raw, + Confidence: "low", + Reason: "AI response could not be parsed as structured data", + }}, + } + } + + return result +} + +func stripJSONComments(s string) string { + var result []byte + inString := false + escaped := false + for i := 0; i < len(s); i++ { + c := s[i] + if escaped { + result = append(result, c) + escaped = false + continue + } + if c == '\\' && inString { + result = append(result, c) + escaped = true + continue + } + if c == '"' { + inString = !inString + result = append(result, c) + continue + } + if !inString && c == '/' && i+1 < len(s) && s[i+1] == '/' { + // Skip until end of line + for i < len(s) && s[i] != '\n' { + i++ + } + continue + } + result = append(result, c) + } + return string(result) +} + +func findJSONStart(s string) int { + for i, c := range s { + if c == '{' { + return i + } + } + return -1 +} + +func findJSONEnd(s string) int { + depth := 0 + for i, c := range s { + switch c { + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return i + } + } + } + return -1 +} diff --git a/backend/internal/domain/market/routes.go b/backend/internal/domain/market/routes.go index 2768efc..5e4950a 100644 --- a/backend/internal/domain/market/routes.go +++ b/backend/internal/domain/market/routes.go @@ -2,16 +2,18 @@ package market import "github.com/gin-gonic/gin" -func RegisterRoutes(rg *gin.RouterGroup, h *Handler, subH *SubmissionHandler, submitLimit gin.HandlerFunc) { +func RegisterRoutes(rg *gin.RouterGroup, h *Handler, subH *SubmissionHandler, geoH *GeocodeHandler, submitLimit, geocodeLimit gin.HandlerFunc) { markets := rg.Group("/markets") { markets.GET("", h.Search) markets.GET("/:slug", h.GetBySlug) markets.POST("/submit", submitLimit, subH.Submit) } + + rg.POST("/geocode", geocodeLimit, geoH.Geocode) } -func RegisterAdminRoutes(rg *gin.RouterGroup, h *AdminHandler, requireAuth, requireAdmin gin.HandlerFunc) { +func RegisterAdminRoutes(rg *gin.RouterGroup, h *AdminHandler, rh *ResearchHandler, requireAuth, requireAdmin gin.HandlerFunc) { admin := rg.Group("/admin/markets", requireAuth, requireAdmin) { admin.GET("", h.List) @@ -20,5 +22,7 @@ func RegisterAdminRoutes(rg *gin.RouterGroup, h *AdminHandler, requireAuth, requ admin.PUT("/:id", h.Update) admin.DELETE("/:id", h.Delete) admin.PATCH("/:id/status", h.UpdateStatus) + admin.POST("/:id/research", rh.Research) + admin.GET("/:id/duplicates", h.FindDuplicates) } } diff --git a/backend/internal/domain/market/service.go b/backend/internal/domain/market/service.go index ce24ef7..99d6c91 100644 --- a/backend/internal/domain/market/service.go +++ b/backend/internal/domain/market/service.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" "marktvogt.de/backend/internal/pkg/email" + "marktvogt.de/backend/internal/pkg/geocode" "marktvogt.de/backend/internal/pkg/slug" "marktvogt.de/backend/internal/pkg/turnstile" ) @@ -18,6 +19,7 @@ type Service struct { repo Repository email email.Sender turnstile turnstile.Verifier + geocoder *geocode.Geocoder adminEmail string frontendURL string } @@ -26,11 +28,12 @@ func NewService(repo Repository) *Service { return &Service{repo: repo} } -func NewServiceFull(repo Repository, emailSender email.Sender, ts turnstile.Verifier, adminEmail, frontendURL string) *Service { +func NewServiceFull(repo Repository, emailSender email.Sender, ts turnstile.Verifier, geocoder *geocode.Geocoder, adminEmail, frontendURL string) *Service { return &Service{ repo: repo, email: emailSender, turnstile: ts, + geocoder: geocoder, adminEmail: adminEmail, frontendURL: frontendURL, } @@ -91,6 +94,8 @@ func (s *Service) Create(ctx context.Context, req CreateMarketRequest) (Market, m.Longitude = req.Longitude } + s.tryGeocode(ctx, &m) + return s.repo.Create(ctx, m) } @@ -156,10 +161,33 @@ func (s *Service) Update(ctx context.Context, id uuid.UUID, req UpdateMarketRequ if req.ImageURL != nil { existing.ImageURL = *req.ImageURL } + if req.AdminNotes != nil { + existing.AdminNotes = *req.AdminNotes + } return s.repo.Update(ctx, id, existing) } +func (s *Service) tryGeocode(ctx context.Context, m *Market) { + if s.geocoder == nil || m.City == "" { + return + } + hasCoords := m.Latitude != nil && m.Longitude != nil && (*m.Latitude != 0 || *m.Longitude != 0) + if hasCoords { + return + } + lat, lon, err := s.geocoder.Geocode(ctx, m.Street, m.City, m.Zip, m.Country) + if err != nil { + slog.WarnContext(ctx, "geocoding failed", "city", m.City, "error", err) + return + } + if lat != nil && lon != nil { + m.Latitude = lat + m.Longitude = lon + slog.InfoContext(ctx, "auto-geocoded market", "city", m.City, "lat", *lat, "lon", *lon) + } +} + func (s *Service) Delete(ctx context.Context, id uuid.UUID) error { return s.repo.Delete(ctx, id) } @@ -169,6 +197,14 @@ func (s *Service) AdminSearch(ctx context.Context, params AdminSearchParams) ([] return s.repo.AdminSearch(ctx, params) } +func (s *Service) FindDuplicates(ctx context.Context, id uuid.UUID) ([]DuplicateCandidate, error) { + m, err := s.repo.GetByID(ctx, id) + if err != nil { + return nil, err + } + return s.repo.FindSimilar(ctx, id, m.Name, m.City, m.StartDate, m.EndDate) +} + func (s *Service) UpdateStatus(ctx context.Context, id uuid.UUID, status string, reviewedBy uuid.UUID, adminNotes string) error { return s.repo.UpdateStatus(ctx, id, status, reviewedBy, adminNotes) } @@ -197,14 +233,18 @@ func (s *Service) SubmitMarket(ctx context.Context, req SubmitMarketRequest, rem Slug: uniqueSlug, Name: req.Name, Description: req.Description, + Street: req.Street, City: req.City, State: req.State, Zip: req.Zip, Country: req.Country, StartDate: startDate, EndDate: endDate, + OpeningHours: req.OpeningHours, + AdmissionInfo: req.AdmissionInfo, Website: req.Website, OrganizerName: req.OrganizerName, + ImageURL: req.ImageURL, Status: "pending", SubmitterEmail: &req.SubmitterEmail, SubmitterName: req.SubmitterName, @@ -214,6 +254,8 @@ func (s *Service) SubmitMarket(ctx context.Context, req SubmitMarketRequest, rem m.Longitude = req.Longitude } + s.tryGeocode(ctx, &m) + if _, err := s.repo.Create(ctx, m); err != nil { return fmt.Errorf("creating submission: %w", err) } diff --git a/backend/internal/pkg/ai/client.go b/backend/internal/pkg/ai/client.go new file mode 100644 index 0000000..4302006 --- /dev/null +++ b/backend/internal/pkg/ai/client.go @@ -0,0 +1,219 @@ +package ai + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "sync" + "time" +) + +type Client struct { + apiKey string + model string + baseURL string + client *http.Client + + mu sync.Mutex + agentID string + noAgent bool // true if agents API is not available +} + +func New(apiKey, model string) *Client { + if model == "" { + model = "mistral-medium-latest" + } + return &Client{ + apiKey: apiKey, + model: model, + baseURL: "https://api.mistral.ai/v1", + client: &http.Client{Timeout: 120 * time.Second}, + } +} + +func (c *Client) Enabled() bool { + return c.apiKey != "" +} + +// --- Agents API types --- + +type createAgentRequest struct { + Model string `json:"model"` + Name string `json:"name"` + Description string `json:"description"` + Instructions string `json:"instructions"` + Tools []agentTool `json:"tools"` +} + +type agentTool struct { + Type string `json:"type"` +} + +type createAgentResponse struct { + ID string `json:"id"` +} + +type agentCompletionRequest struct { + AgentID string `json:"agent_id"` + Messages []chatMessage `json:"messages"` +} + +// --- Chat API types --- + +type chatRequest struct { + Model string `json:"model"` + Messages []chatMessage `json:"messages"` +} + +type chatMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type chatResponse struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` +} + +func (c *Client) ensureAgent(ctx context.Context) (string, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.noAgent { + return "", fmt.Errorf("agents API not available") + } + if c.agentID != "" { + return c.agentID, nil + } + + body := createAgentRequest{ + Model: c.model, + Name: "Marktvogt Researcher", + Description: "Researches medieval market information from the web", + Instructions: "Du bist ein Recherche-Assistent für Mittelaltermärkte im DACH-Raum. " + + "Nutze die Websuche um aktuelle Informationen zu finden. " + + "Antworte immer auf Deutsch und im angeforderten JSON-Format.", + Tools: []agentTool{ + {Type: "web_search"}, + }, + } + + id, err := c.doPost(ctx, "/agents", body, func(resp []byte) (string, error) { + var r createAgentResponse + if err := json.Unmarshal(resp, &r); err != nil { + return "", err + } + return r.ID, nil + }) + if err != nil { + return "", fmt.Errorf("creating agent: %w", err) + } + + c.agentID = id + slog.Info("created Mistral research agent", "agent_id", id) + return id, nil +} + +func (c *Client) Research(ctx context.Context, prompt string) (string, error) { + if !c.Enabled() { + return "", fmt.Errorf("AI client not configured (missing API key)") + } + + // Try agents API with web search first + result, err := c.researchWithAgent(ctx, prompt) + if err == nil { + return result, nil + } + + slog.Warn("agents API failed, falling back to chat completions", "error", err) + + // Mark agents as unavailable so we skip on next call + c.mu.Lock() + c.noAgent = true + c.mu.Unlock() + + // Fallback: plain chat completions (no web search, uses training data) + return c.researchWithChat(ctx, prompt) +} + +func (c *Client) researchWithAgent(ctx context.Context, prompt string) (string, error) { + agentID, err := c.ensureAgent(ctx) + if err != nil { + return "", err + } + + body := agentCompletionRequest{ + AgentID: agentID, + Messages: []chatMessage{ + {Role: "user", Content: prompt}, + }, + } + + return c.doPost(ctx, "/agents/completions", body, parseChatResponse) +} + +func (c *Client) researchWithChat(ctx context.Context, prompt string) (string, error) { + body := chatRequest{ + Model: c.model, + Messages: []chatMessage{ + { + Role: "system", + Content: "Du bist ein Recherche-Assistent für Mittelaltermärkte im DACH-Raum. " + + "Antworte immer auf Deutsch und im angeforderten JSON-Format. " + + "Nutze dein Wissen um möglichst genaue Informationen zu liefern.", + }, + {Role: "user", Content: prompt}, + }, + } + + return c.doPost(ctx, "/chat/completions", body, parseChatResponse) +} + +func parseChatResponse(resp []byte) (string, error) { + var chatResp chatResponse + if err := json.Unmarshal(resp, &chatResp); err != nil { + return "", err + } + if len(chatResp.Choices) == 0 { + return "", fmt.Errorf("no choices in response") + } + return chatResp.Choices[0].Message.Content, nil +} + +func (c *Client) doPost(ctx context.Context, path string, body any, parse func([]byte) (string, error)) (string, error) { + jsonBody, err := json.Marshal(body) + if err != nil { + return "", fmt.Errorf("marshaling request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, bytes.NewReader(jsonBody)) + if err != nil { + return "", fmt.Errorf("creating request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.apiKey) + + resp, err := c.client.Do(req) + if err != nil { + return "", fmt.Errorf("API request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("reading response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(respBody)) + } + + return parse(respBody) +} diff --git a/backend/internal/pkg/geocode/nominatim.go b/backend/internal/pkg/geocode/nominatim.go new file mode 100644 index 0000000..f4e4f84 --- /dev/null +++ b/backend/internal/pkg/geocode/nominatim.go @@ -0,0 +1,93 @@ +package geocode + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "sync" + "time" +) + +type Geocoder struct { + client *http.Client + mu sync.Mutex + lastReq time.Time +} + +func New() *Geocoder { + return &Geocoder{ + client: &http.Client{Timeout: 10 * time.Second}, + } +} + +type nominatimResult struct { + Lat string `json:"lat"` + Lon string `json:"lon"` +} + +func (g *Geocoder) Geocode(ctx context.Context, street, city, zip, country string) (*float64, *float64, error) { + if city == "" { + return nil, nil, fmt.Errorf("city is required for geocoding") + } + + // Respect Nominatim rate limit: max 1 req/sec + g.mu.Lock() + since := time.Since(g.lastReq) + if since < time.Second { + time.Sleep(time.Second - since) + } + g.lastReq = time.Now() + g.mu.Unlock() + + params := url.Values{} + params.Set("format", "json") + params.Set("limit", "1") + params.Set("city", city) + if street != "" { + params.Set("street", street) + } + if zip != "" { + params.Set("postalcode", zip) + } + if country != "" { + params.Set("countrycodes", country) + } + + reqURL := "https://nominatim.openstreetmap.org/search?" + params.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, nil, fmt.Errorf("creating request: %w", err) + } + req.Header.Set("User-Agent", "Marktvogt/1.0") + + resp, err := g.client.Do(req) + if err != nil { + return nil, nil, fmt.Errorf("nominatim request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("nominatim returned status %d", resp.StatusCode) + } + + var results []nominatimResult + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + return nil, nil, fmt.Errorf("decoding response: %w", err) + } + + if len(results) == 0 { + return nil, nil, nil + } + + var lat, lon float64 + if _, err := fmt.Sscanf(results[0].Lat, "%f", &lat); err != nil { + return nil, nil, fmt.Errorf("parsing lat: %w", err) + } + if _, err := fmt.Sscanf(results[0].Lon, "%f", &lon); err != nil { + return nil, nil, fmt.Errorf("parsing lon: %w", err) + } + + return &lat, &lon, nil +} diff --git a/backend/internal/server/routes.go b/backend/internal/server/routes.go index ddcd909..3a58836 100644 --- a/backend/internal/server/routes.go +++ b/backend/internal/server/routes.go @@ -9,7 +9,9 @@ import ( "marktvogt.de/backend/internal/domain/market" "marktvogt.de/backend/internal/domain/user" "marktvogt.de/backend/internal/middleware" + "marktvogt.de/backend/internal/pkg/ai" "marktvogt.de/backend/internal/pkg/email" + "marktvogt.de/backend/internal/pkg/geocode" "marktvogt.de/backend/internal/pkg/turnstile" ) @@ -51,16 +53,21 @@ func (s *Server) registerRoutes() { tsVerifier := turnstile.New(s.cfg.Turnstile.SecretKey) marketRepo := market.NewRepository(s.db) - marketSvc := market.NewServiceFull(marketRepo, emailSender, tsVerifier, s.cfg.Notification.AdminEmail, s.cfg.Notification.FrontendURL) + geocoder := geocode.New() + marketSvc := market.NewServiceFull(marketRepo, emailSender, tsVerifier, geocoder, s.cfg.Notification.AdminEmail, s.cfg.Notification.FrontendURL) marketHandler := market.NewHandler(marketSvc) submissionHandler := market.NewSubmissionHandler(marketSvc) - submitLimit := middleware.RateLimit(3.0/3600.0, 3) // 3 per hour per IP - market.RegisterRoutes(v1, marketHandler, submissionHandler, submitLimit) + geocodeHandler := market.NewGeocodeHandler(geocoder) + submitLimit := middleware.RateLimit(3.0/3600.0, 3) // 3 per hour per IP + geocodeLimit := middleware.RateLimit(10.0/60.0, 10) // 10 per minute per IP + market.RegisterRoutes(v1, marketHandler, submissionHandler, geocodeHandler, submitLimit, geocodeLimit) // Admin market routes adminMarketHandler := market.NewAdminHandler(marketSvc) + aiClient := ai.New(s.cfg.AI.APIKey, s.cfg.AI.Model) + researchHandler := market.NewResearchHandler(marketSvc, aiClient) requireAdmin := middleware.RequireRole("admin") - market.RegisterAdminRoutes(v1, adminMarketHandler, requireAuth, requireAdmin) + market.RegisterAdminRoutes(v1, adminMarketHandler, researchHandler, requireAuth, requireAdmin) } func (s *Server) healthz(c *gin.Context) { diff --git a/backend/migrations/000009_add_pg_trgm.down.sql b/backend/migrations/000009_add_pg_trgm.down.sql new file mode 100644 index 0000000..7ce1fe8 --- /dev/null +++ b/backend/migrations/000009_add_pg_trgm.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_markets_name_trgm; diff --git a/backend/migrations/000009_add_pg_trgm.up.sql b/backend/migrations/000009_add_pg_trgm.up.sql new file mode 100644 index 0000000..8b3ae51 --- /dev/null +++ b/backend/migrations/000009_add_pg_trgm.up.sql @@ -0,0 +1,3 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE INDEX IF NOT EXISTS idx_markets_name_trgm ON markets USING gin (name gin_trgm_ops);