[update] added migration scripts for migrating news data to articles, geographic and category tagging, and default sharing templates

backend-rust/migrations/003_create_articles_table.down.sql

@@ -0,0 +1,7 @@
+-- Drop articles table and its indexes
+DROP INDEX IF EXISTS idx_articles_read_at;
+DROP INDEX IF EXISTS idx_articles_source_type;
+DROP INDEX IF EXISTS idx_articles_processing_status;
+DROP INDEX IF EXISTS idx_articles_added_at;
+DROP INDEX IF EXISTS idx_articles_published_at;
+DROP TABLE IF EXISTS articles;

backend-rust/migrations/003_create_articles_table.up.sql

@@ -0,0 +1,27 @@
+-- Create enhanced articles table to replace news table structure
+CREATE TABLE IF NOT EXISTS articles
+(
+    id                INTEGER PRIMARY KEY AUTOINCREMENT,
+    title             TEXT      NOT NULL,
+    url               TEXT      NOT NULL,
+    source_type       TEXT      NOT NULL DEFAULT 'rss',     -- 'rss', 'manual'
+    rss_content       TEXT,                                 -- RSS description/excerpt
+    full_content      TEXT,                                 -- Scraped full content
+    summary           TEXT,                                 -- AI-generated summary
+    processing_status TEXT      NOT NULL DEFAULT 'pending', -- 'pending', 'processing', 'completed', 'failed'
+    published_at      TIMESTAMP NOT NULL,
+    added_at          TIMESTAMP NOT NULL DEFAULT (datetime('now')),
+    read_at           TIMESTAMP,
+    read_count        INTEGER   NOT NULL DEFAULT 0,
+    reading_time      INTEGER,                              -- in seconds
+    ai_enabled        BOOLEAN   NOT NULL DEFAULT 1,
+    created_at        TIMESTAMP NOT NULL DEFAULT (datetime('now')),
+    updated_at        TIMESTAMP NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Create indexes for performance
+CREATE INDEX IF NOT EXISTS idx_articles_published_at ON articles (published_at);
+CREATE INDEX IF NOT EXISTS idx_articles_added_at ON articles (added_at);
+CREATE INDEX IF NOT EXISTS idx_articles_processing_status ON articles (processing_status);
+CREATE INDEX IF NOT EXISTS idx_articles_source_type ON articles (source_type);
+CREATE INDEX IF NOT EXISTS idx_articles_read_at ON articles (read_at);

backend-rust/migrations/004_create_tags_table.down.sql

@@ -0,0 +1,9 @@
+-- Drop tag system tables and indexes
+DROP INDEX IF EXISTS idx_article_tags_ai_generated;
+DROP INDEX IF EXISTS idx_article_tags_tag_id;
+DROP INDEX IF EXISTS idx_article_tags_article_id;
+DROP INDEX IF EXISTS idx_tags_usage_count;
+DROP INDEX IF EXISTS idx_tags_parent_id;
+DROP INDEX IF EXISTS idx_tags_category;
+DROP TABLE IF EXISTS article_tags;
+DROP TABLE IF EXISTS tags;

backend-rust/migrations/004_create_tags_table.up.sql

@@ -0,0 +1,31 @@
+-- Create tags table with hierarchical support
+CREATE TABLE IF NOT EXISTS tags
+(
+    id          INTEGER PRIMARY KEY AUTOINCREMENT,
+    name        TEXT      NOT NULL UNIQUE,
+    category    TEXT      NOT NULL,           -- 'geographic', 'content', 'source', 'custom'
+    description TEXT,
+    color       TEXT,                         -- Hex color for UI display
+    usage_count INTEGER   NOT NULL DEFAULT 0,
+    parent_id   INTEGER REFERENCES tags (id), -- For hierarchical tags (e.g., Country -> Region -> City)
+    created_at  TIMESTAMP NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Create article_tags junction table
+CREATE TABLE IF NOT EXISTS article_tags
+(
+    article_id       INTEGER   NOT NULL REFERENCES articles (id) ON DELETE CASCADE,
+    tag_id           INTEGER   NOT NULL REFERENCES tags (id) ON DELETE CASCADE,
+    confidence_score REAL               DEFAULT 1.0, -- AI confidence (0.0-1.0)
+    ai_generated     BOOLEAN   NOT NULL DEFAULT 0,
+    created_at       TIMESTAMP NOT NULL DEFAULT (datetime('now')),
+    PRIMARY KEY (article_id, tag_id)
+);
+
+-- Create indexes
+CREATE INDEX IF NOT EXISTS idx_tags_category ON tags (category);
+CREATE INDEX IF NOT EXISTS idx_tags_parent_id ON tags (parent_id);
+CREATE INDEX IF NOT EXISTS idx_tags_usage_count ON tags (usage_count DESC);
+CREATE INDEX IF NOT EXISTS idx_article_tags_article_id ON article_tags (article_id);
+CREATE INDEX IF NOT EXISTS idx_article_tags_tag_id ON article_tags (tag_id);
+CREATE INDEX IF NOT EXISTS idx_article_tags_ai_generated ON article_tags (ai_generated);

backend-rust/migrations/005_create_statistics_table.down.sql

@@ -0,0 +1,11 @@
+-- Drop analytics system tables and indexes
+DROP INDEX IF EXISTS idx_legacy_migration_old_filter_type;
+DROP INDEX IF EXISTS idx_share_templates_format;
+DROP INDEX IF EXISTS idx_filter_presets_user_id;
+DROP INDEX IF EXISTS idx_reading_stats_read_at;
+DROP INDEX IF EXISTS idx_reading_stats_article_id;
+DROP INDEX IF EXISTS idx_reading_stats_user_id;
+DROP TABLE IF EXISTS legacy_migration;
+DROP TABLE IF EXISTS share_templates;
+DROP TABLE IF EXISTS filter_presets;
+DROP TABLE IF EXISTS reading_stats;

backend-rust/migrations/005_create_statistics_table.up.sql

@@ -0,0 +1,50 @@
+-- Create reading statistics table
+CREATE TABLE IF NOT EXISTS reading_stats
+(
+    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id         INTEGER            DEFAULT 1,   -- For future multi-user support
+    article_id      INTEGER   NOT NULL REFERENCES articles (id) ON DELETE CASCADE,
+    read_at         TIMESTAMP NOT NULL,
+    reading_time    INTEGER,                        -- in seconds
+    completion_rate REAL               DEFAULT 1.0, -- 0.0-1.0, how much of the article was read
+    created_at      TIMESTAMP NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Create filter presets table
+CREATE TABLE IF NOT EXISTS filter_presets
+(
+    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+    name            TEXT      NOT NULL,
+    filter_criteria TEXT      NOT NULL,           -- JSON string of filter parameters
+    user_id         INTEGER            DEFAULT 1, -- For future multi-user support
+    created_at      TIMESTAMP NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Create share templates table
+CREATE TABLE IF NOT EXISTS share_templates
+(
+    id               INTEGER PRIMARY KEY AUTOINCREMENT,
+    name             TEXT      NOT NULL,
+    format           TEXT      NOT NULL, -- 'text', 'markdown', 'html', 'json'
+    template_content TEXT      NOT NULL,
+    is_default       BOOLEAN   NOT NULL DEFAULT 0,
+    created_at       TIMESTAMP NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Create legacy migration tracking table
+CREATE TABLE IF NOT EXISTS legacy_migration
+(
+    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+    old_filter_type TEXT      NOT NULL, -- 'country', 'category', etc.
+    old_value       TEXT      NOT NULL,
+    new_tag_ids     TEXT,               -- JSON array of tag IDs
+    migrated_at     TIMESTAMP NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Create indexes
+CREATE INDEX IF NOT EXISTS idx_reading_stats_user_id ON reading_stats (user_id);
+CREATE INDEX IF NOT EXISTS idx_reading_stats_article_id ON reading_stats (article_id);
+CREATE INDEX IF NOT EXISTS idx_reading_stats_read_at ON reading_stats (read_at);
+CREATE INDEX IF NOT EXISTS idx_filter_presets_user_id ON filter_presets (user_id);
+CREATE INDEX IF NOT EXISTS idx_share_templates_format ON share_templates (format);
+CREATE INDEX IF NOT EXISTS idx_legacy_migration_old_filter_type ON legacy_migration (old_filter_type);

backend-rust/migrations/006_update_settings_table.down.sql

@@ -0,0 +1,18 @@
+-- Remove enhanced settings columns and indexes
+DROP INDEX IF EXISTS idx_settings_user_id;
+DROP INDEX IF EXISTS idx_settings_category;
+
+-- Note: SQLite doesn't support DROP COLUMN, so we recreate the table
+CREATE TABLE settings_backup AS
+SELECT key, val
+FROM settings;
+DROP TABLE settings;
+CREATE TABLE settings
+(
+    key TEXT PRIMARY KEY,
+    val TEXT NOT NULL
+);
+INSERT INTO settings
+SELECT key, val
+FROM settings_backup;
+DROP TABLE settings_backup;

backend-rust/migrations/006_update_settings_table.up.sql

@@ -0,0 +1,74 @@
+-- Enhance settings table to support more structured configuration
+ALTER TABLE settings
+    ADD COLUMN category TEXT DEFAULT 'general';
+ALTER TABLE settings
+    ADD COLUMN user_id INTEGER DEFAULT 1;
+ALTER TABLE settings
+    ADD COLUMN updated_at TIMESTAMP DEFAULT (datetime('now'));
+
+-- Create index for better performance
+CREATE INDEX IF NOT EXISTS idx_settings_category ON settings (category);
+CREATE INDEX IF NOT EXISTS idx_settings_user_id ON settings (user_id);
+
+-- Insert default settings based on roadmap configuration
+INSERT OR IGNORE INTO settings (key, val, category)
+VALUES
+-- Display settings
+('default_view', 'compact', 'display'),
+('articles_per_page', '50', 'display'),
+('show_reading_time', '1', 'display'),
+('show_word_count', '0', 'display'),
+('highlight_unread', '1', 'display'),
+('theme', 'auto', 'display'),
+
+-- Analytics settings
+('analytics_enabled', '1', 'analytics'),
+('track_reading_time', '1', 'analytics'),
+('track_scroll_position', '1', 'analytics'),
+('retention_days', '365', 'analytics'),
+('aggregate_older_data', '1', 'analytics'),
+
+-- Filtering settings
+('enable_smart_suggestions', '1', 'filtering'),
+('max_recent_filters', '10', 'filtering'),
+('auto_save_filters', '1', 'filtering'),
+('default_sort', 'added_desc', 'filtering'),
+('enable_geographic_hierarchy', '1', 'filtering'),
+('auto_migrate_country_filters', '1', 'filtering'),
+
+-- Sharing settings
+('default_share_format', 'text', 'sharing'),
+('include_summary', '1', 'sharing'),
+('include_tags', '1', 'sharing'),
+('include_source', '1', 'sharing'),
+('copy_to_clipboard', '1', 'sharing'),
+
+-- AI settings
+('ai_enabled', '1', 'ai'),
+('ai_provider', 'ollama', 'ai'),
+('ai_timeout_seconds', '120', 'ai'),
+('ai_summary_enabled', '1', 'ai'),
+('ai_summary_temperature', '0.1', 'ai'),
+('ai_summary_max_tokens', '1000', 'ai'),
+('ai_tagging_enabled', '1', 'ai'),
+('ai_tagging_temperature', '0.3', 'ai'),
+('ai_tagging_max_tokens', '200', 'ai'),
+('max_tags_per_article', '10', 'ai'),
+('min_confidence_threshold', '0.7', 'ai'),
+('enable_geographic_tagging', '1', 'ai'),
+('enable_category_tagging', '1', 'ai'),
+('geographic_hierarchy_levels', '3', 'ai'),
+
+-- Scraping settings
+('scraping_timeout_seconds', '30', 'scraping'),
+('scraping_max_retries', '3', 'scraping'),
+('max_content_length', '50000', 'scraping'),
+('respect_robots_txt', '1', 'scraping'),
+('rate_limit_delay_ms', '1000', 'scraping'),
+
+-- Processing settings
+('batch_size', '10', 'processing'),
+('max_concurrent', '5', 'processing'),
+('retry_attempts', '3', 'processing'),
+('priority_manual', '1', 'processing'),
+('auto_mark_read_on_view', '0', 'processing');

backend-rust/migrations/007_migrate_data_from_news_to_articles.down.sql

@@ -0,0 +1,39 @@
+-- Remove migrated data (this will remove all articles and tags created from migration)
+-- WARNING: This will delete all migrated data
+
+-- Remove legacy migration records
+DELETE
+FROM legacy_migration
+WHERE old_filter_type IN ('country', 'category');
+
+-- Remove article-tag associations for migrated data (non-AI generated)
+DELETE
+FROM article_tags
+WHERE ai_generated = 0;
+
+-- Remove migrated geographic tags (only those created from country data)
+DELETE
+FROM tags
+WHERE tags.category = 'geographic'
+  AND EXISTS (SELECT 1 FROM news WHERE news.country = tags.name);
+
+-- Remove migrated content tags (only those created from category data)
+DELETE
+FROM tags
+WHERE tags.category = 'content'
+  AND EXISTS (SELECT 1 FROM news WHERE news.category = tags.name);
+
+-- Remove migrated articles (only those that match news entries)
+DELETE
+FROM articles
+WHERE EXISTS (SELECT 1
+              FROM news
+              WHERE news.url = articles.url
+                AND news.title = articles.title
+                AND articles.source_type = 'rss');
+
+-- Reset tag usage counts
+UPDATE tags
+SET usage_count = (SELECT COUNT(*)
+                   FROM article_tags
+                   WHERE tag_id = tags.id);

backend-rust/migrations/007_migrate_data_from_news_to_articles.up.sql

@@ -0,0 +1,84 @@
+
+-- Migrate data from old news table to new articles table
+INSERT INTO articles (title, url, summary, published_at, added_at, source_type, processing_status)
+SELECT title,
+       url,
+       summary,
+       published,
+       datetime(created_at, 'unixepoch'),
+       'rss',
+       CASE
+           WHEN summary IS NOT NULL AND summary != '' THEN 'completed'
+           ELSE 'pending'
+           END
+FROM news;
+
+-- Create geographic tags from existing country data
+INSERT OR IGNORE INTO tags (name, category, description, usage_count)
+SELECT DISTINCT country,
+                'geographic',
+                'Geographic location: ' || country,
+                COUNT(*)
+FROM news
+WHERE country IS NOT NULL
+  AND country != ''
+GROUP BY country;
+
+-- Link articles to their geographic tags
+INSERT OR IGNORE INTO article_tags (article_id, tag_id, ai_generated, confidence_score)
+SELECT a.id,
+       t.id,
+       0,  -- Not AI generated, migrated from legacy data
+       1.0 -- Full confidence for existing data
+FROM articles a
+         JOIN news n ON a.url = n.url AND a.title = n.title
+         JOIN tags t ON t.name = n.country AND t.category = 'geographic'
+WHERE n.country IS NOT NULL
+  AND n.country != '';
+
+-- Create category tags if category column exists in news table
+INSERT OR IGNORE INTO tags (name, category, description, usage_count)
+SELECT DISTINCT n.category,
+                'content',
+                'Content category: ' || n.category,
+                COUNT(*)
+FROM news n
+WHERE n.category IS NOT NULL
+  AND n.category != ''
+GROUP BY n.category;
+
+-- Link articles to their category tags
+INSERT OR IGNORE INTO article_tags (article_id, tag_id, ai_generated, confidence_score)
+SELECT a.id,
+       t.id,
+       0,  -- Not AI generated, migrated from legacy data
+       1.0 -- Full confidence for existing data
+FROM articles a
+         JOIN news n ON a.url = n.url AND a.title = n.title
+         JOIN tags t ON t.name = n.category AND t.category = 'content'
+WHERE n.category IS NOT NULL
+  AND n.category != '';
+
+-- Record migration in legacy_migration table for countries
+INSERT INTO legacy_migration (old_filter_type, old_value, new_tag_ids)
+SELECT 'country',
+       n.country,
+       '[' || GROUP_CONCAT(t.id) || ']'
+FROM (SELECT DISTINCT country FROM news WHERE country IS NOT NULL AND country != '') n
+         JOIN tags t ON t.name = n.country AND t.category = 'geographic'
+GROUP BY n.country;
+
+-- Record migration in legacy_migration table for categories (if they exist)
+INSERT INTO legacy_migration (old_filter_type, old_value, new_tag_ids)
+SELECT 'category',
+       n.category,
+       '[' || GROUP_CONCAT(t.id) || ']'
+FROM (SELECT DISTINCT category FROM news WHERE category IS NOT NULL AND category != '') n
+         JOIN tags t ON t.name = n.category AND t.category = 'content'
+GROUP BY n.category;
+
+-- Update tag usage counts
+UPDATE tags
+SET usage_count = (SELECT COUNT(*)
+                   FROM article_tags
+                   WHERE tag_id = tags.id);

backend-rust/migrations/008_insert_default_sharing_templates.down.sql

@@ -0,0 +1,4 @@
+-- Remove default sharing templates
+DELETE
+FROM share_templates
+WHERE name IN ('Default Text', 'Markdown', 'Simple Text', 'HTML Email');

backend-rust/migrations/008_insert_default_sharing_templates.up.sql

@@ -0,0 +1,39 @@
+-- Insert default sharing templates
+INSERT INTO share_templates (name, format, template_content, is_default)
+VALUES ('Default Text', 'text', '📰 {title}
+
+{summary}
+
+🏷️ Tags: {tags}
+🌍 Location: {geographic_tags}
+🔗 Source: {url}
+📅 Published: {published_at}
+
+Shared via Owly News Summariser', 1),
+
+       ('Markdown', 'markdown', '# {title}
+
+{summary}
+
+**Tags:** {tags}
+**Location:** {geographic_tags}
+**Source:** [{url}]({url})
+**Published:** {published_at}
+
+---
+*Shared via Owly News Summariser*', 1),
+
+       ('Simple Text', 'text', '{title}
+
+{summary}
+
+Source: {url}', 0),
+
+       ('HTML Email', 'html', '<h2>{title}</h2>
+<p>{summary}</p>
+<p><strong>Tags:</strong> {tags}<br>
+<strong>Location:</strong> {geographic_tags}<br>
+<strong>Source:</strong> <a href="{url}">{url}</a><br>
+<strong>Published:</strong> {published_at}</p>
+<hr>
+<small>Shared via Owly News Summariser</small>', 0);