[refactor] removed analytics, config, and modular crates to simplify the codebase and streamline architecture

backend-rust/Cargo.lock

@@ -59,9 +59,11 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
+ "axum",
  "once_cell",
  "serde",
  "serde_json",
+ "sqlx",
  "toml",
  "tracing",
 ]
@@ -320,6 +322,16 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "db"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "api",
+ "sqlx",
+ "tracing",
+]
+
 [[package]]
 name = "der"
 version = "0.7.10"
@@ -869,6 +881,7 @@ version = "0.30.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149"
 dependencies = [
+ "cc",
  "pkg-config",
  "vcpkg",
 ]
@@ -1394,6 +1407,7 @@ dependencies = [
  "anyhow",
  "api",
  "axum",
+ "db",
  "dotenv",
  "http",
  "once_cell",

backend-rust/Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "crates/api",
     "crates/server",
     "crates/cli",
+    "crates/db",
 ]
 resolver = "3"
 

backend-rust/crates/api/Cargo.toml

@@ -11,6 +11,8 @@ once_cell = { workspace = true }
 toml = { workspace = true }
 tracing = { workspace = true }
 async-trait = "0.1.89"
+axum = { workspace = true }
+sqlx = { workspace = true, features = ["sqlite"] }
 
 [features]
 default = []

backend-rust/crates/api/src/lib.rs

@@ -3,3 +3,4 @@
 pub mod config;
 pub mod types;
 pub mod services;
+pub mod api;

backend-rust/crates/api/src/services.rs

@@ -1,6 +1,16 @@
 use crate::types::Health;
 use async_trait::async_trait;
 
+// Submodules that host various domain services. These were refactored from the
+// legacy root src folder into this workspace crate. Each component is its own module file.
+pub mod summary_service;
+pub mod news_service;
+pub mod scraping_service;
+pub mod tagging_service;
+pub mod analytics_service;
+pub mod sharing_service;
+pub(crate) mod content_processor;
+
 // Implement your service traits here. Example:
 #[async_trait]
 pub trait HealthService: Send + Sync {

backend-rust/crates/api/src/services/analytics_service.rs

@@ -0,0 +1,4 @@
+//! Analytics service module.
+//! Implement logic for tracking and aggregating analytics here.
+
+// Placeholder for analytics-related types and functions.

backend-rust/crates/api/src/services/content_processor.rs

@@ -0,0 +1,3 @@
+//! Content processor utilities shared by services.
+
+// Placeholder module for content processing helpers (e.g., cleaning, tokenization).

backend-rust/crates/api/src/services/news_service.rs

@@ -0,0 +1,4 @@
+//! News service module.
+//! Implement logic related to news retrieval/management here.
+
+// Placeholder for news-related types and functions.

backend-rust/crates/api/src/services/scraping_service.rs

@@ -0,0 +1,4 @@
+//! Scraping service module.
+//! Implement logic related to web scraping, fetchers, and extractors here.
+
+// Placeholder for scraping-related types and functions.

backend-rust/crates/api/src/services/sharing_service.rs

@@ -0,0 +1,4 @@
+//! Sharing service module.
+//! Implement logic related to content sharing here.
+
+// Placeholder for sharing-related types and functions.

backend-rust/crates/api/src/services/summary_service.rs

@@ -0,0 +1,4 @@
+//! Summary service module.
+//! Implement logic for generating summaries from articles here.
+
+// Placeholder for summary-related types and functions.

backend-rust/crates/api/src/services/tagging_service.rs

@@ -0,0 +1,4 @@
+//! Tagging service module.
+//! Implement logic related to tagging articles and managing tags here.
+
+// Placeholder for tagging-related types and functions.

backend-rust/crates/app/Cargo.toml

@@ -1,14 +0,0 @@
-[package]
-name = "owly-news"
-version.workspace = true
-edition.workspace = true
-
-[dependencies]
-owly-news-api = { path = "../api" }
-owly-news-module-host = { path = "../module-host" }
-tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync"] }
-tracing = { workspace = true }
-tracing-subscriber = { workspace = true, features = ["env-filter", "json"] }
-anyhow = { workspace = true }
-serde_json = { workspace = true }
-num_cpus = { workspace = true }

backend-rust/crates/app/src/main.rs

@@ -1,45 +0,0 @@
-use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
-
-#[tokio::main(flavor = "multi_thread")]
-async fn main() -> anyhow::Result<()> {
-    // Tracing setup
-    tracing_subscriber::registry()
-        .with(
-            tracing_subscriber::EnvFilter::try_from_default_env()
-                .unwrap_or_else(|_| "info".into()),
-        )
-        .with(tracing_subscriber::fmt::layer())
-        .init();
-
-    // Limit worker threads for CPU control (can be tuned via env)
-    // Note: When using #[tokio::main], configure via env TOKIO_WORKER_THREADS.
-    // Alternatively, build a Runtime manually for stricter control.
-    if let Ok(threads) = std::env::var("TOKIO_WORKER_THREADS") {
-        tracing::warn!(
-            "TOKIO_WORKER_THREADS is set to {threads}, ensure it matches deployment requirements"
-        );
-    } else {
-        // Provide a sane default via env if not set
-        let default_threads = std::cmp::max(1, num_cpus::get_physical() / 2);
-        unsafe { std::env::set_var("TOKIO_WORKER_THREADS", default_threads.to_string()); }
-        tracing::info!("Defaulting worker threads to {}", default_threads);
-    }
-
-    // Example: lazily load and invoke the "summarizer" module when needed
-    let host = owly_news_module_host::ModuleHost::default();
-
-    // Simulate an on-demand call (e.g., from an HTTP handler)
-    let summarizer = host.get("summarizer").await?;
-    let resp = summarizer.invoke_json(
-        "summarize",
-        serde_json::json!({
-            "text": "Rust enables fearless concurrency with strong guarantees over memory safety.",
-            "ratio": 0.3
-        }),
-    )?;
-    tracing::info!(?resp, "summarizer response");
-
-    // TODO: wire this into your API routes/handlers, using the host.get("<module>").await when needed.
-    tracing::info!("owly-news daemon running");
-    Ok(())
-}

backend-rust/crates/db/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "db"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = { workspace = true }
+sqlx = { workspace = true, features = ["sqlite"] }
+tracing = { workspace = true }
+api = { path = "../api" }

backend-rust/crates/db/src/lib.rs

@@ -1,4 +1,4 @@
-use crate::config::AppSettings;
+use api::config::AppSettings;
 use anyhow::{Context, Result};
 use sqlx::migrate::Migrator;
 use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
@@ -7,10 +7,14 @@ use std::str::FromStr;
 use std::time::Duration;
 use tracing::info;
 
-pub const MIGRATOR: Migrator = sqlx::migrate!("./migrations");
+// Embed migrations from the workspace-level migrations directory.
+// crates/db is two levels below backend-rust where migrations/ resides.
+pub const MIGRATOR: Migrator = sqlx::migrate!("../../migrations");
 
 pub async fn initialize_db(app_settings: &AppSettings) -> Result<Pool<Sqlite>> {
-    app_settings.ensure_default_directory()?;
+    app_settings
+        .ensure_default_directory()
+        .context("Failed to ensure default directory for database")?;
 
     let options = SqliteConnectOptions::from_str(&app_settings.database_url())?
         .create_if_missing(true)
@@ -25,7 +29,10 @@ pub async fn initialize_db(app_settings: &AppSettings) -> Result<Pool<Sqlite>> {
         .connect_with(options)
         .await?;
 
-    MIGRATOR.run(&pool).await.with_context(|| "Database migrations failed")?;
+    MIGRATOR
+        .run(&pool)
+        .await
+        .with_context(|| "Database migrations failed")?;
     info!("Database migrations completed successfully");
 
     Ok(pool)
@@ -33,6 +40,5 @@ pub async fn initialize_db(app_settings: &AppSettings) -> Result<Pool<Sqlite>> {
 
 pub async fn create_pool(opts: SqliteConnectOptions) -> Result<SqlitePool> {
     let pool = SqlitePool::connect_with(opts).await?;
-
     Ok(pool)
 }

backend-rust/crates/module-api/Cargo.toml

@@ -1,12 +0,0 @@
-[package]
-name = "owly-news-module-api"
-version.workspace = true
-edition.workspace = true
-
-[lib]
-path = "src/lib.rs"
-
-[dependencies]
-anyhow = { workspace = true }
-serde = { workspace = true, features = ["derive"] }
-serde_json = { workspace = true }

backend-rust/crates/module-api/src/lib.rs

@@ -1,30 +0,0 @@
-use std::ffi::{CStr, CString};
-use std::os::raw::c_char;
-
-// Symbols every module must export with `extern "C"` and `#[no_mangle]`.
-// Signature: fn module_name() -> *const c_char
-// Signature: fn module_invoke(op: *const c_char, payload: *const c_char) -> *mut c_char
-pub const SYMBOL_NAME: &str = "module_name";
-pub const SYMBOL_INVOKE: &str = "module_invoke";
-
-// Helper to convert C char* to &str
-pub unsafe fn cstr_to_str<'a>(ptr: *const c_char) -> anyhow::Result<&'a str> {
-    if ptr.is_null() {
-        anyhow::bail!("null pointer");
-    }
-    Ok(CStr::from_ptr(ptr).to_str()?)
-}
-
-// Helper to allocate a CString for return across FFI boundary (module side)
-pub fn string_to_cstring_ptr(s: String) -> *mut c_char {
-    CString::new(s).unwrap().into_raw()
-}
-
-// Helper to take back ownership of a CString (host side), then free by letting CString drop
-pub unsafe fn take_cstring(ptr: *mut c_char) -> anyhow::Result<String> {
-    if ptr.is_null() {
-        anyhow::bail!("null pointer");
-    }
-    let s = CString::from_raw(ptr);
-    Ok(s.into_string()?)
-}

backend-rust/crates/module-host/Cargo.toml

@@ -1,17 +0,0 @@
-[package]
-name = "owly-news-module-host"
-version.workspace = true
-edition.workspace = true
-
-[lib]
-path = "src/lib.rs"
-
-[dependencies]
-anyhow = { workspace = true }
-libloading = { workspace = true }
-once_cell = { workspace = true }
-serde = { workspace = true, features = ["derive"] }
-serde_json = { workspace = true }
-tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync"] }
-tracing = { workspace = true }
-owly-news-module-api = { path = "../module-api" }

backend-rust/crates/module-host/src/lib.rs

@@ -1,114 +0,0 @@
-use anyhow::Context;
-use libloading::{Library, Symbol};
-use once_cell::sync::OnceCell;
-use owly_news_module_api::{take_cstring, SYMBOL_INVOKE, SYMBOL_NAME};
-use std::collections::HashMap;
-use std::ffi::CString;
-use std::os::raw::c_char;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use tokio::sync::Mutex;
-use tracing::info;
-
-type ModuleNameFn = unsafe extern "C" fn() -> *const c_char;
-type ModuleInvokeFn = unsafe extern "C" fn(*const c_char, *const c_char) -> *mut c_char;
-
-pub struct ModuleHandle {
-    _lib: Arc<Library>,
-    invoke: ModuleInvokeFn,
-}
-
-impl ModuleHandle {
-    pub fn invoke_json(&self, op: &str, payload: serde_json::Value) -> anyhow::Result<serde_json::Value> {
-        let op_c = CString::new(op)?;
-        let payload_c = CString::new(serde_json::to_string(&payload)?)?;
-
-        let out_ptr = unsafe { (self.invoke)(op_c.as_ptr(), payload_c.as_ptr()) };
-        let out = unsafe { take_cstring(out_ptr) }?;
-        let val = serde_json::from_str(&out).context("module returned invalid JSON")?;
-        Ok(val)
-    }
-}
-
-pub struct ModuleHost {
-    // Lazy cache of loaded modules by logical name
-    loaded: Mutex<HashMap<String, Arc<ModuleHandle>>>,
-    modules_dir: PathBuf,
-}
-
-static DEFAULT_HOST: OnceCell<Arc<ModuleHost>> = OnceCell::new();
-
-impl ModuleHost {
-    pub fn default() -> Arc<Self> {
-        DEFAULT_HOST
-            .get_or_init(|| {
-                Arc::new(Self::new(
-                    std::env::var_os("OWLY_MODULES_DIR")
-                        .map(PathBuf::from)
-                        .unwrap_or_else(|| PathBuf::from("target/modules")), // default location
-                ))
-            })
-            .clone()
-    }
-
-    pub fn new(modules_dir: PathBuf) -> Self {
-        Self {
-            loaded: Mutex::new(HashMap::new()),
-            modules_dir,
-        }
-    }
-
-    pub async fn get(&self, name: &str) -> anyhow::Result<Arc<ModuleHandle>> {
-        if let Some(h) = self.loaded.lock().await.get(name).cloned() {
-            return Ok(h);
-        }
-        let handle = Arc::new(self.load_module(name)?);
-        self.loaded.lock().await.insert(name.to_string(), handle.clone());
-        Ok(handle)
-    }
-
-    fn load_module(&self, name: &str) -> anyhow::Result<ModuleHandle> {
-        let lib_path = resolve_module_path(&self.modules_dir, name)?;
-        info!(module = name, path = %lib_path.display(), "loading module");
-
-        // SAFETY: we keep Library alive in ModuleHandle to ensure symbols remain valid
-        let lib = unsafe { Library::new(lib_path) }.with_context(|| "failed to load module library")?;
-
-        // Validate and bind symbols
-        let name_fn: Symbol<ModuleNameFn> = unsafe { lib.get(SYMBOL_NAME.as_bytes()) }
-            .with_context(|| "missing symbol `module_name`")?;
-        let invoke_fn: Symbol<ModuleInvokeFn> = unsafe { lib.get(SYMBOL_INVOKE.as_bytes()) }
-            .with_context(|| "missing symbol `module_invoke`")?;
-
-        // Optional: verify reported name matches requested
-        let c_name_ptr = unsafe { name_fn() };
-        let c_name = unsafe { std::ffi::CStr::from_ptr(c_name_ptr) }.to_string_lossy().into_owned();
-        if c_name != name {
-            anyhow::bail!("module reported name `{c_name}`, expected `{name}`");
-        }
-
-        // Copy the function pointer before moving the library
-        let invoke_fn_copy = *invoke_fn;
-
-        Ok(ModuleHandle {
-            _lib: Arc::new(lib),
-            invoke: invoke_fn_copy,
-        })
-    }
-}
-
-fn resolve_module_path(dir: &Path, name: &str) -> anyhow::Result<PathBuf> {
-    #[cfg(target_os = "windows")]
-    const EXT: &str = "dll";
-    #[cfg(target_os = "macos")]
-    const EXT: &str = "dylib";
-    #[cfg(all(unix, not(target_os = "macos")))]
-    const EXT: &str = "so";
-
-    let fname = format!("lib{name}.{EXT}");
-    let path = dir.join(fname);
-    if !path.exists() {
-        anyhow::bail!("module `{name}` not found at {}", path.display());
-    }
-    Ok(path)
-}

backend-rust/crates/modules/summarizer/Cargo.toml

@@ -1,14 +0,0 @@
-[package]
-name = "owly-news-module-summarizer"
-version.workspace = true
-edition.workspace = true
-
-[lib]
-crate-type = ["cdylib"]
-path = "src/lib.rs"
-
-[dependencies]
-anyhow = { workspace = true }
-serde = { workspace = true, features = ["derive"] }
-serde_json = { workspace = true }
-owly-news-module-api = { path = "../../module-api" }

backend-rust/crates/modules/summarizer/src/lib.rs

@@ -1,50 +0,0 @@
-use owly_news_module_api::{cstr_to_str, string_to_cstring_ptr};
-use serde::{Deserialize, Serialize};
-use std::os::raw::c_char;
-
-#[derive(Deserialize)]
-struct SummarizeReq {
-    text: String,
-    #[serde(default = "default_ratio")]
-    ratio: f32,
-}
-
-fn default_ratio() -> f32 { 0.2 }
-
-#[derive(Serialize)]
-struct SummarizeResp {
-    summary: String,
-}
-
-#[unsafe(no_mangle)]
-pub extern "C" fn module_name() -> *const c_char {
-    // IMPORTANT: string must live forever; use a const C string
-    static NAME: &str = "summarizer\0";
-    NAME.as_ptr() as *const c_char
-}
-
-#[unsafe(no_mangle)]
-pub extern "C" fn module_invoke(op: *const c_char, payload: *const c_char) -> *mut c_char {
-    // SAFETY: called by trusted host with valid pointers
-    let res = (|| -> anyhow::Result<String> {
-        let op = unsafe { cstr_to_str(op)? };
-        let payload = unsafe { cstr_to_str(payload)? };
-
-        match op {
-            "summarize" => {
-                let req: SummarizeReq = serde_json::from_str(payload)?;
-                // Placeholder summarization logic. Replace with real algorithm.
-                let words: Vec<&str> = req.text.split_whitespace().collect();
-                let take = ((words.len() as f32) * req.ratio).max(1.0).round() as usize;
-                let summary = words.into_iter().take(take).collect::<Vec<_>>().join(" ");
-                let resp = SummarizeResp { summary };
-                Ok(serde_json::to_string(&resp)?)
-            }
-            _ => anyhow::bail!("unknown op: {op}"),
-        }
-    })();
-
-    let json = res.unwrap_or_else(|e| serde_json::json!({ "error": e.to_string() }).to_string());
-
-    string_to_cstring_ptr(json)
-}

backend-rust/crates/server/Cargo.toml

@@ -11,11 +11,12 @@ tracing-subscriber = { workspace = true }
 axum = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-sqlx = { workspace = true }
+sqlx = { workspace = true, features = ["sqlite"] }
 dotenv = { workspace = true }
 once_cell = { workspace = true }
 
 api = { path = "../api" }
+db = { path = "../db" }
 http = "1.3.1"
 
 [features]

backend-rust/crates/server/src/lib.rs

@@ -6,6 +6,7 @@ use tracing_subscriber::EnvFilter;
 
 use api::services::{DefaultHealthService, HealthService};
 use api::types::Health;
+use api::config::AppSettings;
 
 pub struct AppState {
     pub health_service: Arc<dyn HealthService>,
@@ -29,14 +30,18 @@ async fn health_handler(state: Arc<AppState>) -> Json<Health> {
 pub async fn start_server(addr: SocketAddr) -> anyhow::Result<()> {
     init_tracing();
 
-    // TODO: initialize database pools and other infrastructure here.
+    // Load application settings and initialize the database pool (sqlite).
-    // let pool = sqlx::PgPool::connect(&db_url).await?;
+    let app_settings = AppSettings::get_app_settings();
+    let pool = db::initialize_db(&app_settings).await?;
 
     let state = Arc::new(AppState {
         health_service: Arc::new(DefaultHealthService),
     });
 
-    let app = build_router(state).await;
+    // Base daemon router
+    let app = build_router(state).await
+        // Attach API under /api and provide DB state
+        .nest("/api", api::api::routes::routes().with_state(pool.clone()));
 
     let listener = TcpListener::bind(addr).await?;
     info!("HTTP server listening on http://{}", addr);

backend-rust/src/main.rs

@@ -1,104 +0,0 @@
-mod config;
-mod db;
-mod models;
-mod services;
-
-use crate::config::{AppSettings, ConfigFile};
-use anyhow::Result;
-use axum::Router;
-use axum::routing::get;
-use tokio::signal;
-use tracing::info;
-use tracing_subscriber;
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    init_logging();
-
-    info!("Starting server");
-
-    let app_settings = load_app_settings();
-
-    let pool = db::initialize_db(&app_settings).await?;
-    let app = create_app(pool);
-
-    let listener = tokio::net::TcpListener::bind(format!(
-        "{}:{}",
-        &app_settings.config.server.host, &app_settings.config.server.port
-    ))
-    .await?;
-    info!(
-        "Server starting on http://{}:{}",
-        &app_settings.config.server.host, &app_settings.config.server.port
-    );
-
-    axum::serve(listener, app)
-        .with_graceful_shutdown(shutdown_signal())
-        .await?;
-    info!("Server stopped");
-
-    Ok(())
-}
-
-fn create_app(pool: sqlx::SqlitePool) -> Router {
-    Router::new()
-        .route("/health", get(health_check))
-        .nest("/api", api::routes::routes())
-        .with_state(pool)
-}
-
-async fn health_check() -> &'static str {
-    "OK"
-}
-
-fn init_logging() {
-    tracing_subscriber::fmt()
-        .with_target(false)
-        .compact()
-        // .with_env_filter(EnvFilter::from_default_env())
-        // .json() // For Production
-        .init();
-}
-
-fn load_app_settings() -> AppSettings {
-    AppSettings::default();
-    let app_settings = AppSettings::get_app_settings();
-
-    AppSettings::ensure_default_directory(&app_settings)
-        .expect("Failed to create default directory");
-
-    let config = ConfigFile::load_from_file(&AppSettings::get_app_settings())
-        .expect("Failed to load config file");
-
-    let app_settings = AppSettings {
-        config,
-        ..app_settings
-    };
-    app_settings
-}
-
-async fn shutdown_signal() {
-    let ctrl_c = async {
-        signal::ctrl_c()
-            .await
-            .expect("failed to install CTRL+C handler");
-    };
-
-    #[cfg(unix)]
-    let terminate = async {
-        signal::unix::signal(signal::unix::SignalKind::terminate())
-            .expect("failed to install terminate handler")
-            .recv()
-            .await;
-    };
-
-    #[cfg(not(unix))]
-    let terminate = std::future::pending::<()>();
-
-    tokio::select! {
-        _ = ctrl_c => {},
-        _ = terminate => {},
-    }
-
-    info!("Signal received, shutting down");
-}

backend-rust/src/models.rs

@@ -1,6 +0,0 @@
-mod article;
-mod summary;
-mod user;
-mod tag;
-mod analytics;
-mod settings;

backend-rust/src/services.rs

@@ -1,7 +0,0 @@
-mod summary_service;
-mod news_service;
-mod scraping_service;
-mod tagging_service;
-mod analytics_service;
-mod sharing_service;
-pub(crate) mod content_processor;