diff --git a/backend/app/config.py b/backend/app/config.py index c79a5d9..e23dee1 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -8,11 +8,11 @@ MIN_CRON_HOURS = float(os.getenv("MIN_CRON_HOURS", 0.5)) DEFAULT_CRON_HOURS = float(os.getenv("CRON_HOURS", MIN_CRON_HOURS)) CRON_HOURS = max(MIN_CRON_HOURS, DEFAULT_CRON_HOURS) SYNC_COOLDOWN_MINUTES = int(os.getenv("SYNC_COOLDOWN_MINUTES", 30)) -LLM_MODEL = os.getenv("LLM_MODEL", "mistral-nemo:12b") +LLM_MODEL = os.getenv("LLM_MODEL", "phi3:3.8b-mini-128k-instruct-q4_0") LLM_TIMEOUT_SECONDS = int(os.getenv("LLM_TIMEOUT_SECONDS", 180)) OLLAMA_API_TIMEOUT_SECONDS = int(os.getenv("OLLAMA_API_TIMEOUT_SECONDS", 10)) ARTICLE_FETCH_TIMEOUT = int(os.getenv("ARTICLE_FETCH_TIMEOUT", 30)) -MAX_ARTICLE_LENGTH = int(os.getenv("MAX_ARTICLE_LENGTH", 10_000)) +MAX_ARTICLE_LENGTH = int(os.getenv("MAX_ARTICLE_LENGTH", 40_000)) frontend_path = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(__file__))), diff --git a/backend/app/services.py b/backend/app/services.py index 7fa75e5..1df8c95 100644 --- a/backend/app/services.py +++ b/backend/app/services.py @@ -209,7 +209,7 @@ class NewsFetcher: "format": "json", "options": { "num_gpu": 1, # Force GPU usage - "num_ctx": 128_000, # Context size + "num_ctx": 64_000, # Context size } } diff --git a/backend/example.env b/backend/example.env index 4b9fe24..de702c3 100644 --- a/backend/example.env +++ b/backend/example.env @@ -11,10 +11,11 @@ MIN_CRON_HOURS=0.5 SYNC_COOLDOWN_MINUTES=30 # LLM model to use for summarization -LLM_MODEL=qwen2:7b-instruct-q4_K_M -LLM_MODEL=phi3:3.8b-mini-128k-instruct-q4_0 -LLM_MODEL=mistral-nemo:12b -LLM_MODEL=cnjack/mistral-samll-3.1:24b-it-q4_K_S +LLM_MODEL=qwen2:7b-instruct-q4_K_M # ca 7-9GB (typisch 8GB) +LLM_MODEL=phi3:3.8b-mini-128k-instruct-q4_0 # ca 6-8GB (langer kontext) +LLM_MODEL=mistral-nemo:12b # ca 16-24+GB +LLM_MODEL=cnjack/mistral-samll-3.1:24b-it-q4_K_S # ca 22GB +LLM_MODEL=yarn-mistral:7b-64k-q4_K_M # ca 11GB # Timeout in seconds for LLM requests LLM_TIMEOUT_SECONDS=180 diff --git a/frontend/.yarn/install-state.gz b/frontend/.yarn/install-state.gz index a23fb69..b48a5f4 100644 Binary files a/frontend/.yarn/install-state.gz and b/frontend/.yarn/install-state.gz differ diff --git a/frontend/cypress.config.ts b/frontend/cypress.config.ts deleted file mode 100644 index 4a22885..0000000 --- a/frontend/cypress.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { defineConfig } from 'cypress' - -export default defineConfig({ - e2e: { - specPattern: 'cypress/e2e/**/*.{cy,spec}.{js,jsx,ts,tsx}', - baseUrl: 'http://localhost:4173', - }, -}) diff --git a/frontend/src/components/NewsList.vue b/frontend/src/components/NewsList.vue index 1f043db..1f04f6e 100644 --- a/frontend/src/components/NewsList.vue +++ b/frontend/src/components/NewsList.vue @@ -14,75 +14,77 @@
-
- -
-
+
@@ -100,9 +102,9 @@
@@ -129,17 +131,48 @@ const loadMoreArticles = async () => { } }; +interface Article { + id: number; + title: string; + summary: string; + url: string; + published: number; + country: string; + created_at: number; +} + +const INVALID_MARKERS = ['---', '...', '…', 'Title', 'Summary', 'Titel', 'Zusammenfassung'] as const; +const REQUIRED_TEXT_FIELDS = ['title', 'summary', 'url'] as const; + +const isValidArticleContent = (article: Article): boolean => { + const hasEmptyRequiredFields = REQUIRED_TEXT_FIELDS.some( + field => article[field as keyof Pick].length === 0 + ); + + if (hasEmptyRequiredFields) { + return false; + } + + const hasInvalidMarkers = REQUIRED_TEXT_FIELDS.some(field => + INVALID_MARKERS.some(marker => + article[field as keyof Pick].includes(marker) + ) + ); + + return !hasInvalidMarkers; +}; + const observer = ref(null); const loadMoreTrigger = ref(null); onMounted(() => { observer.value = new IntersectionObserver( - (entries) => { - if (entries[0].isIntersecting) { - loadMoreArticles(); - } - }, - {threshold: 0.5} + (entries) => { + if (entries[0].isIntersecting) { + loadMoreArticles(); + } + }, + {threshold: 0.5} ); if (loadMoreTrigger.value) { diff --git a/frontend/yarn.lock b/frontend/yarn.lock index b8d1a3b..5829eda 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -6470,9 +6470,9 @@ __metadata: languageName: node linkType: hard -"owly-news-summariser@workspace:.": +"owly-news@workspace:.": version: 0.0.0-use.local - resolution: "owly-news-summariser@workspace:." + resolution: "owly-news@workspace:." dependencies: "@tailwindcss/vite": "npm:^4.1.11" "@tsconfig/node22": "npm:^22.0.2"