From 98c9328a3db53ef892b338392f35c6e649cb8a97 Mon Sep 17 00:00:00 2001 From: dimitar Date: Thu, 10 Apr 2025 00:27:55 +0200 Subject: [PATCH] index added to db, pagination working --- README.md | 24 ++++ architecture.md | 84 +++++++++++ .../20250409214100_add_indexes/migration.sql | 17 +++ backend/prisma/migrations/migration_lock.toml | 2 +- backend/prisma/schema.prisma | 6 + backend/src/articles/articles.controller.ts | 17 ++- backend/src/articles/articles.service.ts | 46 +++++- backend/src/articles/types.ts | 4 + frontend/src/api/articles.ts | 29 +++- frontend/src/routeTree.gen.ts | 51 ++++++- frontend/src/routes/articles.tsx | 134 ++++++++++++------ frontend/src/routes/dashboard/index.tsx | 61 ++++++++ frontend/src/routes/index.tsx | 30 ++-- 13 files changed, 428 insertions(+), 77 deletions(-) create mode 100644 architecture.md create mode 100644 backend/prisma/migrations/20250409214100_add_indexes/migration.sql create mode 100644 backend/src/articles/types.ts create mode 100644 frontend/src/routes/dashboard/index.tsx diff --git a/README.md b/README.md index e69de29..bfeec8c 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,24 @@ +# News writing solution implementation complete. The application consists of + +1. Backend: NestJS API running on port 3000 with: + - Articles endpoint at /api/articles + - PostgreSQL database connection + - Prisma ORM integration + +2. Frontend: React application with: + - TanStack Router for navigation + - TanStack Query for data fetching + - Basic articles listing page + +To access the application: + +1. Backend API: +2. Frontend: (or the port shown in terminal) +3. Articles page: + +Next steps could include: + +- Adding authentication +- Implementing article writing/editing features +- Enhancing search functionality +- Adding social sharing integration diff --git a/architecture.md b/architecture.md new file mode 100644 index 0000000..b342d77 --- /dev/null +++ b/architecture.md @@ -0,0 +1,84 @@ +# News Writing Solution Architecture (Revised) + +## System Overview + +```mermaid +graph TD + A[Data Collection] --> B[Database] + B --> C[NestJS API] + C --> D[React Frontend] + D --> E[User Browsers] + C --> F[Social Media APIs] +``` + +## Tech Stack + +### Backend Services + +- **Framework**: NestJS (Node.js) +- **Database**: PostgreSQL + Prisma ORM +- **Search**: ElasticSearch +- **Queue**: BullMQ +- **Caching**: Redis + +### Frontend Application + +- **Core**: React 18 + Vite +- **State**: TanStack Store +- **Routing**: TanStack Router +- **Data**: TanStack Query +- **Styling**: TailwindCSS + shadcn + +## Key Components + +### Data Collection Layer + +- RSS feed processor (rss-parser) +- Web scraping (Puppeteer) +- Data normalization pipeline +- Scheduled jobs (BullMQ) + +### API Services + +- Articles module +- Search module +- User authentication +- Social integration +- Rate limiting + +### Frontend Features + +- Article browsing interface +- Advanced search with filters +- WYSIWYG editor (Tiptap) +- Social sharing +- User profiles + +## Implementation Phases + +1. **Core Infrastructure** + - VPS setup with Docker + - Database deployment + - CI/CD pipeline + +2. **API Development** + - NestJS modules + - Prisma schema + - Authentication + +3. **Frontend Implementation** + - Core layout + - Article components + - Search interface + +4. **Data Pipeline** + - RSS integration + - Scraping services + - Queue system + +## Scalability Considerations + +- Database read replicas +- Horizontal API scaling +- Caching strategy +- Monitoring (Prometheus + Grafana) diff --git a/backend/prisma/migrations/20250409214100_add_indexes/migration.sql b/backend/prisma/migrations/20250409214100_add_indexes/migration.sql new file mode 100644 index 0000000..8d11749 --- /dev/null +++ b/backend/prisma/migrations/20250409214100_add_indexes/migration.sql @@ -0,0 +1,17 @@ +-- DropForeignKey +ALTER TABLE "Note" DROP CONSTRAINT "Note_userId_fkey"; + +-- AlterTable +ALTER TABLE "Note" ALTER COLUMN "userId" DROP NOT NULL; + +-- CreateIndex +CREATE INDEX "Article_url_idx" ON "Article"("url"); + +-- CreateIndex +CREATE INDEX "Collection_userId_idx" ON "Collection"("userId"); + +-- CreateIndex +CREATE INDEX "QuickNote_collectionId_idx" ON "QuickNote"("collectionId"); + +-- AddForeignKey +ALTER TABLE "Note" ADD CONSTRAINT "Note_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/backend/prisma/migrations/migration_lock.toml b/backend/prisma/migrations/migration_lock.toml index 648c57f..044d57c 100644 --- a/backend/prisma/migrations/migration_lock.toml +++ b/backend/prisma/migrations/migration_lock.toml @@ -1,3 +1,3 @@ # Please do not edit this file manually # It should be added in your version-control system (e.g., Git) -provider = "postgresql" \ No newline at end of file +provider = "postgresql" diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index 0b523ae..a67e0ba 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -18,6 +18,8 @@ model Collection { notes Note[] userId String user User @relation(fields: [userId], references: [id]) + + @@index([userId]) } model Article { @@ -34,6 +36,8 @@ model Article { categories String[] authors String[] Note Note[] + + @@index([url]) } model QuickNote { @@ -50,6 +54,8 @@ model QuickNote { articleId String? createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + + @@index([collectionId]) } model Note { diff --git a/backend/src/articles/articles.controller.ts b/backend/src/articles/articles.controller.ts index 93e92b5..0bde02d 100644 --- a/backend/src/articles/articles.controller.ts +++ b/backend/src/articles/articles.controller.ts @@ -1,4 +1,4 @@ -import { Controller, Get } from '@nestjs/common'; +import { Controller, Get, Query } from '@nestjs/common'; import { ArticlesService } from './articles.service'; @Controller('api/articles') @@ -6,7 +6,18 @@ export class ArticlesController { constructor(private readonly articlesService: ArticlesService) {} @Get() - async findAll() { - return this.articlesService.findAll(); + async findAll( + @Query('limit') limit?: number, + @Query('cursor') cursor?: string, + ) { + return this.articlesService.findAll( + limit ? Number(limit) : undefined, + cursor, + ); + } + + @Get('count') + async getCount() { + return { count: await this.articlesService.count() }; } } diff --git a/backend/src/articles/articles.service.ts b/backend/src/articles/articles.service.ts index 879bc5f..e48a9f7 100644 --- a/backend/src/articles/articles.service.ts +++ b/backend/src/articles/articles.service.ts @@ -1,14 +1,52 @@ import { Injectable } from '@nestjs/common'; import { PrismaService } from '../prisma/prisma.service'; +import { PaginatedArticles } from './types'; @Injectable() export class ArticlesService { constructor(private prisma: PrismaService) {} - async findAll() { - return this.prisma.article.findMany({ - orderBy: { publishedAt: 'desc' }, - take: 100, + async findAll( + limit: number = 20, + cursor?: string, + ): Promise { + const articles = await this.prisma.article.findMany({ + take: limit + 1, + skip: cursor ? 1 : 0, + cursor: cursor + ? { + id: cursor, + } + : undefined, + orderBy: [ + { publishedAt: 'desc' }, + { id: 'desc' }, // Secondary sort for stability + ], + select: { + id: true, + title: true, + content: true, + source: true, + url: true, + publishedAt: true, + categories: true, + authors: true, + }, }); + + let nextCursor: string | null = null; + if (articles.length > limit) { + const nextItem = articles.pop()!; + nextCursor = nextItem.id; + } + + return { + articles, + nextCursor, + }; + } + + async count(): Promise { + return this.prisma.article.count(); } } diff --git a/backend/src/articles/types.ts b/backend/src/articles/types.ts new file mode 100644 index 0000000..af737ee --- /dev/null +++ b/backend/src/articles/types.ts @@ -0,0 +1,4 @@ +export interface PaginatedArticles { + articles: any[]; + nextCursor: string | null; +} diff --git a/frontend/src/api/articles.ts b/frontend/src/api/articles.ts index 370dd6f..7b152c0 100644 --- a/frontend/src/api/articles.ts +++ b/frontend/src/api/articles.ts @@ -1,11 +1,20 @@ import { Article } from '@/types' -export async function fetchArticles(): Promise { - const response = await fetch('/api/articles') +export interface PaginatedArticles { + articles: Article[]; + nextCursor: string | null; +} + +export async function fetchArticles(cursor?: string, limit: number = 20): Promise { + const params = new URLSearchParams(); + if (cursor) params.append('cursor', cursor); + if (limit) params.append('limit', limit.toString()); + + const response = await fetch(`/api/articles?${params.toString()}`); if (!response.ok) { - throw new Error('Failed to fetch articles') + throw new Error('Failed to fetch articles'); } - return response.json() + return response.json(); } export async function fetchAllArticles(): Promise { @@ -13,5 +22,15 @@ export async function fetchAllArticles(): Promise { if (!response.ok) { throw new Error('Failed to fetch articles'); } - return response.json(); + const data: PaginatedArticles = await response.json(); + return data.articles; +} + +export async function getArticleCount(): Promise { + const response = await fetch('/api/articles/count'); + if (!response.ok) { + throw new Error('Failed to fetch article count'); + } + const data = await response.json(); + return data.count; } \ No newline at end of file diff --git a/frontend/src/routeTree.gen.ts b/frontend/src/routeTree.gen.ts index e59c76c..7409686 100644 --- a/frontend/src/routeTree.gen.ts +++ b/frontend/src/routeTree.gen.ts @@ -18,6 +18,7 @@ import { Route as DashboardImport } from './routes/dashboard' import { Route as CollectionsImport } from './routes/collections' import { Route as ArticlesImport } from './routes/articles' import { Route as IndexImport } from './routes/index' +import { Route as DashboardIndexImport } from './routes/dashboard/index' import { Route as CollectionsnameImport } from './routes/collections/[name]' // Create/Update Routes @@ -64,6 +65,12 @@ const IndexRoute = IndexImport.update({ getParentRoute: () => rootRoute, } as any) +const DashboardIndexRoute = DashboardIndexImport.update({ + id: '/', + path: '/', + getParentRoute: () => DashboardRoute, +} as any) + const CollectionsnameRoute = CollectionsnameImport.update({ id: '/[name]', path: '/[name]', @@ -130,6 +137,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof CollectionsnameImport parentRoute: typeof CollectionsImport } + '/dashboard/': { + id: '/dashboard/' + path: '/' + fullPath: '/dashboard/' + preLoaderRoute: typeof DashboardIndexImport + parentRoute: typeof DashboardImport + } } } @@ -147,26 +161,39 @@ const CollectionsRouteWithChildren = CollectionsRoute._addFileChildren( CollectionsRouteChildren, ) +interface DashboardRouteChildren { + DashboardIndexRoute: typeof DashboardIndexRoute +} + +const DashboardRouteChildren: DashboardRouteChildren = { + DashboardIndexRoute: DashboardIndexRoute, +} + +const DashboardRouteWithChildren = DashboardRoute._addFileChildren( + DashboardRouteChildren, +) + export interface FileRoutesByFullPath { '/': typeof IndexRoute '/articles': typeof ArticlesRoute '/collections': typeof CollectionsRouteWithChildren - '/dashboard': typeof DashboardRoute + '/dashboard': typeof DashboardRouteWithChildren '/login': typeof LoginRoute '/notFound': typeof NotFoundRoute '/register': typeof RegisterRoute '/collections/[name]': typeof CollectionsnameRoute + '/dashboard/': typeof DashboardIndexRoute } export interface FileRoutesByTo { '/': typeof IndexRoute '/articles': typeof ArticlesRoute '/collections': typeof CollectionsRouteWithChildren - '/dashboard': typeof DashboardRoute '/login': typeof LoginRoute '/notFound': typeof NotFoundRoute '/register': typeof RegisterRoute '/collections/[name]': typeof CollectionsnameRoute + '/dashboard': typeof DashboardIndexRoute } export interface FileRoutesById { @@ -174,11 +201,12 @@ export interface FileRoutesById { '/': typeof IndexRoute '/articles': typeof ArticlesRoute '/collections': typeof CollectionsRouteWithChildren - '/dashboard': typeof DashboardRoute + '/dashboard': typeof DashboardRouteWithChildren '/login': typeof LoginRoute '/notFound': typeof NotFoundRoute '/register': typeof RegisterRoute '/collections/[name]': typeof CollectionsnameRoute + '/dashboard/': typeof DashboardIndexRoute } export interface FileRouteTypes { @@ -192,16 +220,17 @@ export interface FileRouteTypes { | '/notFound' | '/register' | '/collections/[name]' + | '/dashboard/' fileRoutesByTo: FileRoutesByTo to: | '/' | '/articles' | '/collections' - | '/dashboard' | '/login' | '/notFound' | '/register' | '/collections/[name]' + | '/dashboard' id: | '__root__' | '/' @@ -212,6 +241,7 @@ export interface FileRouteTypes { | '/notFound' | '/register' | '/collections/[name]' + | '/dashboard/' fileRoutesById: FileRoutesById } @@ -219,7 +249,7 @@ export interface RootRouteChildren { IndexRoute: typeof IndexRoute ArticlesRoute: typeof ArticlesRoute CollectionsRoute: typeof CollectionsRouteWithChildren - DashboardRoute: typeof DashboardRoute + DashboardRoute: typeof DashboardRouteWithChildren LoginRoute: typeof LoginRoute NotFoundRoute: typeof NotFoundRoute RegisterRoute: typeof RegisterRoute @@ -229,7 +259,7 @@ const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, ArticlesRoute: ArticlesRoute, CollectionsRoute: CollectionsRouteWithChildren, - DashboardRoute: DashboardRoute, + DashboardRoute: DashboardRouteWithChildren, LoginRoute: LoginRoute, NotFoundRoute: NotFoundRoute, RegisterRoute: RegisterRoute, @@ -267,7 +297,10 @@ export const routeTree = rootRoute ] }, "/dashboard": { - "filePath": "dashboard.tsx" + "filePath": "dashboard.tsx", + "children": [ + "/dashboard/" + ] }, "/login": { "filePath": "login.tsx" @@ -281,6 +314,10 @@ export const routeTree = rootRoute "/collections/[name]": { "filePath": "collections/[name].tsx", "parent": "/collections" + }, + "/dashboard/": { + "filePath": "dashboard/index.tsx", + "parent": "/dashboard" } } } diff --git a/frontend/src/routes/articles.tsx b/frontend/src/routes/articles.tsx index 8f0e4b9..78c8bd6 100644 --- a/frontend/src/routes/articles.tsx +++ b/frontend/src/routes/articles.tsx @@ -1,11 +1,12 @@ import { createFileRoute } from '@tanstack/react-router' -import { useQuery } from '@tanstack/react-query' +import { useInfiniteQuery } from '@tanstack/react-query' import { fetchArticles } from '@/api/articles' import { Button } from '@/components/ui/button' import { addArticleToCollection, createCollection, fetchCollections } from '@/api/collections'; import { useUser } from '@/store/user'; import { useNavigate } from '@tanstack/react-router'; import { Article } from '@/types'; +import { useEffect, useRef, useCallback } from 'react'; export const Route = createFileRoute('/articles')({ component: ArticlesPage @@ -52,29 +53,62 @@ async function handleAddToCollection( } function ArticlesPage() { - const { data: articles, isLoading, isError, error } = useQuery({ + const { + data, + isLoading, + isError, + error, + fetchNextPage, + hasNextPage, + isFetchingNextPage + } = useInfiniteQuery({ queryKey: ['articles'], - queryFn: async () => { - try { - const response = await fetchArticles(); - console.log('Articles API response:', response); - return response; - } catch (err) { - console.error('Error fetching articles:', err); - throw err; - } + queryFn: async ({ pageParam = undefined }) => { + const response = await fetchArticles(pageParam); + return response; }, + getNextPageParam: (lastPage) => lastPage.nextCursor, + initialPageParam: undefined as string | undefined, staleTime: 1000 * 60 // Cache for 1 minute }); + const user = useUser(); const navigate = useNavigate(); + const loadMoreRef = useRef(null); - if (isLoading) { + const handleObserver = useCallback((entries: IntersectionObserverEntry[]) => { + const [target] = entries; + if (target.isIntersecting && hasNextPage && !isFetchingNextPage) { + void fetchNextPage(); + } + }, [fetchNextPage, hasNextPage, isFetchingNextPage]); + + useEffect(() => { + const element = loadMoreRef.current; + const observer = new IntersectionObserver(handleObserver, { + root: null, + rootMargin: '0px', + threshold: 0.1, + }); + + if (element) { + observer.observe(element); + } + + return () => { + if (element) { + observer.unobserve(element); + } + }; + }, [handleObserver]); + + // Show initial loading state + if (isLoading && !data?.pages?.length) { return (
- ) + ); } if (isError) { @@ -86,9 +120,11 @@ function ArticlesPage() { Retry - ) + ); } + const allArticles = data?.pages.flatMap(page => page.articles) ?? []; + return (
@@ -96,35 +132,49 @@ function ArticlesPage() {

Browse the latest news from various sources

- {articles && articles.length > 0 ? ( -
- {articles.map(article => ( -
-
-

{article.title}

-

- {article.content?.substring(0, 120)}... -

-
- {article.source} - - {new Date(article.publishedAt).toLocaleDateString()} - + {allArticles.length > 0 ? ( + <> +
+ {allArticles.map(article => ( +
+
+

{article.title}

+

+ {article.content?.substring(0, 120)}... +

+
+ {article.source} + + {new Date(article.publishedAt).toLocaleDateString()} + +
+
-
-
- ))} -
+ ))} +
+ + {/* Loading more trigger and indicator */} +
+ {isFetchingNextPage ? ( +
+ ) : hasNextPage ? ( + Scroll to load more + ) : ( + No more articles to load + )} +
+ ) : (

No articles found

@@ -132,6 +182,6 @@ function ArticlesPage() {
)}
- ) + ); } diff --git a/frontend/src/routes/dashboard/index.tsx b/frontend/src/routes/dashboard/index.tsx new file mode 100644 index 0000000..71f54b8 --- /dev/null +++ b/frontend/src/routes/dashboard/index.tsx @@ -0,0 +1,61 @@ +import { createFileRoute } from '@tanstack/react-router'; +import { useQuery } from '@tanstack/react-query'; +import { getArticleCount } from '@/api/articles'; +import { NewspaperIcon, BookmarkIcon, UsersIcon } from 'lucide-react'; + +function DashboardIndex() { + const { data: articleCount = 0, isLoading } = useQuery({ + queryKey: ['articleCount'], + queryFn: getArticleCount, + }); + + return ( +
+

Dashboard

+ +
+
+
+
+ +
+
+

Total Articles

+

+ {isLoading ? '...' : articleCount} +

+
+
+
+ +
+
+
+ +
+
+

Collections

+

-

+
+
+
+ +
+
+
+ +
+
+

Active Users

+

-

+
+
+
+
+
+ ); +} + +export const Route = createFileRoute('/dashboard/')({ + component: DashboardIndex, +}); diff --git a/frontend/src/routes/index.tsx b/frontend/src/routes/index.tsx index bcd2143..3c7d7a4 100644 --- a/frontend/src/routes/index.tsx +++ b/frontend/src/routes/index.tsx @@ -1,8 +1,8 @@ import { createFileRoute } from '@tanstack/react-router' import { Button } from '@/components/ui/button' import { useRouter } from '@tanstack/react-router' -import { useQuery } from '@tanstack/react-query'; -import { fetchAllArticles } from '@/api/articles'; +// import { useQuery } from '@tanstack/react-query'; +// import { fetchAllArticles } from '@/api/articles'; export const Route = createFileRoute('/')({ component: IndexPage, @@ -15,18 +15,18 @@ function IndexPage() { router.navigate({ to: path }); }; - const { data: articles, isLoading, isError } = useQuery({ - queryKey: ['allArticles'], - queryFn: fetchAllArticles, - }); + // const { data: articles = [], isLoading, isError } = useQuery({ + // queryKey: ['allArticles'], + // queryFn: fetchAllArticles, + // }); - if (isLoading) { - return
Loading articles...
; - } + // if (isLoading) { + // return
Loading articles...
; + // } - if (isError) { - return
Failed to load articles. Please try again later.
; - } + // if (isError) { + // return
Failed to load articles. Please try again later.
; + // } return (
@@ -53,9 +53,9 @@ function IndexPage() {
-
+ {/*
- {articles?.map((article) => ( + {articles.map((article) => (

{article.title}

@@ -67,7 +67,7 @@ function IndexPage() {

))}
-
+
*/}
) }