index added to db, pagination working

README.md

@@ -0,0 +1,24 @@
+# News writing solution implementation complete. The application consists of
+
+1. Backend: NestJS API running on port 3000 with:
+   - Articles endpoint at /api/articles
+   - PostgreSQL database connection
+   - Prisma ORM integration
+
+2. Frontend: React application with:
+   - TanStack Router for navigation
+   - TanStack Query for data fetching
+   - Basic articles listing page
+
+To access the application:
+
+1. Backend API: <http://localhost:3000>
+2. Frontend: <http://localhost:5173> (or the port shown in terminal)
+3. Articles page: <http://localhost:5173/articles>
+
+Next steps could include:
+
+- Adding authentication
+- Implementing article writing/editing features
+- Enhancing search functionality
+- Adding social sharing integration

architecture.md

@@ -0,0 +1,84 @@
+# News Writing Solution Architecture (Revised)
+
+## System Overview
+
+```mermaid
+graph TD
+    A[Data Collection] --> B[Database]
+    B --> C[NestJS API]
+    C --> D[React Frontend]
+    D --> E[User Browsers]
+    C --> F[Social Media APIs]
+```
+
+## Tech Stack
+
+### Backend Services
+
+- **Framework**: NestJS (Node.js)
+- **Database**: PostgreSQL + Prisma ORM
+- **Search**: ElasticSearch
+- **Queue**: BullMQ
+- **Caching**: Redis
+
+### Frontend Application
+
+- **Core**: React 18 + Vite
+- **State**: TanStack Store
+- **Routing**: TanStack Router
+- **Data**: TanStack Query
+- **Styling**: TailwindCSS + shadcn
+
+## Key Components
+
+### Data Collection Layer
+
+- RSS feed processor (rss-parser)
+- Web scraping (Puppeteer)
+- Data normalization pipeline
+- Scheduled jobs (BullMQ)
+
+### API Services
+
+- Articles module
+- Search module
+- User authentication
+- Social integration
+- Rate limiting
+
+### Frontend Features
+
+- Article browsing interface
+- Advanced search with filters
+- WYSIWYG editor (Tiptap)
+- Social sharing
+- User profiles
+
+## Implementation Phases
+
+1. **Core Infrastructure**
+   - VPS setup with Docker
+   - Database deployment
+   - CI/CD pipeline
+
+2. **API Development**
+   - NestJS modules
+   - Prisma schema
+   - Authentication
+
+3. **Frontend Implementation**
+   - Core layout
+   - Article components
+   - Search interface
+
+4. **Data Pipeline**
+   - RSS integration
+   - Scraping services
+   - Queue system
+
+## Scalability Considerations
+
+- Database read replicas
+- Horizontal API scaling
+- Caching strategy
+- Monitoring (Prometheus + Grafana)

backend/prisma/migrations/20250409214100_add_indexes/migration.sql

@@ -0,0 +1,17 @@
+-- DropForeignKey
+ALTER TABLE "Note" DROP CONSTRAINT "Note_userId_fkey";
+
+-- AlterTable
+ALTER TABLE "Note" ALTER COLUMN "userId" DROP NOT NULL;
+
+-- CreateIndex
+CREATE INDEX "Article_url_idx" ON "Article"("url");
+
+-- CreateIndex
+CREATE INDEX "Collection_userId_idx" ON "Collection"("userId");
+
+-- CreateIndex
+CREATE INDEX "QuickNote_collectionId_idx" ON "QuickNote"("collectionId");
+
+-- AddForeignKey
+ALTER TABLE "Note" ADD CONSTRAINT "Note_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE SET NULL ON UPDATE CASCADE;

backend/prisma/schema.prisma

@@ -18,6 +18,8 @@ model Collection {
   notes       Note[]
   userId      String
   user        User        @relation(fields: [userId], references: [id])
+
+  @@index([userId])
 }
 
 model Article {
@@ -34,6 +36,8 @@ model Article {
   categories  String[]
   authors     String[]
   Note        Note[]
+
+  @@index([url])
 }
 
 model QuickNote {
@@ -50,6 +54,8 @@ model QuickNote {
   articleId    String?
   createdAt    DateTime   @default(now())
   updatedAt    DateTime   @updatedAt
+
+  @@index([collectionId])
 }
 
 model Note {

backend/src/articles/articles.controller.ts

@@ -1,4 +1,4 @@
-import { Controller, Get } from '@nestjs/common';
+import { Controller, Get, Query } from '@nestjs/common';
 import { ArticlesService } from './articles.service';
 
 @Controller('api/articles')
@@ -6,7 +6,18 @@ export class ArticlesController {
   constructor(private readonly articlesService: ArticlesService) {}
 
   @Get()
-  async findAll() {
-    return this.articlesService.findAll();
+  async findAll(
+    @Query('limit') limit?: number,
+    @Query('cursor') cursor?: string,
+  ) {
+    return this.articlesService.findAll(
+      limit ? Number(limit) : undefined,
+      cursor,
+    );
+  }
+
+  @Get('count')
+  async getCount() {
+    return { count: await this.articlesService.count() };
   }
 }

backend/src/articles/articles.service.ts

@@ -1,14 +1,52 @@
 import { Injectable } from '@nestjs/common';
 import { PrismaService } from '../prisma/prisma.service';
+import { PaginatedArticles } from './types';
 
 @Injectable()
 export class ArticlesService {
   constructor(private prisma: PrismaService) {}
 
-  async findAll() {
-    return this.prisma.article.findMany({
-      orderBy: { publishedAt: 'desc' },
-      take: 100,
+  async findAll(
+    limit: number = 20,
+    cursor?: string,
+  ): Promise<PaginatedArticles> {
+    const articles = await this.prisma.article.findMany({
+      take: limit + 1,
+      skip: cursor ? 1 : 0,
+      cursor: cursor
+        ? {
+            id: cursor,
+          }
+        : undefined,
+      orderBy: [
+        { publishedAt: 'desc' },
+        { id: 'desc' }, // Secondary sort for stability
+      ],
+      select: {
+        id: true,
+        title: true,
+        content: true,
+        source: true,
+        url: true,
+        publishedAt: true,
+        categories: true,
+        authors: true,
+      },
     });
+
+    let nextCursor: string | null = null;
+    if (articles.length > limit) {
+      const nextItem = articles.pop()!;
+      nextCursor = nextItem.id;
+    }
+
+    return {
+      articles,
+      nextCursor,
+    };
+  }
+
+  async count(): Promise<number> {
+    return this.prisma.article.count();
   }
 }

backend/src/articles/types.ts

@@ -0,0 +1,4 @@
+export interface PaginatedArticles {
+  articles: any[];
+  nextCursor: string | null;
+}

frontend/src/api/articles.ts

@@ -1,11 +1,20 @@
 import { Article } from '@/types'
 
-export async function fetchArticles(): Promise<Article[]> {
-  const response = await fetch('/api/articles')
-  if (!response.ok) {
-    throw new Error('Failed to fetch articles')
+export interface PaginatedArticles {
+  articles: Article[];
+  nextCursor: string | null;
 }
-  return response.json()
+
+export async function fetchArticles(cursor?: string, limit: number = 20): Promise<PaginatedArticles> {
+  const params = new URLSearchParams();
+  if (cursor) params.append('cursor', cursor);
+  if (limit) params.append('limit', limit.toString());
+
+  const response = await fetch(`/api/articles?${params.toString()}`);
+  if (!response.ok) {
+    throw new Error('Failed to fetch articles');
+  }
+  return response.json();
 }
 
 export async function fetchAllArticles(): Promise<Article[]> {
@@ -13,5 +22,15 @@ export async function fetchAllArticles(): Promise<Article[]> {
   if (!response.ok) {
     throw new Error('Failed to fetch articles');
   }
-  return response.json();
+  const data: PaginatedArticles = await response.json();
+  return data.articles;
+}
+
+export async function getArticleCount(): Promise<number> {
+  const response = await fetch('/api/articles/count');
+  if (!response.ok) {
+    throw new Error('Failed to fetch article count');
+  }
+  const data = await response.json();
+  return data.count;
 }

frontend/src/routeTree.gen.ts

@@ -18,6 +18,7 @@ import { Route as DashboardImport } from './routes/dashboard'
 import { Route as CollectionsImport } from './routes/collections'
 import { Route as ArticlesImport } from './routes/articles'
 import { Route as IndexImport } from './routes/index'
+import { Route as DashboardIndexImport } from './routes/dashboard/index'
 import { Route as CollectionsnameImport } from './routes/collections/[name]'
 
 // Create/Update Routes
@@ -64,6 +65,12 @@ const IndexRoute = IndexImport.update({
   getParentRoute: () => rootRoute,
 } as any)
 
+const DashboardIndexRoute = DashboardIndexImport.update({
+  id: '/',
+  path: '/',
+  getParentRoute: () => DashboardRoute,
+} as any)
+
 const CollectionsnameRoute = CollectionsnameImport.update({
   id: '/[name]',
   path: '/[name]',
@@ -130,6 +137,13 @@ declare module '@tanstack/react-router' {
       preLoaderRoute: typeof CollectionsnameImport
       parentRoute: typeof CollectionsImport
     }
+    '/dashboard/': {
+      id: '/dashboard/'
+      path: '/'
+      fullPath: '/dashboard/'
+      preLoaderRoute: typeof DashboardIndexImport
+      parentRoute: typeof DashboardImport
+    }
   }
 }
 
@@ -147,26 +161,39 @@ const CollectionsRouteWithChildren = CollectionsRoute._addFileChildren(
   CollectionsRouteChildren,
 )
 
+interface DashboardRouteChildren {
+  DashboardIndexRoute: typeof DashboardIndexRoute
+}
+
+const DashboardRouteChildren: DashboardRouteChildren = {
+  DashboardIndexRoute: DashboardIndexRoute,
+}
+
+const DashboardRouteWithChildren = DashboardRoute._addFileChildren(
+  DashboardRouteChildren,
+)
+
 export interface FileRoutesByFullPath {
   '/': typeof IndexRoute
   '/articles': typeof ArticlesRoute
   '/collections': typeof CollectionsRouteWithChildren
-  '/dashboard': typeof DashboardRoute
+  '/dashboard': typeof DashboardRouteWithChildren
   '/login': typeof LoginRoute
   '/notFound': typeof NotFoundRoute
   '/register': typeof RegisterRoute
   '/collections/[name]': typeof CollectionsnameRoute
+  '/dashboard/': typeof DashboardIndexRoute
 }
 
 export interface FileRoutesByTo {
   '/': typeof IndexRoute
   '/articles': typeof ArticlesRoute
   '/collections': typeof CollectionsRouteWithChildren
-  '/dashboard': typeof DashboardRoute
   '/login': typeof LoginRoute
   '/notFound': typeof NotFoundRoute
   '/register': typeof RegisterRoute
   '/collections/[name]': typeof CollectionsnameRoute
+  '/dashboard': typeof DashboardIndexRoute
 }
 
 export interface FileRoutesById {
@@ -174,11 +201,12 @@ export interface FileRoutesById {
   '/': typeof IndexRoute
   '/articles': typeof ArticlesRoute
   '/collections': typeof CollectionsRouteWithChildren
-  '/dashboard': typeof DashboardRoute
+  '/dashboard': typeof DashboardRouteWithChildren
   '/login': typeof LoginRoute
   '/notFound': typeof NotFoundRoute
   '/register': typeof RegisterRoute
   '/collections/[name]': typeof CollectionsnameRoute
+  '/dashboard/': typeof DashboardIndexRoute
 }
 
 export interface FileRouteTypes {
@@ -192,16 +220,17 @@ export interface FileRouteTypes {
     | '/notFound'
     | '/register'
     | '/collections/[name]'
+    | '/dashboard/'
   fileRoutesByTo: FileRoutesByTo
   to:
     | '/'
     | '/articles'
     | '/collections'
-    | '/dashboard'
     | '/login'
     | '/notFound'
     | '/register'
     | '/collections/[name]'
+    | '/dashboard'
   id:
     | '__root__'
     | '/'
@@ -212,6 +241,7 @@ export interface FileRouteTypes {
     | '/notFound'
     | '/register'
     | '/collections/[name]'
+    | '/dashboard/'
   fileRoutesById: FileRoutesById
 }
 
@@ -219,7 +249,7 @@ export interface RootRouteChildren {
   IndexRoute: typeof IndexRoute
   ArticlesRoute: typeof ArticlesRoute
   CollectionsRoute: typeof CollectionsRouteWithChildren
-  DashboardRoute: typeof DashboardRoute
+  DashboardRoute: typeof DashboardRouteWithChildren
   LoginRoute: typeof LoginRoute
   NotFoundRoute: typeof NotFoundRoute
   RegisterRoute: typeof RegisterRoute
@@ -229,7 +259,7 @@ const rootRouteChildren: RootRouteChildren = {
   IndexRoute: IndexRoute,
   ArticlesRoute: ArticlesRoute,
   CollectionsRoute: CollectionsRouteWithChildren,
-  DashboardRoute: DashboardRoute,
+  DashboardRoute: DashboardRouteWithChildren,
   LoginRoute: LoginRoute,
   NotFoundRoute: NotFoundRoute,
   RegisterRoute: RegisterRoute,
@@ -267,7 +297,10 @@ export const routeTree = rootRoute
       ]
     },
     "/dashboard": {
-      "filePath": "dashboard.tsx"
+      "filePath": "dashboard.tsx",
+      "children": [
+        "/dashboard/"
+      ]
     },
     "/login": {
       "filePath": "login.tsx"
@@ -281,6 +314,10 @@ export const routeTree = rootRoute
     "/collections/[name]": {
       "filePath": "collections/[name].tsx",
       "parent": "/collections"
+    },
+    "/dashboard/": {
+      "filePath": "dashboard/index.tsx",
+      "parent": "/dashboard"
     }
   }
 }

frontend/src/routes/articles.tsx

@@ -1,11 +1,12 @@
 import { createFileRoute } from '@tanstack/react-router'
-import { useQuery } from '@tanstack/react-query'
+import { useInfiniteQuery } from '@tanstack/react-query'
 import { fetchArticles } from '@/api/articles'
 import { Button } from '@/components/ui/button'
 import { addArticleToCollection, createCollection, fetchCollections } from '@/api/collections';
 import { useUser } from '@/store/user';
 import { useNavigate } from '@tanstack/react-router';
 import { Article } from '@/types';
+import { useEffect, useRef, useCallback } from 'react';
 
 export const Route = createFileRoute('/articles')({
   component: ArticlesPage
@@ -52,29 +53,62 @@ async function handleAddToCollection(
 }
 
 function ArticlesPage() {
-  const { data: articles, isLoading, isError, error } = useQuery({
+  const {
+    data,
+    isLoading,
+    isError,
+    error,
+    fetchNextPage,
+    hasNextPage,
+    isFetchingNextPage
+  } = useInfiniteQuery({
     queryKey: ['articles'],
-    queryFn: async () => {
-      try {
-        const response = await fetchArticles();
-        console.log('Articles API response:', response);
+    queryFn: async ({ pageParam = undefined }) => {
+      const response = await fetchArticles(pageParam);
       return response;
-      } catch (err) {
-        console.error('Error fetching articles:', err);
-        throw err;
-      }
     },
+    getNextPageParam: (lastPage) => lastPage.nextCursor,
+    initialPageParam: undefined as string | undefined,
     staleTime: 1000 * 60 // Cache for 1 minute
   });
+
   const user = useUser();
   const navigate = useNavigate();
+  const loadMoreRef = useRef<HTMLDivElement>(null);
 
-  if (isLoading) {
+  const handleObserver = useCallback((entries: IntersectionObserverEntry[]) => {
+    const [target] = entries;
+    if (target.isIntersecting && hasNextPage && !isFetchingNextPage) {
+      void fetchNextPage();
+    }
+  }, [fetchNextPage, hasNextPage, isFetchingNextPage]);
+
+  useEffect(() => {
+    const element = loadMoreRef.current;
+    const observer = new IntersectionObserver(handleObserver, {
+      root: null,
+      rootMargin: '0px',
+      threshold: 0.1,
+    });
+
+    if (element) {
+      observer.observe(element);
+    }
+
+    return () => {
+      if (element) {
+        observer.unobserve(element);
+      }
+    };
+  }, [handleObserver]);
+
+  // Show initial loading state
+  if (isLoading && !data?.pages?.length) {
     return (
       <div className="flex items-center justify-center h-64">
         <div className="animate-spin w-8 h-8 border-4 border-primary border-t-transparent rounded-full"></div>
       </div>
-    )
+    );
   }
 
   if (isError) {
@@ -86,9 +120,11 @@ function ArticlesPage() {
           Retry
         </Button>
       </div>
-    )
+    );
   }
 
+  const allArticles = data?.pages.flatMap(page => page.articles) ?? [];
+
   return (
     <div className="space-y-6">
       <div>
@@ -96,9 +132,10 @@ function ArticlesPage() {
         <p className="text-muted-foreground mt-2">Browse the latest news from various sources</p>
       </div>
       
-      {articles && articles.length > 0 ? (
+      {allArticles.length > 0 ? (
+        <>
           <div className="grid gap-6 sm:grid-cols-2 lg:grid-cols-3">
-          {articles.map(article => (
+            {allArticles.map(article => (
               <div 
                 key={article.id} 
                 className="rounded-lg border bg-card shadow-sm overflow-hidden hover:shadow-md transition-shadow"
@@ -117,6 +154,7 @@ function ArticlesPage() {
                   <Button
                     variant="outline"
                     size="sm"
+                    className="mt-4 w-full"
                     onClick={() => handleAddToCollection(article, user?.id, navigate)}
                   >
                     Add to Collection
@@ -125,6 +163,18 @@ function ArticlesPage() {
               </div>
             ))}
           </div>
+
+          {/* Loading more trigger and indicator */}
+          <div ref={loadMoreRef} className="flex justify-center py-8">
+            {isFetchingNextPage ? (
+              <div className="animate-spin w-6 h-6 border-3 border-primary border-t-transparent rounded-full"></div>
+            ) : hasNextPage ? (
+              <span className="text-sm text-muted-foreground">Scroll to load more</span>
+            ) : (
+              <span className="text-sm text-muted-foreground">No more articles to load</span>
+            )}
+          </div>
+        </>
       ) : (
         <div className="text-center p-12 border rounded-lg bg-background">
           <h3 className="text-lg font-medium">No articles found</h3>
@@ -132,6 +182,6 @@ function ArticlesPage() {
         </div>
       )}
     </div>
-  )
+  );
 }
 

frontend/src/routes/dashboard/index.tsx

@@ -0,0 +1,61 @@
+import { createFileRoute } from '@tanstack/react-router';
+import { useQuery } from '@tanstack/react-query';
+import { getArticleCount } from '@/api/articles';
+import { NewspaperIcon, BookmarkIcon, UsersIcon } from 'lucide-react';
+
+function DashboardIndex() {
+  const { data: articleCount = 0, isLoading } = useQuery({
+    queryKey: ['articleCount'],
+    queryFn: getArticleCount,
+  });
+
+  return (
+    <div className="space-y-6">
+      <h1 className="text-3xl font-bold">Dashboard</h1>
+      
+      <div className="grid gap-6 sm:grid-cols-2 lg:grid-cols-3">
+        <div className="rounded-lg border bg-card p-6">
+          <div className="flex items-center gap-4">
+            <div className="rounded-full bg-primary/10 p-3">
+              <NewspaperIcon className="h-6 w-6 text-primary" />
+            </div>
+            <div>
+              <p className="text-sm text-muted-foreground">Total Articles</p>
+              <h3 className="text-2xl font-bold">
+                {isLoading ? '...' : articleCount}
+              </h3>
+            </div>
+          </div>
+        </div>
+
+        <div className="rounded-lg border bg-card p-6">
+          <div className="flex items-center gap-4">
+            <div className="rounded-full bg-primary/10 p-3">
+              <BookmarkIcon className="h-6 w-6 text-primary" />
+            </div>
+            <div>
+              <p className="text-sm text-muted-foreground">Collections</p>
+              <h3 className="text-2xl font-bold">-</h3>
+            </div>
+          </div>
+        </div>
+
+        <div className="rounded-lg border bg-card p-6">
+          <div className="flex items-center gap-4">
+            <div className="rounded-full bg-primary/10 p-3">
+              <UsersIcon className="h-6 w-6 text-primary" />
+            </div>
+            <div>
+              <p className="text-sm text-muted-foreground">Active Users</p>
+              <h3 className="text-2xl font-bold">-</h3>
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}
+
+export const Route = createFileRoute('/dashboard/')({
+  component: DashboardIndex,
+});

frontend/src/routes/index.tsx

@@ -1,8 +1,8 @@
 import { createFileRoute } from '@tanstack/react-router'
 import { Button } from '@/components/ui/button'
 import { useRouter } from '@tanstack/react-router'
-import { useQuery } from '@tanstack/react-query';
-import { fetchAllArticles } from '@/api/articles';
+// import { useQuery } from '@tanstack/react-query';
+// import { fetchAllArticles } from '@/api/articles';
 
 export const Route = createFileRoute('/')({
   component: IndexPage,
@@ -15,18 +15,18 @@ function IndexPage() {
     router.navigate({ to: path });
   };
 
-  const { data: articles, isLoading, isError } = useQuery({
-    queryKey: ['allArticles'],
-    queryFn: fetchAllArticles,
-  });
+  // const { data: articles = [], isLoading, isError } = useQuery({
+  //   queryKey: ['allArticles'],
+  //   queryFn: fetchAllArticles,
+  // });
 
-  if (isLoading) {
-    return <div>Loading articles...</div>;
-  }
+  // if (isLoading) {
+  //   return <div>Loading articles...</div>;
+  // }
 
-  if (isError) {
-    return <div>Failed to load articles. Please try again later.</div>;
-  }
+  // if (isError) {
+  //   return <div>Failed to load articles. Please try again later.</div>;
+  // }
 
   return (
     <div className="space-y-8">
@@ -53,9 +53,9 @@ function IndexPage() {
         </div>
       </div>
       
-      <div className="container px-4 md:px-6">
+      {/* <div className="container px-4 md:px-6">
         <div className="grid gap-6 md:grid-cols-2 lg:grid-cols-3">
-          {articles?.map((article) => (
+          {articles.map((article) => (
             <div key={article.id} className="rounded-lg border bg-card p-6 text-card-foreground shadow">
               <h3 className="text-lg font-semibold">{article.title}</h3>
               <p className="text-sm text-muted-foreground">
@@ -67,7 +67,7 @@ function IndexPage() {
             </div>
           ))}
         </div>
-      </div>
+      </div> */}
     </div>
   )
 }