feat: implement feed agg and browse

go.mod

@@ -5,4 +5,6 @@ go 1.26.3
 require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/lib/pq v1.12.3 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/term v0.45.0 // indirect
 )

go.sum

@@ -2,3 +2,7 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
 github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=

internal/command/command.go

@@ -5,14 +5,19 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"os"
+	"os/signal"
 	"slices"
+	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/StevanFreeborn/gator/internal/database"
 	"github.com/StevanFreeborn/gator/internal/rss"
 	"github.com/StevanFreeborn/gator/internal/state"
 	"github.com/google/uuid"
+	"golang.org/x/term"
 )
 
 type Command struct {
@@ -54,6 +59,7 @@ func NewRegistry() *CommandRegistry {
 		followCommand(),
 		followingCommand(),
 		unfollowCommand(),
+		browseCommand(),
 	}
 
 	for _, cmd := range commands {
@@ -172,25 +178,60 @@ func usersCommand() *Command {
 
 func aggCommand() *Command {
 	return newCommand("agg", func(s *state.State) error {
-		// if len(s.Arguments) == 0 {
+		if len(s.Arguments) == 0 {
-		// 	return fmt.Errorf("Did not receive expected feed argument")
+			return fmt.Errorf("Did not receive expected time between requests argument")
-		// }
-
-		// feed := s.Arguments[0]
-		//
-		// validUrl, err := url.Parse(feed)
-		//
-		// if err != nil {
-		// 	return fmt.Errorf("Feed '%s' is not a valid url", feed)
-		// }
-
-		rssFeed, err := rss.FetchFeed(context.Background(), "https://www.wagslane.dev/index.xml")
-
-		if err != nil {
-			return err
 		}
 
-		fmt.Printf("%v\n", rssFeed)
+		timeBetweenRequests := s.Arguments[0]
+		validDuration, err := time.ParseDuration(timeBetweenRequests)
+
+		if err != nil {
+			return fmt.Errorf("Time between requests argument '%s' not valid duration string", timeBetweenRequests)
+		}
+
+		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+		defer cancel()
+
+		oldState, err := term.MakeRaw(int(os.Stdin.Fd()))
+
+		if err != nil {
+			return fmt.Errorf("Failed to enable raw mode: %v", err)
+		}
+
+		defer term.Restore(int(os.Stdin.Fd()), oldState)
+
+		go func() {
+			buf := make([]byte, 1)
+
+			for {
+				n, err := os.Stdin.Read(buf)
+
+				if err != nil || n == 0 {
+					return
+				}
+
+				if buf[0] == 'q' || buf[0] == 'Q' || buf[0] == 3 {
+					cancel()
+					return
+				}
+			}
+		}()
+
+		ticker := time.NewTicker(validDuration)
+		defer ticker.Stop()
+
+		rss.ScrapeNextFeed(ctx, s)
+
+	fetchFeeds:
+		for {
+			select {
+			case <-ctx.Done():
+				fmt.Print("Stopping feed agg command\r\n")
+				break fetchFeeds
+			case <-ticker.C:
+				rss.ScrapeNextFeed(ctx, s)
+			}
+		}
 
 		return nil
 	})
@@ -321,7 +362,7 @@ func followingCommand() *Command {
 		}
 
 		for _, follow := range follows {
-			fmt.Printf("* %s", follow.FeedName)
+			fmt.Printf("* %s\n", follow.FeedName)
 		}
 
 		return nil
@@ -358,3 +399,34 @@ func unfollowCommand() *Command {
 		return nil
 	}))
 }
+
+func browseCommand() *Command {
+	return newCommand("browse", requiresLoggedInUser(func(s *state.State, currentUser database.User) error {
+		limit := int32(2)
+
+		if len(s.Arguments) > 0 {
+			parsedLimit, err := strconv.ParseInt(s.Arguments[0], 10, 32)
+
+			if err == nil {
+				limit = int32(parsedLimit)
+			}
+		}
+
+		getUserPostsParams := database.GetPostsForUserParams{
+			UserID: currentUser.ID,
+			Limit:  limit,
+		}
+
+		posts, err := s.Database.GetPostsForUser(context.Background(), getUserPostsParams)
+
+		if err != nil {
+			return err
+		}
+
+		for _, post := range posts {
+			fmt.Printf("* %s - %s\n", post.FeedName, post.Title)
+		}
+
+		return nil
+	}))
+}

internal/database/feeds.sql.go

@@ -7,31 +7,34 @@ package database
 
 import (
 	"context"
+	"database/sql"
 	"time"
 
 	"github.com/google/uuid"
 )
 
 const createFeed = `-- name: CreateFeed :one
-INSERT INTO feeds (id, created_at, updated_at, name, url, user_id)
+INSERT INTO feeds (id, created_at, updated_at, name, url, user_id, last_fetched_at)
 VALUES (
   $1,
   $2,
   $3,
   $4,
   $5,
-  $6
+  $6,
+  $7
 )
-RETURNING id, created_at, updated_at, name, url, user_id
+RETURNING id, created_at, updated_at, name, url, user_id, last_fetched_at
 `
 
 type CreateFeedParams struct {
-	ID        uuid.UUID
+	ID            uuid.UUID
-	CreatedAt time.Time
+	CreatedAt     time.Time
-	UpdatedAt time.Time
+	UpdatedAt     time.Time
-	Name      string
+	Name          string
-	Url       string
+	Url           string
-	UserID    uuid.UUID
+	UserID        uuid.UUID
+	LastFetchedAt sql.NullTime
 }
 
 func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, error) {
@@ -42,6 +45,7 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e
 		arg.Name,
 		arg.Url,
 		arg.UserID,
+		arg.LastFetchedAt,
 	)
 	var i Feed
 	err := row.Scan(
@@ -51,12 +55,13 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e
 		&i.Name,
 		&i.Url,
 		&i.UserID,
+		&i.LastFetchedAt,
 	)
 	return i, err
 }
 
 const getAllFeeds = `-- name: GetAllFeeds :many
-SELECT id, created_at, updated_at, name, url, user_id FROM feeds
+SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
 `
 
 func (q *Queries) GetAllFeeds(ctx context.Context) ([]Feed, error) {
@@ -75,6 +80,7 @@ func (q *Queries) GetAllFeeds(ctx context.Context) ([]Feed, error) {
 			&i.Name,
 			&i.Url,
 			&i.UserID,
+			&i.LastFetchedAt,
 		); err != nil {
 			return nil, err
 		}
@@ -90,7 +96,7 @@ func (q *Queries) GetAllFeeds(ctx context.Context) ([]Feed, error) {
 }
 
 const getFeedByUrl = `-- name: GetFeedByUrl :one
-SELECT id, created_at, updated_at, name, url, user_id FROM feeds
+SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
 WHERE url = $1
 `
 
@@ -104,6 +110,43 @@ func (q *Queries) GetFeedByUrl(ctx context.Context, url string) (Feed, error) {
 		&i.Name,
 		&i.Url,
 		&i.UserID,
+		&i.LastFetchedAt,
 	)
 	return i, err
 }
+
+const getNextFeedToFetch = `-- name: GetNextFeedToFetch :one
+SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
+ORDER BY last_fetched_at ASC NULLS FIRST
+`
+
+func (q *Queries) GetNextFeedToFetch(ctx context.Context) (Feed, error) {
+	row := q.db.QueryRowContext(ctx, getNextFeedToFetch)
+	var i Feed
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.Name,
+		&i.Url,
+		&i.UserID,
+		&i.LastFetchedAt,
+	)
+	return i, err
+}
+
+const markFeedAsFetched = `-- name: MarkFeedAsFetched :exec
+UPDATE feeds
+SET last_fetched_at = $1
+WHERE feeds.id = $2
+`
+
+type MarkFeedAsFetchedParams struct {
+	LastFetchedAt sql.NullTime
+	ID            uuid.UUID
+}
+
+func (q *Queries) MarkFeedAsFetched(ctx context.Context, arg MarkFeedAsFetchedParams) error {
+	_, err := q.db.ExecContext(ctx, markFeedAsFetched, arg.LastFetchedAt, arg.ID)
+	return err
+}

internal/database/models.go

@@ -5,18 +5,20 @@
 package database
 
 import (
+	"database/sql"
 	"time"
 
 	"github.com/google/uuid"
 )
 
 type Feed struct {
-	ID        uuid.UUID
+	ID            uuid.UUID
-	CreatedAt time.Time
+	CreatedAt     time.Time
-	UpdatedAt time.Time
+	UpdatedAt     time.Time
-	Name      string
+	Name          string
-	Url       string
+	Url           string
-	UserID    uuid.UUID
+	UserID        uuid.UUID
+	LastFetchedAt sql.NullTime
 }
 
 type Follow struct {
@@ -27,6 +29,17 @@ type Follow struct {
 	FeedID    uuid.UUID
 }
 
+type Post struct {
+	ID          uuid.UUID
+	CreatedAt   time.Time
+	UpdatedAt   time.Time
+	Title       string
+	Url         string
+	Description string
+	PublishedAt sql.NullTime
+	FeedID      uuid.UUID
+}
+
 type User struct {
 	ID        uuid.UUID
 	CreatedAt time.Time

internal/database/posts.sql.go

@@ -0,0 +1,136 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.31.1
+// source: posts.sql
+
+package database
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+const createPost = `-- name: CreatePost :one
+INSERT INTO posts (id, created_at, updated_at, title, url, description, published_at, feed_id)
+VALUES (
+  $1,
+  $2,
+  $3,
+  $4,
+  $5,
+  $6,
+  $7,
+  $8
+)
+RETURNING id, created_at, updated_at, title, url, description, published_at, feed_id
+`
+
+type CreatePostParams struct {
+	ID          uuid.UUID
+	CreatedAt   time.Time
+	UpdatedAt   time.Time
+	Title       string
+	Url         string
+	Description string
+	PublishedAt sql.NullTime
+	FeedID      uuid.UUID
+}
+
+func (q *Queries) CreatePost(ctx context.Context, arg CreatePostParams) (Post, error) {
+	row := q.db.QueryRowContext(ctx, createPost,
+		arg.ID,
+		arg.CreatedAt,
+		arg.UpdatedAt,
+		arg.Title,
+		arg.Url,
+		arg.Description,
+		arg.PublishedAt,
+		arg.FeedID,
+	)
+	var i Post
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.Title,
+		&i.Url,
+		&i.Description,
+		&i.PublishedAt,
+		&i.FeedID,
+	)
+	return i, err
+}
+
+const getPostsForUser = `-- name: GetPostsForUser :many
+SELECT 
+  posts.id, 
+  posts.created_at, 
+  posts.updated_at, 
+  posts.title, 
+  posts.url, 
+  posts.description, 
+  posts.published_at, 
+  posts.feed_id,
+  feeds.name as feed_name
+FROM posts
+INNER JOIN feeds
+  ON posts.feed_id = feeds.id
+INNER JOIN follows
+  ON posts.feed_id = follows.feed_id
+WHERE follows.user_id = $1
+ORDER BY posts.published_at DESC, posts.title ASC
+LIMIT $2
+`
+
+type GetPostsForUserParams struct {
+	UserID uuid.UUID
+	Limit  int32
+}
+
+type GetPostsForUserRow struct {
+	ID          uuid.UUID
+	CreatedAt   time.Time
+	UpdatedAt   time.Time
+	Title       string
+	Url         string
+	Description string
+	PublishedAt sql.NullTime
+	FeedID      uuid.UUID
+	FeedName    string
+}
+
+func (q *Queries) GetPostsForUser(ctx context.Context, arg GetPostsForUserParams) ([]GetPostsForUserRow, error) {
+	rows, err := q.db.QueryContext(ctx, getPostsForUser, arg.UserID, arg.Limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []GetPostsForUserRow
+	for rows.Next() {
+		var i GetPostsForUserRow
+		if err := rows.Scan(
+			&i.ID,
+			&i.CreatedAt,
+			&i.UpdatedAt,
+			&i.Title,
+			&i.Url,
+			&i.Description,
+			&i.PublishedAt,
+			&i.FeedID,
+			&i.FeedName,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}

internal/rss/rss.go

@@ -2,9 +2,16 @@ package rss
 
 import (
 	"context"
+	"database/sql"
 	"encoding/xml"
+	"fmt"
 	"html"
 	"net/http"
+	"time"
+
+	"github.com/StevanFreeborn/gator/internal/database"
+	"github.com/StevanFreeborn/gator/internal/state"
+	"github.com/google/uuid"
 )
 
 type RSSItem struct {
@@ -25,7 +32,7 @@ type RSSFeed struct {
 	Channel RSSChannel `xml:"channel"`
 }
 
-func FetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
+func fetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
 	var rssFeed RSSFeed
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
@@ -59,3 +66,63 @@ func FetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
 
 	return &rssFeed, nil
 }
+
+func ScrapeNextFeed(ctx context.Context, s *state.State) {
+	nextFeed, err := s.Database.GetNextFeedToFetch(ctx)
+
+	if err != nil {
+		fmt.Print("Failed to lookup next feed to fetch\r\n")
+		return
+	}
+
+	fmt.Printf("Fetching feed '%s'\r\n", nextFeed.Name)
+
+	fetchedFeed, err := fetchFeed(ctx, nextFeed.Url)
+
+	if err != nil {
+		fmt.Printf("Failed to fetch feed '%s'\r\n", nextFeed.Name)
+		return
+	}
+
+	markFeedParams := database.MarkFeedAsFetchedParams{
+		ID: nextFeed.ID,
+		LastFetchedAt: sql.NullTime{
+			Time:  time.Now(),
+			Valid: true,
+		},
+	}
+
+	err = s.Database.MarkFeedAsFetched(ctx, markFeedParams)
+
+	for _, item := range fetchedFeed.Channel.Item {
+		publishedAt, err := time.Parse(time.RFC3339, item.PubDate)
+		publishedAtValue := sql.NullTime{
+			Time:  publishedAt,
+			Valid: err == nil,
+		}
+
+		createPostParams := database.CreatePostParams{
+			ID:          uuid.New(),
+			CreatedAt:   time.Now(),
+			UpdatedAt:   time.Now(),
+			Title:       item.Title,
+			Url:         item.Link,
+			Description: item.Description,
+			PublishedAt: publishedAtValue,
+			FeedID:      nextFeed.ID,
+		}
+
+		_, err = s.Database.CreatePost(ctx, createPostParams)
+
+		if err != nil {
+			fmt.Printf("  Skipping saving post '%s'\r\n", item.Title)
+			continue
+		}
+
+		fmt.Printf("  Saved post '%s'\r\n", item.Title)
+	}
+
+	if err != nil {
+		fmt.Printf("Failed to mark feed as fetched.\r\n")
+	}
+}

sql/queries/feeds.sql

@@ -1,18 +1,28 @@
 -- name: CreateFeed :one
-INSERT INTO feeds (id, created_at, updated_at, name, url, user_id)
+INSERT INTO feeds (id, created_at, updated_at, name, url, user_id, last_fetched_at)
 VALUES (
   $1,
   $2,
   $3,
   $4,
   $5,
-  $6
+  $6,
+  $7
 )
 RETURNING *;
 
 -- name: GetAllFeeds :many
-SELECT id, created_at, updated_at, name, url, user_id FROM feeds;
+SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds;
 
 -- name: GetFeedByUrl :one
-SELECT id, created_at, updated_at, name, url, user_id FROM feeds
+SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
 WHERE url = $1;
+
+-- name: GetNextFeedToFetch :one
+SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
+ORDER BY last_fetched_at ASC NULLS FIRST;
+
+-- name: MarkFeedAsFetched :exec
+UPDATE feeds
+SET last_fetched_at = $1
+WHERE feeds.id = $2;

sql/queries/posts.sql

@@ -0,0 +1,33 @@
+-- name: CreatePost :one
+INSERT INTO posts (id, created_at, updated_at, title, url, description, published_at, feed_id)
+VALUES (
+  $1,
+  $2,
+  $3,
+  $4,
+  $5,
+  $6,
+  $7,
+  $8
+)
+RETURNING *;
+
+-- name: GetPostsForUser :many
+SELECT 
+  posts.id, 
+  posts.created_at, 
+  posts.updated_at, 
+  posts.title, 
+  posts.url, 
+  posts.description, 
+  posts.published_at, 
+  posts.feed_id,
+  feeds.name as feed_name
+FROM posts
+INNER JOIN feeds
+  ON posts.feed_id = feeds.id
+INNER JOIN follows
+  ON posts.feed_id = follows.feed_id
+WHERE follows.user_id = $1
+ORDER BY posts.published_at DESC, posts.title ASC
+LIMIT $2;

sql/schema/004_lastfetchedat.sql

@@ -0,0 +1,8 @@
+-- +goose Up
+ALTER TABLE feeds
+ADD COLUMN last_fetched_at TIMESTAMP NULL;
+
+-- +goose Down
+ALTER TABLE feeds
+DROP COLUMN last_fetched_at;
+

sql/schema/005_posts.sql

@@ -0,0 +1,16 @@
+-- +goose Up
+CREATE TABLE posts (
+  id UUID PRIMARY KEY,
+  created_at TIMESTAMP NOT NULL,
+  updated_at TIMESTAMP NOT NULL,
+  title TEXT NOT NULL,
+  url TEXT NOT NULL,
+  description TEXT NOT NULL,
+  published_at TIMESTAMP NULL,
+  feed_id UUID NOT NULL,
+  FOREIGN KEY(feed_id) REFERENCES feeds(id) ON DELETE CASCADE,
+  CONSTRAINT unique_url UNIQUE(url)
+);
+
+-- +goose Down
+DROP TABLE posts;