feat: address TODOs and write README.md

- added follow command
- added unfollow command
- following command
- introduced logged in user middleware

README.md

@@ -1,3 +1,77 @@
 # gator
 
-This is a blog aggregator that was built as part of a [boot.dev](https://boot.dev) course.
+A blog aggregator CLI built as part of the [boot.dev](https://boot.dev) course. Follow RSS feeds and browse their latest posts right from your terminal.
+
+## Requirements
+
+Before you can run gator you'll need to have the following installed:
+
+- **Go** (1.26 or newer)
+- **PostgreSQL** (a running server, local or remote)
+
+## Installation
+
+Install the gator CLI with `go install`:
+
+```sh
+go install github.com/StevanFreeborn/gator@latest
+```
+
+The binary is placed in `$(go env GOPATH)/bin`. Make sure that directory is on your `PATH`:
+
+```sh
+export PATH="$(go env GOPATH)/bin:$PATH"
+```
+
+## Setup
+
+### 1. Create a database
+
+Create a database for gator in your Postgres server:
+
+```sh
+createdb gator
+```
+
+The schema is managed with [goose](https://github.com/pressly/goose) migrations in `sql/schema`. Install goose and apply the migrations:
+
+```sh
+go install github.com/pressly/goose/v3/cmd/goose@latest
+goose -dir sql/schema postgres "postgres://<user>:<password>@localhost:5432/gator?sslmode=disable" up
+```
+
+### 2. Create the config file
+
+gator reads its configuration from `~/.gatorconfig.json`. Create that file with the connection string for your database:
+
+```json
+{
+  "db_url": "postgres://<user>:<password>@localhost:5432/gator?sslmode=disable",
+  "current_user_name": ""
+}
+```
+
+## Usage
+
+Run gator with `gator <command> <args>`. Start by registering a user, then log in:
+
+```sh
+gator register myusername
+gator login myusername
+```
+
+A few commands you can run:
+
+| Command                        | Description                                                                             |
+| ------------------------------ | --------------------------------------------------------------------------------------- |
+| `gator register <name>`        | Register a new user and log in as them                                                  |
+| `gator login <name>`           | Log in as an existing user                                                              |
+| `gator users`                  | List all registered users                                                               |
+| `gator addfeed <name> <url>`   | Add an RSS feed and follow it                                                           |
+| `gator feeds`                  | List all feeds                                                                          |
+| `gator follow <name-or-url>`   | Follow a feed by name or url                                                            |
+| `gator following`              | List the feeds you're following                                                         |
+| `gator unfollow <name-or-url>` | Unfollow a feed by name or url                                                          |
+| `gator browse [limit]`         | Browse posts from your followed feeds                                                   |
+| `gator agg <duration>`         | Continuously fetch posts every `<duration>` (e.g. `60s`); press `q` or `Ctrl-C` to stop |
+| `gator reset`                  | Delete all users and their data                                                         |

go.mod

@@ -5,4 +5,6 @@ go 1.26.3
 require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/lib/pq v1.12.3 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/term v0.45.0 // indirect
 )

go.sum

@@ -2,3 +2,7 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
 github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=

internal/command/command.go

@@ -4,9 +4,12 @@ import (
 	"cmp"
 	"context"
 	"fmt"
-	"net/url"
+	"os"
+	"os/signal"
 	"slices"
+	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/StevanFreeborn/gator/internal/database"
@@ -51,6 +54,10 @@ func NewRegistry() *CommandRegistry {
 		aggCommand(),
 		addFeedCommand(),
 		feedsCommand(),
+		followCommand(),
+		followingCommand(),
+		unfollowCommand(),
+		browseCommand(),
 	}
 
 	for _, cmd := range commands {
@@ -169,32 +176,50 @@ func usersCommand() *Command {
 
 func aggCommand() *Command {
 	return newCommand("agg", func(s *state.State) error {
-		// if len(s.Arguments) == 0 {
+		if len(s.Arguments) == 0 {
-		// 	return fmt.Errorf("Did not receive expected feed argument")
+			return fmt.Errorf("Did not receive expected time between requests argument")
-		// }
+		}
 
-		// feed := s.Arguments[0]
+		timeBetweenRequests := s.Arguments[0]
-		//
+		validDuration, err := time.ParseDuration(timeBetweenRequests)
-		// validUrl, err := url.Parse(feed)
-		//
-		// if err != nil {
-		// 	return fmt.Errorf("Feed '%s' is not a valid url", feed)
-		// }
 
-		rssFeed, err := rss.FetchFeed(context.Background(), "https://www.wagslane.dev/index.xml")
+		if err != nil {
+			return fmt.Errorf("Time between requests argument '%s' not valid duration string", timeBetweenRequests)
+		}
+
+		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+		defer cancel()
+
+		restore, err := enableKeypressExit(cancel)
 
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("%v\n", rssFeed)
+		defer restore()
+
+		ticker := time.NewTicker(validDuration)
+		defer ticker.Stop()
+
+		rss.ScrapeNextFeed(ctx, s)
+
+	fetchFeeds:
+		for {
+			select {
+			case <-ctx.Done():
+				fmt.Print("Stopping feed agg command\r\n")
+				break fetchFeeds
+			case <-ticker.C:
+				rss.ScrapeNextFeed(ctx, s)
+			}
+		}
 
 		return nil
 	})
 }
 
 func addFeedCommand() *Command {
-	return newCommand("addfeed", func(s *state.State) error {
+	return newCommand("addfeed", requiresLoggedInUser(func(s *state.State, currentUser database.User) error {
 		if len(s.Arguments) < 2 {
 			return fmt.Errorf("Did not receive expected feed name and url")
 		}
@@ -202,27 +227,27 @@ func addFeedCommand() *Command {
 		feedName := s.Arguments[0]
 		feedUrl := s.Arguments[1]
 
-		validatedFeedUrl, err := url.Parse(feedUrl)
-
 		if strings.TrimSpace(feedName) == "" {
 			return fmt.Errorf("Feed name cannot be empty")
 		}
 
+		canonicalFeedUrl, err := normalizeFeedURL(feedUrl)
+
 		if err != nil {
 			return fmt.Errorf("Feed url '%s' is not a valid url", feedUrl)
 		}
 
-		currentUser, err := s.GetCurrentUser(context.Background())
+		_, err = s.Database.GetFeedByUrl(context.Background(), canonicalFeedUrl)
 
-		if err != nil {
+		if err == nil {
-			return fmt.Errorf("Currently logged in user does not exist. Cannot add feed for non-existent user.")
+			return fmt.Errorf("Feed with url '%s' already exists; use 'follow <name>' to follow it", canonicalFeedUrl)
 		}
 
 		createFeedParams := database.CreateFeedParams{
 			ID:        uuid.New(),
 			UserID:    currentUser.ID,
 			Name:      feedName,
-			Url:       validatedFeedUrl.String(),
+			Url:       canonicalFeedUrl,
 			CreatedAt: time.Now(),
 			UpdatedAt: time.Now(),
 		}
@@ -233,14 +258,28 @@ func addFeedCommand() *Command {
 			return err
 		}
 
-		fmt.Printf("Successfully added feed '%s' with url '%s'\n", createdFeed.Name, createdFeed.Url)
+		createFollowParams := database.CreateFollowParams{
+			ID:        uuid.New(),
+			CreatedAt: time.Now(),
+			UpdatedAt: time.Now(),
+			UserID:    currentUser.ID,
+			FeedID:    createdFeed.ID,
+		}
+
+		_, err = s.Database.CreateFollow(context.Background(), createFollowParams)
+
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Successfully added and followed feed '%s' with url '%s'\n", createdFeed.Name, createdFeed.Url)
 		fmt.Printf("  Id        => %s\n", createdFeed.ID)
 		fmt.Printf("  UserId    => %s\n", createdFeed.UserID)
 		fmt.Printf("  CreatedAt => %s\n", createdFeed.CreatedAt)
 		fmt.Printf("  UpdatedAt => %s\n", createdFeed.UpdatedAt)
 
 		return nil
-	})
+	}))
 }
 
 func feedsCommand() *Command {
@@ -266,3 +305,115 @@ func feedsCommand() *Command {
 		return nil
 	})
 }
+
+func followCommand() *Command {
+	return newCommand("follow", requiresLoggedInUser(func(s *state.State, currentUser database.User) error {
+		if len(s.Arguments) == 0 {
+			return fmt.Errorf("Did not receive expected url argument")
+		}
+
+		feedIdentifier := s.Arguments[0]
+
+		feed, err := resolveFeedByNameOrURL(s, feedIdentifier)
+
+		if err != nil {
+			return err
+		}
+
+		createFollowParams := database.CreateFollowParams{
+			ID:        uuid.New(),
+			CreatedAt: time.Now(),
+			UpdatedAt: time.Now(),
+			FeedID:    feed.ID,
+			UserID:    currentUser.ID,
+		}
+
+		createdFollow, err := s.Database.CreateFollow(context.Background(), createFollowParams)
+
+		if err != nil {
+			return fmt.Errorf("Unable to follow feed '%s'", feed.Name)
+		}
+
+		fmt.Printf("'%s' successfully followed feed '%s'", createdFollow.UserName, createdFollow.FeedName)
+
+		return nil
+	}))
+}
+
+func followingCommand() *Command {
+	return newCommand("following", requiresLoggedInUser(func(s *state.State, currentUser database.User) error {
+		follows, err := s.Database.GetFollowsForUser(context.Background(), currentUser.ID)
+
+		if err != nil {
+			return fmt.Errorf("Unable to find follows for user")
+		}
+
+		for _, follow := range follows {
+			fmt.Printf("* %s\n", follow.FeedName)
+		}
+
+		return nil
+	}))
+}
+
+func unfollowCommand() *Command {
+	return newCommand("unfollow", requiresLoggedInUser(func(s *state.State, currentUser database.User) error {
+		if len(s.Arguments) == 0 {
+			return fmt.Errorf("Did not receive expected feed url argument")
+		}
+
+		feedIdentifier := s.Arguments[0]
+
+		feed, err := resolveFeedByNameOrURL(s, feedIdentifier)
+
+		if err != nil {
+			return err
+		}
+
+		deleteFollowParams := database.DeleteFollowForUserParams{
+			UserID: currentUser.ID,
+			FeedID: feed.ID,
+		}
+
+		err = s.Database.DeleteFollowForUser(context.Background(), deleteFollowParams)
+
+		if err != nil {
+			return fmt.Errorf("Unable to unfollow feed '%s'", feed.Name)
+		}
+
+		fmt.Printf("Successfully unfollowed feed '%s'", feed.Name)
+
+		return nil
+	}))
+}
+
+func browseCommand() *Command {
+	return newCommand("browse", requiresLoggedInUser(func(s *state.State, currentUser database.User) error {
+		limit := int32(2)
+
+		if len(s.Arguments) > 0 {
+			parsedLimit, err := strconv.ParseInt(s.Arguments[0], 10, 32)
+
+			if err == nil {
+				limit = int32(parsedLimit)
+			}
+		}
+
+		getUserPostsParams := database.GetPostsForUserParams{
+			UserID: currentUser.ID,
+			Limit:  limit,
+		}
+
+		posts, err := s.Database.GetPostsForUser(context.Background(), getUserPostsParams)
+
+		if err != nil {
+			return err
+		}
+
+		for _, post := range posts {
+			fmt.Printf("* %s - %s\n", post.FeedName, post.Title)
+		}
+
+		return nil
+	}))
+}

internal/command/helpers.go

@@ -0,0 +1,107 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"strings"
+
+	"github.com/StevanFreeborn/gator/internal/database"
+	"github.com/StevanFreeborn/gator/internal/state"
+	"golang.org/x/term"
+)
+
+func normalizeFeedURL(raw string) (string, error) {
+	u, err := url.Parse(raw)
+
+	if err != nil {
+		return "", err
+	}
+
+	u.Scheme = strings.ToLower(u.Scheme)
+	u.Host = strings.ToLower(u.Host)
+
+	if (u.Scheme == "http" && u.Port() == "80") || (u.Scheme == "https" && u.Port() == "443") {
+		u.Host = u.Hostname()
+	}
+
+	if len(u.Path) > 1 {
+		u.Path = strings.TrimSuffix(u.Path, "/")
+	}
+
+	u.Fragment = ""
+
+	return u.String(), nil
+}
+
+func resolveFeedByNameOrURL(s *state.State, identifier string) (database.Feed, error) {
+	if strings.TrimSpace(identifier) == "" {
+		return database.Feed{}, fmt.Errorf("Feed identifier cannot be empty")
+	}
+
+	feeds, err := s.Database.GetAllFeeds(context.Background())
+
+	if err != nil {
+		return database.Feed{}, fmt.Errorf("Failed to load feeds")
+	}
+
+	var matches []database.Feed
+
+	for _, feed := range feeds {
+		if strings.EqualFold(feed.Name, identifier) {
+			matches = append(matches, feed)
+		}
+	}
+
+	if len(matches) == 1 {
+		return matches[0], nil
+	}
+
+	if len(matches) > 1 {
+		return database.Feed{}, fmt.Errorf("Multiple feeds match name '%s'; use a url instead", identifier)
+	}
+
+	normalizedURL, err := normalizeFeedURL(identifier)
+
+	if err != nil {
+		return database.Feed{}, fmt.Errorf("No feed found matching name or url '%s'", identifier)
+	}
+
+	feed, err := s.Database.GetFeedByUrl(context.Background(), normalizedURL)
+
+	if err != nil {
+		return database.Feed{}, fmt.Errorf("No feed found matching name or url '%s'", identifier)
+	}
+
+	return feed, nil
+}
+
+func enableKeypressExit(cancel context.CancelFunc) (func(), error) {
+	oldState, err := term.MakeRaw(int(os.Stdin.Fd()))
+
+	if err != nil {
+		return nil, fmt.Errorf("Failed to enable raw mode: %v", err)
+	}
+
+	go func() {
+		buf := make([]byte, 1)
+
+		for {
+			n, err := os.Stdin.Read(buf)
+
+			if err != nil || n == 0 {
+				return
+			}
+
+			if buf[0] == 'q' || buf[0] == 'Q' || buf[0] == 3 {
+				cancel()
+				return
+			}
+		}
+	}()
+
+	return func() {
+		term.Restore(int(os.Stdin.Fd()), oldState)
+	}, nil
+}

internal/command/middleware.go

@@ -0,0 +1,20 @@
+package command
+
+import (
+	"context"
+
+	"github.com/StevanFreeborn/gator/internal/database"
+	"github.com/StevanFreeborn/gator/internal/state"
+)
+
+func requiresLoggedInUser(handler func(*state.State, database.User) error) CommandHandler {
+	return func(s *state.State) error {
+		loggedInUser, err := s.GetCurrentUser(context.Background())
+
+		if err != nil {
+			return err
+		}
+
+		return handler(s, loggedInUser)
+	}
+}

internal/database/feeds.sql.go

@@ -7,31 +7,34 @@ package database
 
 import (
 	"context"
+	"database/sql"
 	"time"
 
 	"github.com/google/uuid"
 )
 
 const createFeed = `-- name: CreateFeed :one
-INSERT INTO feeds (id, created_at, updated_at, name, url, user_id)
+INSERT INTO feeds (id, created_at, updated_at, name, url, user_id, last_fetched_at)
 VALUES (
   $1,
   $2,
   $3,
   $4,
   $5,
-  $6
+  $6,
+  $7
 )
-RETURNING id, created_at, updated_at, name, url, user_id
+RETURNING id, created_at, updated_at, name, url, user_id, last_fetched_at
 `
 
 type CreateFeedParams struct {
-	ID        uuid.UUID
+	ID            uuid.UUID
-	CreatedAt time.Time
+	CreatedAt     time.Time
-	UpdatedAt time.Time
+	UpdatedAt     time.Time
-	Name      string
+	Name          string
-	Url       string
+	Url           string
-	UserID    uuid.UUID
+	UserID        uuid.UUID
+	LastFetchedAt sql.NullTime
 }
 
 func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, error) {
@@ -42,6 +45,7 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e
 		arg.Name,
 		arg.Url,
 		arg.UserID,
+		arg.LastFetchedAt,
 	)
 	var i Feed
 	err := row.Scan(
@@ -51,12 +55,13 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e
 		&i.Name,
 		&i.Url,
 		&i.UserID,
+		&i.LastFetchedAt,
 	)
 	return i, err
 }
 
 const getAllFeeds = `-- name: GetAllFeeds :many
-SELECT id, created_at, updated_at, name, url, user_id FROM feeds
+SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
 `
 
 func (q *Queries) GetAllFeeds(ctx context.Context) ([]Feed, error) {
@@ -75,6 +80,7 @@ func (q *Queries) GetAllFeeds(ctx context.Context) ([]Feed, error) {
 			&i.Name,
 			&i.Url,
 			&i.UserID,
+			&i.LastFetchedAt,
 		); err != nil {
 			return nil, err
 		}
@@ -88,3 +94,59 @@ func (q *Queries) GetAllFeeds(ctx context.Context) ([]Feed, error) {
 	}
 	return items, nil
 }
+
+const getFeedByUrl = `-- name: GetFeedByUrl :one
+SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
+WHERE url = $1
+`
+
+func (q *Queries) GetFeedByUrl(ctx context.Context, url string) (Feed, error) {
+	row := q.db.QueryRowContext(ctx, getFeedByUrl, url)
+	var i Feed
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.Name,
+		&i.Url,
+		&i.UserID,
+		&i.LastFetchedAt,
+	)
+	return i, err
+}
+
+const getNextFeedToFetch = `-- name: GetNextFeedToFetch :one
+SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
+ORDER BY last_fetched_at ASC NULLS FIRST
+`
+
+func (q *Queries) GetNextFeedToFetch(ctx context.Context) (Feed, error) {
+	row := q.db.QueryRowContext(ctx, getNextFeedToFetch)
+	var i Feed
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.Name,
+		&i.Url,
+		&i.UserID,
+		&i.LastFetchedAt,
+	)
+	return i, err
+}
+
+const markFeedAsFetched = `-- name: MarkFeedAsFetched :exec
+UPDATE feeds
+SET last_fetched_at = $1
+WHERE feeds.id = $2
+`
+
+type MarkFeedAsFetchedParams struct {
+	LastFetchedAt sql.NullTime
+	ID            uuid.UUID
+}
+
+func (q *Queries) MarkFeedAsFetched(ctx context.Context, arg MarkFeedAsFetchedParams) error {
+	_, err := q.db.ExecContext(ctx, markFeedAsFetched, arg.LastFetchedAt, arg.ID)
+	return err
+}

internal/database/follows.sql.go

@@ -0,0 +1,135 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.31.1
+// source: follows.sql
+
+package database
+
+import (
+	"context"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+const createFollow = `-- name: CreateFollow :one
+WITH inserted_follow as (
+  INSERT INTO follows (id, created_at, updated_at, feed_id, user_id)
+  VALUES (
+    $1,
+    $2,
+    $3,
+    $4,
+    $5
+  )
+  RETURNING id, created_at, updated_at, user_id, feed_id
+)
+SELECT inserted_follow.id, inserted_follow.created_at, inserted_follow.updated_at, inserted_follow.user_id, inserted_follow.feed_id,
+  feeds.name as feed_name,
+  users.name as user_name
+FROM inserted_follow
+INNER JOIN users
+  ON inserted_follow.user_id = users.id
+INNER JOIN feeds
+  ON inserted_follow.feed_id = feeds.id
+`
+
+type CreateFollowParams struct {
+	ID        uuid.UUID
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	FeedID    uuid.UUID
+	UserID    uuid.UUID
+}
+
+type CreateFollowRow struct {
+	ID        uuid.UUID
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	UserID    uuid.UUID
+	FeedID    uuid.UUID
+	FeedName  string
+	UserName  string
+}
+
+func (q *Queries) CreateFollow(ctx context.Context, arg CreateFollowParams) (CreateFollowRow, error) {
+	row := q.db.QueryRowContext(ctx, createFollow,
+		arg.ID,
+		arg.CreatedAt,
+		arg.UpdatedAt,
+		arg.FeedID,
+		arg.UserID,
+	)
+	var i CreateFollowRow
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.UserID,
+		&i.FeedID,
+		&i.FeedName,
+		&i.UserName,
+	)
+	return i, err
+}
+
+const deleteFollowForUser = `-- name: DeleteFollowForUser :exec
+DELETE FROM follows
+WHERE feed_id = $1 AND user_id = $2
+`
+
+type DeleteFollowForUserParams struct {
+	FeedID uuid.UUID
+	UserID uuid.UUID
+}
+
+func (q *Queries) DeleteFollowForUser(ctx context.Context, arg DeleteFollowForUserParams) error {
+	_, err := q.db.ExecContext(ctx, deleteFollowForUser, arg.FeedID, arg.UserID)
+	return err
+}
+
+const getFollowsForUser = `-- name: GetFollowsForUser :many
+SELECT follows.id, follows.created_at, follows.updated_at, follows.user_id, follows.feed_id, feeds.name as feed_name FROM follows
+INNER JOIN feeds
+  ON follows.feed_id = feeds.id
+WHERE follows.user_id = $1
+`
+
+type GetFollowsForUserRow struct {
+	ID        uuid.UUID
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	UserID    uuid.UUID
+	FeedID    uuid.UUID
+	FeedName  string
+}
+
+func (q *Queries) GetFollowsForUser(ctx context.Context, userID uuid.UUID) ([]GetFollowsForUserRow, error) {
+	rows, err := q.db.QueryContext(ctx, getFollowsForUser, userID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []GetFollowsForUserRow
+	for rows.Next() {
+		var i GetFollowsForUserRow
+		if err := rows.Scan(
+			&i.ID,
+			&i.CreatedAt,
+			&i.UpdatedAt,
+			&i.UserID,
+			&i.FeedID,
+			&i.FeedName,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}

internal/database/models.go

@@ -5,18 +5,39 @@
 package database
 
 import (
+	"database/sql"
 	"time"
 
 	"github.com/google/uuid"
 )
 
 type Feed struct {
+	ID            uuid.UUID
+	CreatedAt     time.Time
+	UpdatedAt     time.Time
+	Name          string
+	Url           string
+	UserID        uuid.UUID
+	LastFetchedAt sql.NullTime
+}
+
+type Follow struct {
 	ID        uuid.UUID
 	CreatedAt time.Time
 	UpdatedAt time.Time
-	Name      string
-	Url       string
 	UserID    uuid.UUID
+	FeedID    uuid.UUID
+}
+
+type Post struct {
+	ID          uuid.UUID
+	CreatedAt   time.Time
+	UpdatedAt   time.Time
+	Title       string
+	Url         string
+	Description string
+	PublishedAt sql.NullTime
+	FeedID      uuid.UUID
 }
 
 type User struct {

internal/database/posts.sql.go

@@ -0,0 +1,136 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.31.1
+// source: posts.sql
+
+package database
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+const createPost = `-- name: CreatePost :one
+INSERT INTO posts (id, created_at, updated_at, title, url, description, published_at, feed_id)
+VALUES (
+  $1,
+  $2,
+  $3,
+  $4,
+  $5,
+  $6,
+  $7,
+  $8
+)
+RETURNING id, created_at, updated_at, title, url, description, published_at, feed_id
+`
+
+type CreatePostParams struct {
+	ID          uuid.UUID
+	CreatedAt   time.Time
+	UpdatedAt   time.Time
+	Title       string
+	Url         string
+	Description string
+	PublishedAt sql.NullTime
+	FeedID      uuid.UUID
+}
+
+func (q *Queries) CreatePost(ctx context.Context, arg CreatePostParams) (Post, error) {
+	row := q.db.QueryRowContext(ctx, createPost,
+		arg.ID,
+		arg.CreatedAt,
+		arg.UpdatedAt,
+		arg.Title,
+		arg.Url,
+		arg.Description,
+		arg.PublishedAt,
+		arg.FeedID,
+	)
+	var i Post
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.Title,
+		&i.Url,
+		&i.Description,
+		&i.PublishedAt,
+		&i.FeedID,
+	)
+	return i, err
+}
+
+const getPostsForUser = `-- name: GetPostsForUser :many
+SELECT 
+  posts.id, 
+  posts.created_at, 
+  posts.updated_at, 
+  posts.title, 
+  posts.url, 
+  posts.description, 
+  posts.published_at, 
+  posts.feed_id,
+  feeds.name as feed_name
+FROM posts
+INNER JOIN feeds
+  ON posts.feed_id = feeds.id
+INNER JOIN follows
+  ON posts.feed_id = follows.feed_id
+WHERE follows.user_id = $1
+ORDER BY posts.published_at DESC, posts.title ASC
+LIMIT $2
+`
+
+type GetPostsForUserParams struct {
+	UserID uuid.UUID
+	Limit  int32
+}
+
+type GetPostsForUserRow struct {
+	ID          uuid.UUID
+	CreatedAt   time.Time
+	UpdatedAt   time.Time
+	Title       string
+	Url         string
+	Description string
+	PublishedAt sql.NullTime
+	FeedID      uuid.UUID
+	FeedName    string
+}
+
+func (q *Queries) GetPostsForUser(ctx context.Context, arg GetPostsForUserParams) ([]GetPostsForUserRow, error) {
+	rows, err := q.db.QueryContext(ctx, getPostsForUser, arg.UserID, arg.Limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []GetPostsForUserRow
+	for rows.Next() {
+		var i GetPostsForUserRow
+		if err := rows.Scan(
+			&i.ID,
+			&i.CreatedAt,
+			&i.UpdatedAt,
+			&i.Title,
+			&i.Url,
+			&i.Description,
+			&i.PublishedAt,
+			&i.FeedID,
+			&i.FeedName,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}

internal/rss/rss.go

@@ -2,9 +2,16 @@ package rss
 
 import (
 	"context"
+	"database/sql"
 	"encoding/xml"
+	"fmt"
 	"html"
 	"net/http"
+	"time"
+
+	"github.com/StevanFreeborn/gator/internal/database"
+	"github.com/StevanFreeborn/gator/internal/state"
+	"github.com/google/uuid"
 )
 
 type RSSItem struct {
@@ -25,7 +32,7 @@ type RSSFeed struct {
 	Channel RSSChannel `xml:"channel"`
 }
 
-func FetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
+func fetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
 	var rssFeed RSSFeed
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
@@ -59,3 +66,63 @@ func FetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
 
 	return &rssFeed, nil
 }
+
+func ScrapeNextFeed(ctx context.Context, s *state.State) {
+	nextFeed, err := s.Database.GetNextFeedToFetch(ctx)
+
+	if err != nil {
+		fmt.Print("Failed to lookup next feed to fetch\r\n")
+		return
+	}
+
+	fmt.Printf("Fetching feed '%s'\r\n", nextFeed.Name)
+
+	fetchedFeed, err := fetchFeed(ctx, nextFeed.Url)
+
+	if err != nil {
+		fmt.Printf("Failed to fetch feed '%s'\r\n", nextFeed.Name)
+		return
+	}
+
+	markFeedParams := database.MarkFeedAsFetchedParams{
+		ID: nextFeed.ID,
+		LastFetchedAt: sql.NullTime{
+			Time:  time.Now(),
+			Valid: true,
+		},
+	}
+
+	err = s.Database.MarkFeedAsFetched(ctx, markFeedParams)
+
+	for _, item := range fetchedFeed.Channel.Item {
+		publishedAt, err := time.Parse(time.RFC3339, item.PubDate)
+		publishedAtValue := sql.NullTime{
+			Time:  publishedAt,
+			Valid: err == nil,
+		}
+
+		createPostParams := database.CreatePostParams{
+			ID:          uuid.New(),
+			CreatedAt:   time.Now(),
+			UpdatedAt:   time.Now(),
+			Title:       item.Title,
+			Url:         item.Link,
+			Description: item.Description,
+			PublishedAt: publishedAtValue,
+			FeedID:      nextFeed.ID,
+		}
+
+		_, err = s.Database.CreatePost(ctx, createPostParams)
+
+		if err != nil {
+			fmt.Printf("  Skipping saving post '%s'\r\n", item.Title)
+			continue
+		}
+
+		fmt.Printf("  Saved post '%s'\r\n", item.Title)
+	}
+
+	if err != nil {
+		fmt.Printf("Failed to mark feed as fetched.\r\n")
+	}
+}

sql/queries/feeds.sql

@@ -1,14 +1,28 @@
 -- name: CreateFeed :one
-INSERT INTO feeds (id, created_at, updated_at, name, url, user_id)
+INSERT INTO feeds (id, created_at, updated_at, name, url, user_id, last_fetched_at)
 VALUES (
   $1,
   $2,
   $3,
   $4,
   $5,
-  $6
+  $6,
+  $7
 )
 RETURNING *;
 
 -- name: GetAllFeeds :many
-SELECT id, created_at, updated_at, name, url, user_id FROM feeds;
+SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds;
+
+-- name: GetFeedByUrl :one
+SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
+WHERE url = $1;
+
+-- name: GetNextFeedToFetch :one
+SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
+ORDER BY last_fetched_at ASC NULLS FIRST;
+
+-- name: MarkFeedAsFetched :exec
+UPDATE feeds
+SET last_fetched_at = $1
+WHERE feeds.id = $2;

sql/queries/follows.sql

@@ -0,0 +1,30 @@
+-- name: CreateFollow :one
+WITH inserted_follow as (
+  INSERT INTO follows (id, created_at, updated_at, feed_id, user_id)
+  VALUES (
+    $1,
+    $2,
+    $3,
+    $4,
+    $5
+  )
+  RETURNING *
+)
+SELECT inserted_follow.*,
+  feeds.name as feed_name,
+  users.name as user_name
+FROM inserted_follow
+INNER JOIN users
+  ON inserted_follow.user_id = users.id
+INNER JOIN feeds
+  ON inserted_follow.feed_id = feeds.id;
+
+-- name: GetFollowsForUser :many
+SELECT follows.*, feeds.name as feed_name FROM follows
+INNER JOIN feeds
+  ON follows.feed_id = feeds.id
+WHERE follows.user_id = $1;
+
+-- name: DeleteFollowForUser :exec
+DELETE FROM follows
+WHERE feed_id = $1 AND user_id = $2;

sql/queries/posts.sql

@@ -0,0 +1,33 @@
+-- name: CreatePost :one
+INSERT INTO posts (id, created_at, updated_at, title, url, description, published_at, feed_id)
+VALUES (
+  $1,
+  $2,
+  $3,
+  $4,
+  $5,
+  $6,
+  $7,
+  $8
+)
+RETURNING *;
+
+-- name: GetPostsForUser :many
+SELECT 
+  posts.id, 
+  posts.created_at, 
+  posts.updated_at, 
+  posts.title, 
+  posts.url, 
+  posts.description, 
+  posts.published_at, 
+  posts.feed_id,
+  feeds.name as feed_name
+FROM posts
+INNER JOIN feeds
+  ON posts.feed_id = feeds.id
+INNER JOIN follows
+  ON posts.feed_id = follows.feed_id
+WHERE follows.user_id = $1
+ORDER BY posts.published_at DESC, posts.title ASC
+LIMIT $2;

sql/schema/003_follows.sql

@@ -0,0 +1,14 @@
+-- +goose Up
+CREATE TABLE follows (
+  id UUID PRIMARY KEY,
+  created_at TIMESTAMP NOT NULL,
+  updated_at TIMESTAMP NOT NULL,
+  user_id UUID NOT NULL,
+  feed_id UUID NOT NULL,
+  FOREIGN KEY(user_id) REFERENCES users(id) ON DELETE CASCADE,
+  FOREIGN KEY(feed_id) REFERENCES feeds(id) ON DELETE CASCADE,
+  CONSTRAINT unique_user_feed UNIQUE(user_id, feed_id)
+);
+
+-- +goose Down
+DROP TABLE follows;

sql/schema/004_lastfetchedat.sql

@@ -0,0 +1,8 @@
+-- +goose Up
+ALTER TABLE feeds
+ADD COLUMN last_fetched_at TIMESTAMP NULL;
+
+-- +goose Down
+ALTER TABLE feeds
+DROP COLUMN last_fetched_at;
+

sql/schema/005_posts.sql

@@ -0,0 +1,16 @@
+-- +goose Up
+CREATE TABLE posts (
+  id UUID PRIMARY KEY,
+  created_at TIMESTAMP NOT NULL,
+  updated_at TIMESTAMP NOT NULL,
+  title TEXT NOT NULL,
+  url TEXT NOT NULL,
+  description TEXT NOT NULL,
+  published_at TIMESTAMP NULL,
+  feed_id UUID NOT NULL,
+  FOREIGN KEY(feed_id) REFERENCES feeds(id) ON DELETE CASCADE,
+  CONSTRAINT unique_url UNIQUE(url)
+);
+
+-- +goose Down
+DROP TABLE posts;