diff --git a/.bin/gator b/.bin/gator new file mode 100755 index 0000000..ff86c9e Binary files /dev/null and b/.bin/gator differ diff --git a/go.mod b/go.mod index 9c3288d..5678d1c 100644 --- a/go.mod +++ b/go.mod @@ -5,4 +5,6 @@ go 1.26.3 require ( github.com/google/uuid v1.6.0 // indirect github.com/lib/pq v1.12.3 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/term v0.45.0 // indirect ) diff --git a/go.sum b/go.sum index 9e3584b..652ce08 100644 --- a/go.sum +++ b/go.sum @@ -2,3 +2,7 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= diff --git a/internal/command/command.go b/internal/command/command.go index e2187a1..cbf8e50 100644 --- a/internal/command/command.go +++ b/internal/command/command.go @@ -5,14 +5,19 @@ import ( "context" "fmt" "net/url" + "os" + "os/signal" "slices" + "strconv" "strings" + "syscall" "time" "github.com/StevanFreeborn/gator/internal/database" "github.com/StevanFreeborn/gator/internal/rss" "github.com/StevanFreeborn/gator/internal/state" "github.com/google/uuid" + "golang.org/x/term" ) type Command struct { @@ -54,6 +59,7 @@ func NewRegistry() *CommandRegistry { followCommand(), followingCommand(), unfollowCommand(), + browseCommand(), } for _, cmd := range commands { @@ -172,25 +178,60 @@ func usersCommand() *Command { func aggCommand() *Command { return newCommand("agg", func(s *state.State) error { - // if len(s.Arguments) == 0 { - // return fmt.Errorf("Did not receive expected feed argument") - // } - - // feed := s.Arguments[0] - // - // validUrl, err := url.Parse(feed) - // - // if err != nil { - // return fmt.Errorf("Feed '%s' is not a valid url", feed) - // } - - rssFeed, err := rss.FetchFeed(context.Background(), "https://www.wagslane.dev/index.xml") - - if err != nil { - return err + if len(s.Arguments) == 0 { + return fmt.Errorf("Did not receive expected time between requests argument") } - fmt.Printf("%v\n", rssFeed) + timeBetweenRequests := s.Arguments[0] + validDuration, err := time.ParseDuration(timeBetweenRequests) + + if err != nil { + return fmt.Errorf("Time between requests argument '%s' not valid duration string", timeBetweenRequests) + } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + oldState, err := term.MakeRaw(int(os.Stdin.Fd())) + + if err != nil { + return fmt.Errorf("Failed to enable raw mode: %v", err) + } + + defer term.Restore(int(os.Stdin.Fd()), oldState) + + go func() { + buf := make([]byte, 1) + + for { + n, err := os.Stdin.Read(buf) + + if err != nil || n == 0 { + return + } + + if buf[0] == 'q' || buf[0] == 'Q' || buf[0] == 3 { + cancel() + return + } + } + }() + + ticker := time.NewTicker(validDuration) + defer ticker.Stop() + + rss.ScrapeNextFeed(ctx, s) + + fetchFeeds: + for { + select { + case <-ctx.Done(): + fmt.Print("Stopping feed agg command\r\n") + break fetchFeeds + case <-ticker.C: + rss.ScrapeNextFeed(ctx, s) + } + } return nil }) @@ -321,7 +362,7 @@ func followingCommand() *Command { } for _, follow := range follows { - fmt.Printf("* %s", follow.FeedName) + fmt.Printf("* %s\n", follow.FeedName) } return nil @@ -358,3 +399,34 @@ func unfollowCommand() *Command { return nil })) } + +func browseCommand() *Command { + return newCommand("browse", requiresLoggedInUser(func(s *state.State, currentUser database.User) error { + limit := int32(2) + + if len(s.Arguments) > 0 { + parsedLimit, err := strconv.ParseInt(s.Arguments[0], 10, 32) + + if err == nil { + limit = int32(parsedLimit) + } + } + + getUserPostsParams := database.GetPostsForUserParams{ + UserID: currentUser.ID, + Limit: limit, + } + + posts, err := s.Database.GetPostsForUser(context.Background(), getUserPostsParams) + + if err != nil { + return err + } + + for _, post := range posts { + fmt.Printf("* %s - %s\n", post.FeedName, post.Title) + } + + return nil + })) +} diff --git a/internal/database/feeds.sql.go b/internal/database/feeds.sql.go index a964550..3f7b83a 100644 --- a/internal/database/feeds.sql.go +++ b/internal/database/feeds.sql.go @@ -7,31 +7,34 @@ package database import ( "context" + "database/sql" "time" "github.com/google/uuid" ) const createFeed = `-- name: CreateFeed :one -INSERT INTO feeds (id, created_at, updated_at, name, url, user_id) +INSERT INTO feeds (id, created_at, updated_at, name, url, user_id, last_fetched_at) VALUES ( $1, $2, $3, $4, $5, - $6 + $6, + $7 ) -RETURNING id, created_at, updated_at, name, url, user_id +RETURNING id, created_at, updated_at, name, url, user_id, last_fetched_at ` type CreateFeedParams struct { - ID uuid.UUID - CreatedAt time.Time - UpdatedAt time.Time - Name string - Url string - UserID uuid.UUID + ID uuid.UUID + CreatedAt time.Time + UpdatedAt time.Time + Name string + Url string + UserID uuid.UUID + LastFetchedAt sql.NullTime } func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, error) { @@ -42,6 +45,7 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e arg.Name, arg.Url, arg.UserID, + arg.LastFetchedAt, ) var i Feed err := row.Scan( @@ -51,12 +55,13 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e &i.Name, &i.Url, &i.UserID, + &i.LastFetchedAt, ) return i, err } const getAllFeeds = `-- name: GetAllFeeds :many -SELECT id, created_at, updated_at, name, url, user_id FROM feeds +SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds ` func (q *Queries) GetAllFeeds(ctx context.Context) ([]Feed, error) { @@ -75,6 +80,7 @@ func (q *Queries) GetAllFeeds(ctx context.Context) ([]Feed, error) { &i.Name, &i.Url, &i.UserID, + &i.LastFetchedAt, ); err != nil { return nil, err } @@ -90,7 +96,7 @@ func (q *Queries) GetAllFeeds(ctx context.Context) ([]Feed, error) { } const getFeedByUrl = `-- name: GetFeedByUrl :one -SELECT id, created_at, updated_at, name, url, user_id FROM feeds +SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds WHERE url = $1 ` @@ -104,6 +110,43 @@ func (q *Queries) GetFeedByUrl(ctx context.Context, url string) (Feed, error) { &i.Name, &i.Url, &i.UserID, + &i.LastFetchedAt, ) return i, err } + +const getNextFeedToFetch = `-- name: GetNextFeedToFetch :one +SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds +ORDER BY last_fetched_at ASC NULLS FIRST +` + +func (q *Queries) GetNextFeedToFetch(ctx context.Context) (Feed, error) { + row := q.db.QueryRowContext(ctx, getNextFeedToFetch) + var i Feed + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.Name, + &i.Url, + &i.UserID, + &i.LastFetchedAt, + ) + return i, err +} + +const markFeedAsFetched = `-- name: MarkFeedAsFetched :exec +UPDATE feeds +SET last_fetched_at = $1 +WHERE feeds.id = $2 +` + +type MarkFeedAsFetchedParams struct { + LastFetchedAt sql.NullTime + ID uuid.UUID +} + +func (q *Queries) MarkFeedAsFetched(ctx context.Context, arg MarkFeedAsFetchedParams) error { + _, err := q.db.ExecContext(ctx, markFeedAsFetched, arg.LastFetchedAt, arg.ID) + return err +} diff --git a/internal/database/models.go b/internal/database/models.go index 613a51f..28e8bd9 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -5,18 +5,20 @@ package database import ( + "database/sql" "time" "github.com/google/uuid" ) type Feed struct { - ID uuid.UUID - CreatedAt time.Time - UpdatedAt time.Time - Name string - Url string - UserID uuid.UUID + ID uuid.UUID + CreatedAt time.Time + UpdatedAt time.Time + Name string + Url string + UserID uuid.UUID + LastFetchedAt sql.NullTime } type Follow struct { @@ -27,6 +29,17 @@ type Follow struct { FeedID uuid.UUID } +type Post struct { + ID uuid.UUID + CreatedAt time.Time + UpdatedAt time.Time + Title string + Url string + Description string + PublishedAt sql.NullTime + FeedID uuid.UUID +} + type User struct { ID uuid.UUID CreatedAt time.Time diff --git a/internal/database/posts.sql.go b/internal/database/posts.sql.go new file mode 100644 index 0000000..6eb7679 --- /dev/null +++ b/internal/database/posts.sql.go @@ -0,0 +1,136 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: posts.sql + +package database + +import ( + "context" + "database/sql" + "time" + + "github.com/google/uuid" +) + +const createPost = `-- name: CreatePost :one +INSERT INTO posts (id, created_at, updated_at, title, url, description, published_at, feed_id) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8 +) +RETURNING id, created_at, updated_at, title, url, description, published_at, feed_id +` + +type CreatePostParams struct { + ID uuid.UUID + CreatedAt time.Time + UpdatedAt time.Time + Title string + Url string + Description string + PublishedAt sql.NullTime + FeedID uuid.UUID +} + +func (q *Queries) CreatePost(ctx context.Context, arg CreatePostParams) (Post, error) { + row := q.db.QueryRowContext(ctx, createPost, + arg.ID, + arg.CreatedAt, + arg.UpdatedAt, + arg.Title, + arg.Url, + arg.Description, + arg.PublishedAt, + arg.FeedID, + ) + var i Post + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.Title, + &i.Url, + &i.Description, + &i.PublishedAt, + &i.FeedID, + ) + return i, err +} + +const getPostsForUser = `-- name: GetPostsForUser :many +SELECT + posts.id, + posts.created_at, + posts.updated_at, + posts.title, + posts.url, + posts.description, + posts.published_at, + posts.feed_id, + feeds.name as feed_name +FROM posts +INNER JOIN feeds + ON posts.feed_id = feeds.id +INNER JOIN follows + ON posts.feed_id = follows.feed_id +WHERE follows.user_id = $1 +ORDER BY posts.published_at DESC, posts.title ASC +LIMIT $2 +` + +type GetPostsForUserParams struct { + UserID uuid.UUID + Limit int32 +} + +type GetPostsForUserRow struct { + ID uuid.UUID + CreatedAt time.Time + UpdatedAt time.Time + Title string + Url string + Description string + PublishedAt sql.NullTime + FeedID uuid.UUID + FeedName string +} + +func (q *Queries) GetPostsForUser(ctx context.Context, arg GetPostsForUserParams) ([]GetPostsForUserRow, error) { + rows, err := q.db.QueryContext(ctx, getPostsForUser, arg.UserID, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetPostsForUserRow + for rows.Next() { + var i GetPostsForUserRow + if err := rows.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.Title, + &i.Url, + &i.Description, + &i.PublishedAt, + &i.FeedID, + &i.FeedName, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/rss/rss.go b/internal/rss/rss.go index 73f9d31..c342a4d 100644 --- a/internal/rss/rss.go +++ b/internal/rss/rss.go @@ -2,9 +2,16 @@ package rss import ( "context" + "database/sql" "encoding/xml" + "fmt" "html" "net/http" + "time" + + "github.com/StevanFreeborn/gator/internal/database" + "github.com/StevanFreeborn/gator/internal/state" + "github.com/google/uuid" ) type RSSItem struct { @@ -25,7 +32,7 @@ type RSSFeed struct { Channel RSSChannel `xml:"channel"` } -func FetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) { +func fetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) { var rssFeed RSSFeed req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil) @@ -59,3 +66,63 @@ func FetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) { return &rssFeed, nil } + +func ScrapeNextFeed(ctx context.Context, s *state.State) { + nextFeed, err := s.Database.GetNextFeedToFetch(ctx) + + if err != nil { + fmt.Print("Failed to lookup next feed to fetch\r\n") + return + } + + fmt.Printf("Fetching feed '%s'\r\n", nextFeed.Name) + + fetchedFeed, err := fetchFeed(ctx, nextFeed.Url) + + if err != nil { + fmt.Printf("Failed to fetch feed '%s'\r\n", nextFeed.Name) + return + } + + markFeedParams := database.MarkFeedAsFetchedParams{ + ID: nextFeed.ID, + LastFetchedAt: sql.NullTime{ + Time: time.Now(), + Valid: true, + }, + } + + err = s.Database.MarkFeedAsFetched(ctx, markFeedParams) + + for _, item := range fetchedFeed.Channel.Item { + publishedAt, err := time.Parse(time.RFC3339, item.PubDate) + publishedAtValue := sql.NullTime{ + Time: publishedAt, + Valid: err == nil, + } + + createPostParams := database.CreatePostParams{ + ID: uuid.New(), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + Title: item.Title, + Url: item.Link, + Description: item.Description, + PublishedAt: publishedAtValue, + FeedID: nextFeed.ID, + } + + _, err = s.Database.CreatePost(ctx, createPostParams) + + if err != nil { + fmt.Printf(" Skipping saving post '%s'\r\n", item.Title) + continue + } + + fmt.Printf(" Saved post '%s'\r\n", item.Title) + } + + if err != nil { + fmt.Printf("Failed to mark feed as fetched.\r\n") + } +} diff --git a/sql/queries/feeds.sql b/sql/queries/feeds.sql index 59f4032..a130cbf 100644 --- a/sql/queries/feeds.sql +++ b/sql/queries/feeds.sql @@ -1,18 +1,28 @@ -- name: CreateFeed :one -INSERT INTO feeds (id, created_at, updated_at, name, url, user_id) +INSERT INTO feeds (id, created_at, updated_at, name, url, user_id, last_fetched_at) VALUES ( $1, $2, $3, $4, $5, - $6 + $6, + $7 ) RETURNING *; -- name: GetAllFeeds :many -SELECT id, created_at, updated_at, name, url, user_id FROM feeds; +SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds; -- name: GetFeedByUrl :one -SELECT id, created_at, updated_at, name, url, user_id FROM feeds +SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds WHERE url = $1; + +-- name: GetNextFeedToFetch :one +SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds +ORDER BY last_fetched_at ASC NULLS FIRST; + +-- name: MarkFeedAsFetched :exec +UPDATE feeds +SET last_fetched_at = $1 +WHERE feeds.id = $2; diff --git a/sql/queries/posts.sql b/sql/queries/posts.sql new file mode 100644 index 0000000..87fd158 --- /dev/null +++ b/sql/queries/posts.sql @@ -0,0 +1,33 @@ +-- name: CreatePost :one +INSERT INTO posts (id, created_at, updated_at, title, url, description, published_at, feed_id) +VALUES ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8 +) +RETURNING *; + +-- name: GetPostsForUser :many +SELECT + posts.id, + posts.created_at, + posts.updated_at, + posts.title, + posts.url, + posts.description, + posts.published_at, + posts.feed_id, + feeds.name as feed_name +FROM posts +INNER JOIN feeds + ON posts.feed_id = feeds.id +INNER JOIN follows + ON posts.feed_id = follows.feed_id +WHERE follows.user_id = $1 +ORDER BY posts.published_at DESC, posts.title ASC +LIMIT $2; diff --git a/sql/schema/004_lastfetchedat.sql b/sql/schema/004_lastfetchedat.sql new file mode 100644 index 0000000..74bdeed --- /dev/null +++ b/sql/schema/004_lastfetchedat.sql @@ -0,0 +1,8 @@ +-- +goose Up +ALTER TABLE feeds +ADD COLUMN last_fetched_at TIMESTAMP NULL; + +-- +goose Down +ALTER TABLE feeds +DROP COLUMN last_fetched_at; + diff --git a/sql/schema/005_posts.sql b/sql/schema/005_posts.sql new file mode 100644 index 0000000..641b05e --- /dev/null +++ b/sql/schema/005_posts.sql @@ -0,0 +1,16 @@ +-- +goose Up +CREATE TABLE posts ( + id UUID PRIMARY KEY, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + title TEXT NOT NULL, + url TEXT NOT NULL, + description TEXT NOT NULL, + published_at TIMESTAMP NULL, + feed_id UUID NOT NULL, + FOREIGN KEY(feed_id) REFERENCES feeds(id) ON DELETE CASCADE, + CONSTRAINT unique_url UNIQUE(url) +); + +-- +goose Down +DROP TABLE posts;