Stevan/gator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement feed agg and browse

Browse Source
This commit is contained in:
Stevan Freeborn
2026-08-03 07:14:33 -05:00
parent f51b1b941a
commit 667f5f112b
12 changed files with 444 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bin/gator
Executable
BIN
View File
Binary file not shown.
go.mod
+2
View File
@@ -5,4 +5,6 @@ go 1.26.3
require (
github.com/google/uuid v1.6.0 // indirect
github.com/lib/pq v1.12.3 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/term v0.45.0 // indirect
)
go.sum
+4
View File
@@ -2,3 +2,7 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
internal/command/command.go
+90 -18
View File
@@ -5,14 +5,19 @@ import (
"context"
"fmt"
"net/url"
"os"
"os/signal"
"slices"
"strconv"
"strings"
"syscall"
"time"
"github.com/StevanFreeborn/gator/internal/database"
"github.com/StevanFreeborn/gator/internal/rss"
"github.com/StevanFreeborn/gator/internal/state"
"github.com/google/uuid"
"golang.org/x/term"
)
type Command struct {
@@ -54,6 +59,7 @@ func NewRegistry() *CommandRegistry {
followCommand(),
followingCommand(),
unfollowCommand(),
browseCommand(),
}
for _, cmd := range commands {
@@ -172,25 +178,60 @@ func usersCommand() *Command {
func aggCommand() *Command {
return newCommand("agg", func(s *state.State) error {
// if len(s.Arguments) == 0 {
// return fmt.Errorf("Did not receive expected feed argument")
// }
// feed := s.Arguments[0]
//
// validUrl, err := url.Parse(feed)
//
// if err != nil {
// return fmt.Errorf("Feed '%s' is not a valid url", feed)
// }
rssFeed, err := rss.FetchFeed(context.Background(), "https://www.wagslane.dev/index.xml")
if err != nil {
return err
if len(s.Arguments) == 0 {
return fmt.Errorf("Did not receive expected time between requests argument")
}
fmt.Printf("%v\n", rssFeed)
timeBetweenRequests := s.Arguments[0]
validDuration, err := time.ParseDuration(timeBetweenRequests)
if err != nil {
return fmt.Errorf("Time between requests argument '%s' not valid duration string", timeBetweenRequests)
}
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
oldState, err := term.MakeRaw(int(os.Stdin.Fd()))
if err != nil {
return fmt.Errorf("Failed to enable raw mode: %v", err)
}
defer term.Restore(int(os.Stdin.Fd()), oldState)
go func() {
buf := make([]byte, 1)
for {
n, err := os.Stdin.Read(buf)
if err != nil || n == 0 {
return
}
if buf[0] == 'q' || buf[0] == 'Q' || buf[0] == 3 {
cancel()
return
}
}
}()
ticker := time.NewTicker(validDuration)
defer ticker.Stop()
rss.ScrapeNextFeed(ctx, s)
fetchFeeds:
for {
select {
case <-ctx.Done():
fmt.Print("Stopping feed agg command\r\n")
break fetchFeeds
case <-ticker.C:
rss.ScrapeNextFeed(ctx, s)
}
}
return nil
})
@@ -321,7 +362,7 @@ func followingCommand() *Command {
}
for _, follow := range follows {
fmt.Printf("* %s", follow.FeedName)
fmt.Printf("* %s\n", follow.FeedName)
}
return nil
@@ -358,3 +399,34 @@ func unfollowCommand() *Command {
return nil
}))
}
func browseCommand() *Command {
return newCommand("browse", requiresLoggedInUser(func(s *state.State, currentUser database.User) error {
limit := int32(2)
if len(s.Arguments) > 0 {
parsedLimit, err := strconv.ParseInt(s.Arguments[0], 10, 32)
if err == nil {
limit = int32(parsedLimit)
}
}
getUserPostsParams := database.GetPostsForUserParams{
UserID: currentUser.ID,
Limit: limit,
}
posts, err := s.Database.GetPostsForUser(context.Background(), getUserPostsParams)
if err != nil {
return err
}
for _, post := range posts {
fmt.Printf("* %s - %s\n", post.FeedName, post.Title)
}
return nil
}))
}
internal/database/feeds.sql.go
+48 -5
View File
@@ -7,22 +7,24 @@ package database
import (
"context"
"database/sql"
"time"
"github.com/google/uuid"
)
const createFeed = `-- name: CreateFeed :one
INSERT INTO feeds (id, created_at, updated_at, name, url, user_id)
INSERT INTO feeds (id, created_at, updated_at, name, url, user_id, last_fetched_at)
VALUES (
$1,
$2,
$3,
$4,
$5,
$6
$6,
$7
)
RETURNING id, created_at, updated_at, name, url, user_id
RETURNING id, created_at, updated_at, name, url, user_id, last_fetched_at
`
type CreateFeedParams struct {
@@ -32,6 +34,7 @@ type CreateFeedParams struct {
Name string
Url string
UserID uuid.UUID
LastFetchedAt sql.NullTime
}
func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, error) {
@@ -42,6 +45,7 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e
arg.Name,
arg.Url,
arg.UserID,
arg.LastFetchedAt,
)
var i Feed
err := row.Scan(
@@ -51,12 +55,13 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e
&i.Name,
&i.Url,
&i.UserID,
&i.LastFetchedAt,
)
return i, err
}
const getAllFeeds = `-- name: GetAllFeeds :many
SELECT id, created_at, updated_at, name, url, user_id FROM feeds
SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
`
func (q *Queries) GetAllFeeds(ctx context.Context) ([]Feed, error) {
@@ -75,6 +80,7 @@ func (q *Queries) GetAllFeeds(ctx context.Context) ([]Feed, error) {
&i.Name,
&i.Url,
&i.UserID,
&i.LastFetchedAt,
); err != nil {
return nil, err
}
@@ -90,7 +96,7 @@ func (q *Queries) GetAllFeeds(ctx context.Context) ([]Feed, error) {
}
const getFeedByUrl = `-- name: GetFeedByUrl :one
SELECT id, created_at, updated_at, name, url, user_id FROM feeds
SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
WHERE url = $1
`
@@ -104,6 +110,43 @@ func (q *Queries) GetFeedByUrl(ctx context.Context, url string) (Feed, error) {
&i.Name,
&i.Url,
&i.UserID,
&i.LastFetchedAt,
)
return i, err
}
const getNextFeedToFetch = `-- name: GetNextFeedToFetch :one
SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
ORDER BY last_fetched_at ASC NULLS FIRST
`
func (q *Queries) GetNextFeedToFetch(ctx context.Context) (Feed, error) {
row := q.db.QueryRowContext(ctx, getNextFeedToFetch)
var i Feed
err := row.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.Name,
&i.Url,
&i.UserID,
&i.LastFetchedAt,
)
return i, err
}
const markFeedAsFetched = `-- name: MarkFeedAsFetched :exec
UPDATE feeds
SET last_fetched_at = $1
WHERE feeds.id = $2
`
type MarkFeedAsFetchedParams struct {
LastFetchedAt sql.NullTime
ID uuid.UUID
}
func (q *Queries) MarkFeedAsFetched(ctx context.Context, arg MarkFeedAsFetchedParams) error {
_, err := q.db.ExecContext(ctx, markFeedAsFetched, arg.LastFetchedAt, arg.ID)
return err
}
internal/database/models.go
+13
View File
@@ -5,6 +5,7 @@
package database
import (
"database/sql"
"time"
"github.com/google/uuid"
@@ -17,6 +18,7 @@ type Feed struct {
Name string
Url string
UserID uuid.UUID
LastFetchedAt sql.NullTime
}
type Follow struct {
@@ -27,6 +29,17 @@ type Follow struct {
FeedID uuid.UUID
}
type Post struct {
ID uuid.UUID
CreatedAt time.Time
UpdatedAt time.Time
Title string
Url string
Description string
PublishedAt sql.NullTime
FeedID uuid.UUID
}
type User struct {
ID uuid.UUID
CreatedAt time.Time
internal/database/posts.sql.go
+136
View File
@@ -0,0 +1,136 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: posts.sql
package database
import (
"context"
"database/sql"
"time"
"github.com/google/uuid"
)
const createPost = `-- name: CreatePost :one
INSERT INTO posts (id, created_at, updated_at, title, url, description, published_at, feed_id)
VALUES (
$1,
$2,
$3,
$4,
$5,
$6,
$7,
$8
)
RETURNING id, created_at, updated_at, title, url, description, published_at, feed_id
`
type CreatePostParams struct {
ID uuid.UUID
CreatedAt time.Time
UpdatedAt time.Time
Title string
Url string
Description string
PublishedAt sql.NullTime
FeedID uuid.UUID
}
func (q *Queries) CreatePost(ctx context.Context, arg CreatePostParams) (Post, error) {
row := q.db.QueryRowContext(ctx, createPost,
arg.ID,
arg.CreatedAt,
arg.UpdatedAt,
arg.Title,
arg.Url,
arg.Description,
arg.PublishedAt,
arg.FeedID,
)
var i Post
err := row.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.Title,
&i.Url,
&i.Description,
&i.PublishedAt,
&i.FeedID,
)
return i, err
}
const getPostsForUser = `-- name: GetPostsForUser :many
SELECT
posts.id,
posts.created_at,
posts.updated_at,
posts.title,
posts.url,
posts.description,
posts.published_at,
posts.feed_id,
feeds.name as feed_name
FROM posts
INNER JOIN feeds
ON posts.feed_id = feeds.id
INNER JOIN follows
ON posts.feed_id = follows.feed_id
WHERE follows.user_id = $1
ORDER BY posts.published_at DESC, posts.title ASC
LIMIT $2
`
type GetPostsForUserParams struct {
UserID uuid.UUID
Limit int32
}
type GetPostsForUserRow struct {
ID uuid.UUID
CreatedAt time.Time
UpdatedAt time.Time
Title string
Url string
Description string
PublishedAt sql.NullTime
FeedID uuid.UUID
FeedName string
}
func (q *Queries) GetPostsForUser(ctx context.Context, arg GetPostsForUserParams) ([]GetPostsForUserRow, error) {
rows, err := q.db.QueryContext(ctx, getPostsForUser, arg.UserID, arg.Limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []GetPostsForUserRow
for rows.Next() {
var i GetPostsForUserRow
if err := rows.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.Title,
&i.Url,
&i.Description,
&i.PublishedAt,
&i.FeedID,
&i.FeedName,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
internal/rss/rss.go
+68 -1
View File
@@ -2,9 +2,16 @@ package rss
import (
"context"
"database/sql"
"encoding/xml"
"fmt"
"html"
"net/http"
"time"
"github.com/StevanFreeborn/gator/internal/database"
"github.com/StevanFreeborn/gator/internal/state"
"github.com/google/uuid"
)
type RSSItem struct {
@@ -25,7 +32,7 @@ type RSSFeed struct {
Channel RSSChannel `xml:"channel"`
}
func FetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
func fetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
var rssFeed RSSFeed
req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
@@ -59,3 +66,63 @@ func FetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
return &rssFeed, nil
}
func ScrapeNextFeed(ctx context.Context, s *state.State) {
nextFeed, err := s.Database.GetNextFeedToFetch(ctx)
if err != nil {
fmt.Print("Failed to lookup next feed to fetch\r\n")
return
}
fmt.Printf("Fetching feed '%s'\r\n", nextFeed.Name)
fetchedFeed, err := fetchFeed(ctx, nextFeed.Url)
if err != nil {
fmt.Printf("Failed to fetch feed '%s'\r\n", nextFeed.Name)
return
}
markFeedParams := database.MarkFeedAsFetchedParams{
ID: nextFeed.ID,
LastFetchedAt: sql.NullTime{
Time: time.Now(),
Valid: true,
},
}
err = s.Database.MarkFeedAsFetched(ctx, markFeedParams)
for _, item := range fetchedFeed.Channel.Item {
publishedAt, err := time.Parse(time.RFC3339, item.PubDate)
publishedAtValue := sql.NullTime{
Time: publishedAt,
Valid: err == nil,
}
createPostParams := database.CreatePostParams{
ID: uuid.New(),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
Title: item.Title,
Url: item.Link,
Description: item.Description,
PublishedAt: publishedAtValue,
FeedID: nextFeed.ID,
}
_, err = s.Database.CreatePost(ctx, createPostParams)
if err != nil {
fmt.Printf(" Skipping saving post '%s'\r\n", item.Title)
continue
}
fmt.Printf(" Saved post '%s'\r\n", item.Title)
}
if err != nil {
fmt.Printf("Failed to mark feed as fetched.\r\n")
}
}
sql/queries/feeds.sql
+14 -4
View File
@@ -1,18 +1,28 @@
-- name: CreateFeed :one
INSERT INTO feeds (id, created_at, updated_at, name, url, user_id)
INSERT INTO feeds (id, created_at, updated_at, name, url, user_id, last_fetched_at)
VALUES (
$1,
$2,
$3,
$4,
$5,
$6
$6,
$7
)
RETURNING *;
-- name: GetAllFeeds :many
SELECT id, created_at, updated_at, name, url, user_id FROM feeds;
SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds;
-- name: GetFeedByUrl :one
SELECT id, created_at, updated_at, name, url, user_id FROM feeds
SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
WHERE url = $1;
-- name: GetNextFeedToFetch :one
SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds
ORDER BY last_fetched_at ASC NULLS FIRST;
-- name: MarkFeedAsFetched :exec
UPDATE feeds
SET last_fetched_at = $1
WHERE feeds.id = $2;
sql/queries/posts.sql
+33
View File
@@ -0,0 +1,33 @@
-- name: CreatePost :one
INSERT INTO posts (id, created_at, updated_at, title, url, description, published_at, feed_id)
VALUES (
$1,
$2,
$3,
$4,
$5,
$6,
$7,
$8
)
RETURNING *;
-- name: GetPostsForUser :many
SELECT
posts.id,
posts.created_at,
posts.updated_at,
posts.title,
posts.url,
posts.description,
posts.published_at,
posts.feed_id,
feeds.name as feed_name
FROM posts
INNER JOIN feeds
ON posts.feed_id = feeds.id
INNER JOIN follows
ON posts.feed_id = follows.feed_id
WHERE follows.user_id = $1
ORDER BY posts.published_at DESC, posts.title ASC
LIMIT $2;
sql/schema/004_lastfetchedat.sql
+8
View File
@@ -0,0 +1,8 @@
-- +goose Up
ALTER TABLE feeds
ADD COLUMN last_fetched_at TIMESTAMP NULL;
-- +goose Down
ALTER TABLE feeds
DROP COLUMN last_fetched_at;
sql/schema/005_posts.sql
+16
View File
@@ -0,0 +1,16 @@
-- +goose Up
CREATE TABLE posts (
id UUID PRIMARY KEY,
created_at TIMESTAMP NOT NULL,
updated_at TIMESTAMP NOT NULL,
title TEXT NOT NULL,
url TEXT NOT NULL,
description TEXT NOT NULL,
published_at TIMESTAMP NULL,
feed_id UUID NOT NULL,
FOREIGN KEY(feed_id) REFERENCES feeds(id) ON DELETE CASCADE,
CONSTRAINT unique_url UNIQUE(url)
);
-- +goose Down
DROP TABLE posts;