feat: implement feed agg and browse

internal/rss/rss.go

@@ -2,9 +2,16 @@ package rss
 
 import (
 	"context"
+	"database/sql"
 	"encoding/xml"
+	"fmt"
 	"html"
 	"net/http"
+	"time"
+
+	"github.com/StevanFreeborn/gator/internal/database"
+	"github.com/StevanFreeborn/gator/internal/state"
+	"github.com/google/uuid"
 )
 
 type RSSItem struct {
@@ -25,7 +32,7 @@ type RSSFeed struct {
 	Channel RSSChannel `xml:"channel"`
 }
 
-func FetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
+func fetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
 	var rssFeed RSSFeed
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
@@ -59,3 +66,63 @@ func FetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
 
 	return &rssFeed, nil
 }
+
+func ScrapeNextFeed(ctx context.Context, s *state.State) {
+	nextFeed, err := s.Database.GetNextFeedToFetch(ctx)
+
+	if err != nil {
+		fmt.Print("Failed to lookup next feed to fetch\r\n")
+		return
+	}
+
+	fmt.Printf("Fetching feed '%s'\r\n", nextFeed.Name)
+
+	fetchedFeed, err := fetchFeed(ctx, nextFeed.Url)
+
+	if err != nil {
+		fmt.Printf("Failed to fetch feed '%s'\r\n", nextFeed.Name)
+		return
+	}
+
+	markFeedParams := database.MarkFeedAsFetchedParams{
+		ID: nextFeed.ID,
+		LastFetchedAt: sql.NullTime{
+			Time:  time.Now(),
+			Valid: true,
+		},
+	}
+
+	err = s.Database.MarkFeedAsFetched(ctx, markFeedParams)
+
+	for _, item := range fetchedFeed.Channel.Item {
+		publishedAt, err := time.Parse(time.RFC3339, item.PubDate)
+		publishedAtValue := sql.NullTime{
+			Time:  publishedAt,
+			Valid: err == nil,
+		}
+
+		createPostParams := database.CreatePostParams{
+			ID:          uuid.New(),
+			CreatedAt:   time.Now(),
+			UpdatedAt:   time.Now(),
+			Title:       item.Title,
+			Url:         item.Link,
+			Description: item.Description,
+			PublishedAt: publishedAtValue,
+			FeedID:      nextFeed.ID,
+		}
+
+		_, err = s.Database.CreatePost(ctx, createPostParams)
+
+		if err != nil {
+			fmt.Printf("  Skipping saving post '%s'\r\n", item.Title)
+			continue
+		}
+
+		fmt.Printf("  Saved post '%s'\r\n", item.Title)
+	}
+
+	if err != nil {
+		fmt.Printf("Failed to mark feed as fetched.\r\n")
+	}
+}