package rss import ( "context" "database/sql" "encoding/xml" "fmt" "html" "net/http" "time" "github.com/StevanFreeborn/gator/internal/database" "github.com/StevanFreeborn/gator/internal/state" "github.com/google/uuid" ) type RSSItem struct { Title string `xml:"title"` Link string `xml:"link"` Description string `xml:"description"` PubDate string `xml:"pubDate"` } type RSSChannel struct { Title string `xml:"title"` Link string `xml:"link"` Description string `xml:"description"` Item []RSSItem `xml:"item"` } type RSSFeed struct { Channel RSSChannel `xml:"channel"` } func fetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) { var rssFeed RSSFeed req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil) req.Header.Set("User-Agent", "gator") if err != nil { return nil, err } res, err := http.DefaultClient.Do(req) if err != nil { return nil, err } defer res.Body.Close() err = xml.NewDecoder(res.Body).Decode(&rssFeed) if err != nil { return nil, err } rssFeed.Channel.Title = html.UnescapeString(rssFeed.Channel.Title) rssFeed.Channel.Description = html.UnescapeString(rssFeed.Channel.Description) for _, item := range rssFeed.Channel.Item { item.Title = html.UnescapeString(item.Title) item.Description = html.UnescapeString(item.Description) } return &rssFeed, nil } func ScrapeNextFeed(ctx context.Context, s *state.State) { nextFeed, err := s.Database.GetNextFeedToFetch(ctx) if err != nil { fmt.Print("Failed to lookup next feed to fetch\r\n") return } fmt.Printf("Fetching feed '%s'\r\n", nextFeed.Name) fetchedFeed, err := fetchFeed(ctx, nextFeed.Url) if err != nil { fmt.Printf("Failed to fetch feed '%s'\r\n", nextFeed.Name) return } markFeedParams := database.MarkFeedAsFetchedParams{ ID: nextFeed.ID, LastFetchedAt: sql.NullTime{ Time: time.Now(), Valid: true, }, } err = s.Database.MarkFeedAsFetched(ctx, markFeedParams) for _, item := range fetchedFeed.Channel.Item { publishedAt, err := time.Parse(time.RFC3339, item.PubDate) publishedAtValue := sql.NullTime{ Time: publishedAt, Valid: err == nil, } createPostParams := database.CreatePostParams{ ID: uuid.New(), CreatedAt: time.Now(), UpdatedAt: time.Now(), Title: item.Title, Url: item.Link, Description: item.Description, PublishedAt: publishedAtValue, FeedID: nextFeed.ID, } _, err = s.Database.CreatePost(ctx, createPostParams) if err != nil { fmt.Printf(" Skipping saving post '%s'\r\n", item.Title) continue } fmt.Printf(" Saved post '%s'\r\n", item.Title) } if err != nil { fmt.Printf("Failed to mark feed as fetched.\r\n") } }