129 lines
2.7 KiB
Go
129 lines
2.7 KiB
Go
package rss
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"html"
|
|
"net/http"
|
|
"time"
|
|
|
|
"github.com/StevanFreeborn/gator/internal/database"
|
|
"github.com/StevanFreeborn/gator/internal/state"
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type RSSItem struct {
|
|
Title string `xml:"title"`
|
|
Link string `xml:"link"`
|
|
Description string `xml:"description"`
|
|
PubDate string `xml:"pubDate"`
|
|
}
|
|
|
|
type RSSChannel struct {
|
|
Title string `xml:"title"`
|
|
Link string `xml:"link"`
|
|
Description string `xml:"description"`
|
|
Item []RSSItem `xml:"item"`
|
|
}
|
|
|
|
type RSSFeed struct {
|
|
Channel RSSChannel `xml:"channel"`
|
|
}
|
|
|
|
func fetchFeed(ctx context.Context, feedURL string) (*RSSFeed, error) {
|
|
var rssFeed RSSFeed
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, feedURL, nil)
|
|
req.Header.Set("User-Agent", "gator")
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
res, err := http.DefaultClient.Do(req)
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
err = xml.NewDecoder(res.Body).Decode(&rssFeed)
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rssFeed.Channel.Title = html.UnescapeString(rssFeed.Channel.Title)
|
|
rssFeed.Channel.Description = html.UnescapeString(rssFeed.Channel.Description)
|
|
|
|
for _, item := range rssFeed.Channel.Item {
|
|
item.Title = html.UnescapeString(item.Title)
|
|
item.Description = html.UnescapeString(item.Description)
|
|
}
|
|
|
|
return &rssFeed, nil
|
|
}
|
|
|
|
func ScrapeNextFeed(ctx context.Context, s *state.State) {
|
|
nextFeed, err := s.Database.GetNextFeedToFetch(ctx)
|
|
|
|
if err != nil {
|
|
fmt.Print("Failed to lookup next feed to fetch\r\n")
|
|
return
|
|
}
|
|
|
|
fmt.Printf("Fetching feed '%s'\r\n", nextFeed.Name)
|
|
|
|
fetchedFeed, err := fetchFeed(ctx, nextFeed.Url)
|
|
|
|
if err != nil {
|
|
fmt.Printf("Failed to fetch feed '%s'\r\n", nextFeed.Name)
|
|
return
|
|
}
|
|
|
|
markFeedParams := database.MarkFeedAsFetchedParams{
|
|
ID: nextFeed.ID,
|
|
LastFetchedAt: sql.NullTime{
|
|
Time: time.Now(),
|
|
Valid: true,
|
|
},
|
|
}
|
|
|
|
err = s.Database.MarkFeedAsFetched(ctx, markFeedParams)
|
|
|
|
for _, item := range fetchedFeed.Channel.Item {
|
|
publishedAt, err := time.Parse(time.RFC3339, item.PubDate)
|
|
publishedAtValue := sql.NullTime{
|
|
Time: publishedAt,
|
|
Valid: err == nil,
|
|
}
|
|
|
|
createPostParams := database.CreatePostParams{
|
|
ID: uuid.New(),
|
|
CreatedAt: time.Now(),
|
|
UpdatedAt: time.Now(),
|
|
Title: item.Title,
|
|
Url: item.Link,
|
|
Description: item.Description,
|
|
PublishedAt: publishedAtValue,
|
|
FeedID: nextFeed.ID,
|
|
}
|
|
|
|
_, err = s.Database.CreatePost(ctx, createPostParams)
|
|
|
|
if err != nil {
|
|
fmt.Printf(" Skipping saving post '%s'\r\n", item.Title)
|
|
continue
|
|
}
|
|
|
|
fmt.Printf(" Saved post '%s'\r\n", item.Title)
|
|
}
|
|
|
|
if err != nil {
|
|
fmt.Printf("Failed to mark feed as fetched.\r\n")
|
|
}
|
|
}
|