Merge pull request #486 from wtfutil/WTF-42-rss-reader

WTF-42 Add FeedReader, an RSS/Atom feed reader

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/jessevdk/go-flags v1.4.0
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/mattn/go-isatty v0.0.8 // indirect
+	github.com/mmcdole/gofeed v1.0.0-beta2.0.20190420154928-0e68beaf6fdf
 	github.com/olebedev/config v0.0.0-20190528211619-364964f3a8e4
 	github.com/onsi/ginkgo v1.8.0 // indirect
 	github.com/onsi/gomega v1.5.0 // indirect

go.sum

@@ -11,6 +11,8 @@ github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFD
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/PagerDuty/go-pagerduty v0.0.0-20190503230806-cf1437c7c8d6 h1:JucouG/P7B+i18/RJbFpbqJyaserYaQzFMlfK/eIEY8=
 github.com/PagerDuty/go-pagerduty v0.0.0-20190503230806-cf1437c7c8d6/go.mod h1:6hH58nzwYc9mw+TPyM1anW0ivbI0ti4lYc+ZBaKmWts=
+github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
+github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
 github.com/StackExchange/wmi v0.0.0-20190523213609-cbe66965904d h1:VWP4o43LuzNbykZJzMUv5b9DWLgn0sn3GUj3RUyWMMQ=
 github.com/StackExchange/wmi v0.0.0-20190523213609-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/adlio/trello v1.0.0 h1:7Mp6DnNXBHBAdhfcutftFDnX7K/G9yEtScAEplJzu+0=
@@ -26,6 +28,8 @@ github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 h1:p9Sln00KOTlrYkx
 github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E=
 github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/andygrunwald/go-gerrit v0.0.0-20190625080919-64931d233c2d h1:VqtwQ/1Q39dznaTGQXmPzwdTbqKd2jdlfNgawTVM6YU=
 github.com/andygrunwald/go-gerrit v0.0.0-20190625080919-64931d233c2d/go.mod h1:0iuRQp6WJ44ts+iihy5E/WlPqfg5RNeQxOmzRkxCdtk=
 github.com/briandowns/openweathermap v0.0.0-20180804155945-5f41b7c9d92d h1:28xWzPQ9bdGxKAAwQpZipZZ9Xz8kQcgMPF9cZnvMeuI=
@@ -33,6 +37,7 @@ github.com/briandowns/openweathermap v0.0.0-20180804155945-5f41b7c9d92d/go.mod h
 github.com/cenkalti/backoff v2.2.0+incompatible h1:8qVbEY6GLhoLlLi1Ac2ZkVhedNwlhQXc39qivKp9+GI=
 github.com/cenkalti/backoff v2.2.0+incompatible/go.mod h1:b6Nc7NRH5C4aCISLry0tLnTjcuTEvoiqcWDdsU0sOGM=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
 github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
 github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
 github.com/darkSasori/todoist v0.0.0-20180703032645-ec6b38b374ab h1:T9EEtA6FSJMVypkNlKjrRo04u1j5Tk+gghymflyivDw=
@@ -103,6 +108,10 @@ github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mmcdole/gofeed v1.0.0-beta2.0.20190420154928-0e68beaf6fdf h1:poo3e5STwUVGyyUX0e3fHQUwT1tV8IYEFUUpYd/7LuA=
+github.com/mmcdole/gofeed v1.0.0-beta2.0.20190420154928-0e68beaf6fdf/go.mod h1:tkVcyzS3qVMlQrQxJoEH1hkTiuo9a8emDzkMi7TZBu0=
+github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
+github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
 github.com/olebedev/config v0.0.0-20190528211619-364964f3a8e4 h1:JnVsYEQzhEcOspy6ngIYNF2u0h2mjkXZptzX0IzZQ4g=
 github.com/olebedev/config v0.0.0-20190528211619-364964f3a8e4/go.mod h1:RL5+WRxWTAXqqCi9i+eZlHrUtO7AQujUqWi+xMohmc4=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -154,10 +163,13 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=

maker/widget_maker.go

@@ -12,6 +12,7 @@ import (
 	"github.com/wtfutil/wtf/modules/cryptoexchanges/blockfolio"
 	"github.com/wtfutil/wtf/modules/cryptoexchanges/cryptolive"
 	"github.com/wtfutil/wtf/modules/datadog"
+	"github.com/wtfutil/wtf/modules/feedreader"
 	"github.com/wtfutil/wtf/modules/gcal"
 	"github.com/wtfutil/wtf/modules/gerrit"
 	"github.com/wtfutil/wtf/modules/git"
@@ -92,6 +93,9 @@ func MakeWidget(
 	case "datadog":
 		settings := datadog.NewSettingsFromYAML(widgetName, moduleConfig, globalConfig)
 		widget = datadog.NewWidget(app, pages, settings)
+	case "feedreader":
+		settings := feedreader.NewSettingsFromYAML(widgetName, moduleConfig, globalConfig)
+		widget = feedreader.NewWidget(app, pages, settings)
 	case "gcal":
 		settings := gcal.NewSettingsFromYAML(widgetName, moduleConfig, globalConfig)
 		widget = gcal.NewWidget(app, settings)

modules/feedreader/keyboard.go

@@ -0,0 +1,16 @@
+package feedreader
+
+import "github.com/gdamore/tcell"
+
+func (widget *Widget) initializeKeyboardControls() {
+	widget.SetKeyboardChar("/", widget.ShowHelp, "Show/hide this help widget")
+	widget.SetKeyboardChar("r", widget.Refresh, "Refresh widget")
+	widget.SetKeyboardChar("j", widget.Next, "Select next item")
+	widget.SetKeyboardChar("k", widget.Prev, "Select previous item")
+	widget.SetKeyboardChar("o", widget.openStory, "Open story in browser")
+
+	widget.SetKeyboardKey(tcell.KeyDown, widget.Next, "Select next item")
+	widget.SetKeyboardKey(tcell.KeyUp, widget.Prev, "Select previous item")
+	widget.SetKeyboardKey(tcell.KeyEnter, widget.openStory, "Open story in browser")
+	widget.SetKeyboardKey(tcell.KeyEsc, widget.Unselect, "Clear selection")
+}

modules/feedreader/settings.go

@@ -0,0 +1,31 @@
+package feedreader
+
+import (
+	"github.com/olebedev/config"
+	"github.com/wtfutil/wtf/cfg"
+	"github.com/wtfutil/wtf/wtf"
+)
+
+const (
+	defaultTitle = "Feed Reader"
+)
+
+// Settings defines the configuration properties for this module
+type Settings struct {
+	common *cfg.Common
+
+	feeds     []string `help:"An array of RSS and Atom feed URLs"`
+	feedLimit int      `help:"The maximum number of stories to display for each feed"`
+}
+
+// NewSettingsFromYAML creates a new settings instance from a YAML config block
+func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *config.Config) *Settings {
+	settings := &Settings{
+		common: cfg.NewCommonSettingsFromModule(name, defaultTitle, ymlConfig, globalConfig),
+
+		feeds:     wtf.ToStrs(ymlConfig.UList("feeds")),
+		feedLimit: ymlConfig.UInt("feedLimit", -1),
+	}
+
+	return settings
+}

modules/feedreader/widget.go

@@ -0,0 +1,165 @@
+package feedreader
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/mmcdole/gofeed"
+	"github.com/rivo/tview"
+	"github.com/wtfutil/wtf/wtf"
+)
+
+// FeedItem represents an item returned from an RSS or Atom feed
+type FeedItem struct {
+	item   *gofeed.Item
+	viewed bool
+}
+
+// Widget is the container for RSS and Atom data
+type Widget struct {
+	wtf.KeyboardWidget
+	wtf.ScrollableWidget
+
+	stories  []*FeedItem
+	parser   *gofeed.Parser
+	settings *Settings
+}
+
+// NewWidget creates a new instance of a widget
+func NewWidget(app *tview.Application, pages *tview.Pages, settings *Settings) *Widget {
+	widget := &Widget{
+		KeyboardWidget:   wtf.NewKeyboardWidget(app, pages, settings.common),
+		ScrollableWidget: wtf.NewScrollableWidget(app, settings.common, true),
+
+		parser:   gofeed.NewParser(),
+		settings: settings,
+	}
+
+	widget.SetRenderFunction(widget.Render)
+	widget.initializeKeyboardControls()
+	widget.View.SetInputCapture(widget.InputCapture)
+
+	widget.KeyboardWidget.SetView(widget.View)
+
+	return widget
+}
+
+/* -------------------- Exported Functions -------------------- */
+
+// Fetch retrieves RSS and Atom feed data
+func (widget *Widget) Fetch(feedURLs []string) ([]*FeedItem, error) {
+	data := []*FeedItem{}
+
+	for _, feedURL := range feedURLs {
+		feedItems, err := widget.fetchForFeed(feedURL)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, feedItem := range feedItems {
+			data = append(data, feedItem)
+		}
+	}
+
+	data = widget.sort(data)
+
+	return data, nil
+}
+
+// Refresh updates the data in the widget
+func (widget *Widget) Refresh() {
+	feedItems, err := widget.Fetch(widget.settings.feeds)
+	if err != nil {
+		widget.Redraw(widget.CommonSettings.Title, err.Error(), true)
+	}
+
+	widget.stories = feedItems
+	widget.SetItemCount(len(feedItems))
+
+	widget.Render()
+}
+
+// Render sets up the widget data for redrawing to the screen
+func (widget *Widget) Render() {
+	if widget.stories == nil {
+		return
+	}
+
+	title := widget.CommonSettings.Title
+	widget.Redraw(title, widget.contentFrom(widget.stories), false)
+}
+
+/* -------------------- Unexported Functions -------------------- */
+
+func (widget *Widget) fetchForFeed(feedURL string) ([]*FeedItem, error) {
+	feed, err := widget.parser.ParseURL(feedURL)
+	if err != nil {
+		return nil, err
+	}
+
+	feedItems := []*FeedItem{}
+
+	for idx, gofeedItem := range feed.Items {
+		if widget.settings.feedLimit >= 1 && idx >= widget.settings.feedLimit {
+			// We only want to get the widget.settings.feedLimit latest articles,
+			// not all of them. To get all, set feedLimit to < 1
+			break
+		}
+
+		feedItem := &FeedItem{
+			item:   gofeedItem,
+			viewed: false,
+		}
+
+		feedItems = append(feedItems, feedItem)
+	}
+
+	return feedItems, nil
+}
+
+func (widget *Widget) contentFrom(data []*FeedItem) string {
+	var str string
+
+	for idx, feedItem := range data {
+		rowColor := widget.RowColor(idx)
+
+		if feedItem.viewed {
+			// Grays out viewed items in the list, while preserving background highlighting when selected
+			rowColor = "gray"
+			if idx == widget.Selected {
+				rowColor = fmt.Sprintf("gray:%s", widget.settings.common.Colors.HighlightBack)
+			}
+		}
+
+		row := fmt.Sprintf(
+			"[%s]%2d. %s[white]",
+			rowColor,
+			idx+1,
+			feedItem.item.Title,
+		)
+
+		str += wtf.HighlightableHelper(widget.View, row, idx, len(feedItem.item.Title))
+	}
+
+	return str
+}
+
+// feedItems are sorted by published date
+func (widget *Widget) sort(feedItems []*FeedItem) []*FeedItem {
+	sort.Slice(feedItems, func(i, j int) bool {
+		return feedItems[i].item.Published < feedItems[j].item.Published
+	})
+
+	return feedItems
+}
+
+func (widget *Widget) openStory() {
+	sel := widget.GetSelected()
+
+	if sel >= 0 && widget.stories != nil && sel < len(widget.stories) {
+		story := widget.stories[sel]
+		story.viewed = true
+
+		wtf.OpenFile(story.item.Link)
+	}
+}

modules/hackernews/widget.go

@@ -18,7 +18,7 @@ type Widget struct {
 }
 
 func NewWidget(app *tview.Application, pages *tview.Pages, settings *Settings) *Widget {
-	widget := Widget{
+	widget := &Widget{
 		KeyboardWidget:   wtf.NewKeyboardWidget(app, pages, settings.common),
 		ScrollableWidget: wtf.NewScrollableWidget(app, settings.common, true),
 
@@ -31,7 +31,7 @@ func NewWidget(app *tview.Application, pages *tview.Pages, settings *Settings) *
 
 	widget.KeyboardWidget.SetView(widget.View)
 
-	return &widget
+	return widget
 }
 
 /* -------------------- Exported Functions -------------------- */
@@ -53,9 +53,7 @@ func (widget *Widget) Refresh() {
 	var stories []Story
 	for idx := 0; idx < widget.settings.numberOfStories; idx++ {
 		story, e := GetStory(storyIds[idx])
-		if e != nil {
+		if e == nil {
-			// panic(e)
-		} else {
 			stories = append(stories, story)
 		}
 	}
@@ -66,21 +64,22 @@ func (widget *Widget) Refresh() {
 	widget.Render()
 }
 
-/* -------------------- Unexported Functions -------------------- */
+// Render sets up the widget data for redrawing to the screen
-
 func (widget *Widget) Render() {
 	if widget.stories == nil {
 		return
 	}
 
-	title := fmt.Sprintf("%s - %sstories", widget.CommonSettings.Title, widget.settings.storyType)
+	title := fmt.Sprintf("%s - %s stories", widget.CommonSettings.Title, widget.settings.storyType)
 	widget.Redraw(title, widget.contentFrom(widget.stories), false)
 }
 
+/* -------------------- Unexported Functions -------------------- */
+
 func (widget *Widget) contentFrom(stories []Story) string {
 	var str string
-	for idx, story := range stories {
 
+	for idx, story := range stories {
 		u, _ := url.Parse(story.URL)
 
 		row := fmt.Sprintf(

modules/hibp/settings.go

@@ -29,7 +29,7 @@ type Settings struct {
 
 // NewSettingsFromYAML creates a new settings instance from a YAML config block
 func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *config.Config) *Settings {
-	settings := Settings{
+	settings := &Settings{
 		common: cfg.NewCommonSettingsFromModule(name, defaultTitle, ymlConfig, globalConfig),
 
 		accounts: wtf.ToStrs(ymlConfig.UList("accounts")),
@@ -45,7 +45,7 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co
 		settings.common.RefreshInterval = minRefreshInterval
 	}
 
-	return &settings
+	return settings
 }
 
 // HasSince returns TRUE if there's a valid "since" value setting, FALSE if there is not

modules/hibp/widget.go

@@ -72,7 +72,7 @@ func (widget *Widget) contentFrom(data []*Status) string {
 			color = widget.settings.colors.pwned
 		}
 
-		str = str + fmt.Sprintf(" [%s]%s[white]\n", color, stat.Account)
+		str += fmt.Sprintf(" [%s]%s[white]\n", color, stat.Account)
 	}
 
 	return str