From f19f1ee86d846121722f7d7a0dcd23a0b1d8b66f Mon Sep 17 00:00:00 2001 From: Chris Cummer Date: Wed, 3 Jul 2019 21:10:52 -0700 Subject: [PATCH 1/2] WTF-42 WIP Add FeedReader, an RSS/Atom feed reader --- go.mod | 1 + go.sum | 12 +++ maker/widget_maker.go | 4 + modules/feedreader/keyboard.go | 16 ++++ modules/feedreader/settings.go | 31 +++++++ modules/feedreader/widget.go | 165 +++++++++++++++++++++++++++++++++ modules/hackernews/widget.go | 17 ++-- modules/hibp/settings.go | 4 +- modules/hibp/widget.go | 2 +- 9 files changed, 240 insertions(+), 12 deletions(-) create mode 100644 modules/feedreader/keyboard.go create mode 100644 modules/feedreader/settings.go create mode 100644 modules/feedreader/widget.go diff --git a/go.mod b/go.mod index e94c79ed..27269669 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/jessevdk/go-flags v1.4.0 github.com/kr/pretty v0.1.0 // indirect github.com/mattn/go-isatty v0.0.8 // indirect + github.com/mmcdole/gofeed v1.0.0-beta2.0.20190420154928-0e68beaf6fdf github.com/olebedev/config v0.0.0-20190528211619-364964f3a8e4 github.com/onsi/ginkgo v1.8.0 // indirect github.com/onsi/gomega v1.5.0 // indirect diff --git a/go.sum b/go.sum index 5dff05c9..628a8205 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFD github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/PagerDuty/go-pagerduty v0.0.0-20190503230806-cf1437c7c8d6 h1:JucouG/P7B+i18/RJbFpbqJyaserYaQzFMlfK/eIEY8= github.com/PagerDuty/go-pagerduty v0.0.0-20190503230806-cf1437c7c8d6/go.mod h1:6hH58nzwYc9mw+TPyM1anW0ivbI0ti4lYc+ZBaKmWts= +github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/StackExchange/wmi v0.0.0-20190523213609-cbe66965904d h1:VWP4o43LuzNbykZJzMUv5b9DWLgn0sn3GUj3RUyWMMQ= github.com/StackExchange/wmi v0.0.0-20190523213609-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/adlio/trello v1.0.0 h1:7Mp6DnNXBHBAdhfcutftFDnX7K/G9yEtScAEplJzu+0= @@ -26,6 +28,8 @@ github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 h1:p9Sln00KOTlrYkx github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E= github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= +github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andygrunwald/go-gerrit v0.0.0-20190625080919-64931d233c2d h1:VqtwQ/1Q39dznaTGQXmPzwdTbqKd2jdlfNgawTVM6YU= github.com/andygrunwald/go-gerrit v0.0.0-20190625080919-64931d233c2d/go.mod h1:0iuRQp6WJ44ts+iihy5E/WlPqfg5RNeQxOmzRkxCdtk= github.com/briandowns/openweathermap v0.0.0-20180804155945-5f41b7c9d92d h1:28xWzPQ9bdGxKAAwQpZipZZ9Xz8kQcgMPF9cZnvMeuI= @@ -33,6 +37,7 @@ github.com/briandowns/openweathermap v0.0.0-20180804155945-5f41b7c9d92d/go.mod h github.com/cenkalti/backoff v2.2.0+incompatible h1:8qVbEY6GLhoLlLi1Ac2ZkVhedNwlhQXc39qivKp9+GI= github.com/cenkalti/backoff v2.2.0+incompatible/go.mod h1:b6Nc7NRH5C4aCISLry0tLnTjcuTEvoiqcWDdsU0sOGM= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= github.com/darkSasori/todoist v0.0.0-20180703032645-ec6b38b374ab h1:T9EEtA6FSJMVypkNlKjrRo04u1j5Tk+gghymflyivDw= @@ -103,6 +108,10 @@ github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mmcdole/gofeed v1.0.0-beta2.0.20190420154928-0e68beaf6fdf h1:poo3e5STwUVGyyUX0e3fHQUwT1tV8IYEFUUpYd/7LuA= +github.com/mmcdole/gofeed v1.0.0-beta2.0.20190420154928-0e68beaf6fdf/go.mod h1:tkVcyzS3qVMlQrQxJoEH1hkTiuo9a8emDzkMi7TZBu0= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= github.com/olebedev/config v0.0.0-20190528211619-364964f3a8e4 h1:JnVsYEQzhEcOspy6ngIYNF2u0h2mjkXZptzX0IzZQ4g= github.com/olebedev/config v0.0.0-20190528211619-364964f3a8e4/go.mod h1:RL5+WRxWTAXqqCi9i+eZlHrUtO7AQujUqWi+xMohmc4= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -154,10 +163,13 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= diff --git a/maker/widget_maker.go b/maker/widget_maker.go index 4d3a2791..ad33a572 100644 --- a/maker/widget_maker.go +++ b/maker/widget_maker.go @@ -12,6 +12,7 @@ import ( "github.com/wtfutil/wtf/modules/cryptoexchanges/blockfolio" "github.com/wtfutil/wtf/modules/cryptoexchanges/cryptolive" "github.com/wtfutil/wtf/modules/datadog" + "github.com/wtfutil/wtf/modules/feedreader" "github.com/wtfutil/wtf/modules/gcal" "github.com/wtfutil/wtf/modules/gerrit" "github.com/wtfutil/wtf/modules/git" @@ -92,6 +93,9 @@ func MakeWidget( case "datadog": settings := datadog.NewSettingsFromYAML(widgetName, moduleConfig, globalConfig) widget = datadog.NewWidget(app, pages, settings) + case "feedreader": + settings := feedreader.NewSettingsFromYAML(widgetName, moduleConfig, globalConfig) + widget = feedreader.NewWidget(app, pages, settings) case "gcal": settings := gcal.NewSettingsFromYAML(widgetName, moduleConfig, globalConfig) widget = gcal.NewWidget(app, settings) diff --git a/modules/feedreader/keyboard.go b/modules/feedreader/keyboard.go new file mode 100644 index 00000000..bc8bd14c --- /dev/null +++ b/modules/feedreader/keyboard.go @@ -0,0 +1,16 @@ +package feedreader + +import "github.com/gdamore/tcell" + +func (widget *Widget) initializeKeyboardControls() { + widget.SetKeyboardChar("/", widget.ShowHelp, "Show/hide this help widget") + widget.SetKeyboardChar("r", widget.Refresh, "Refresh widget") + widget.SetKeyboardChar("j", widget.Next, "Select next item") + widget.SetKeyboardChar("k", widget.Prev, "Select previous item") + widget.SetKeyboardChar("o", widget.openStory, "Open story in browser") + + widget.SetKeyboardKey(tcell.KeyDown, widget.Next, "Select next item") + widget.SetKeyboardKey(tcell.KeyUp, widget.Prev, "Select previous item") + widget.SetKeyboardKey(tcell.KeyEnter, widget.openStory, "Open story in browser") + widget.SetKeyboardKey(tcell.KeyEsc, widget.Unselect, "Clear selection") +} diff --git a/modules/feedreader/settings.go b/modules/feedreader/settings.go new file mode 100644 index 00000000..154032e3 --- /dev/null +++ b/modules/feedreader/settings.go @@ -0,0 +1,31 @@ +package feedreader + +import ( + "github.com/olebedev/config" + "github.com/wtfutil/wtf/cfg" + "github.com/wtfutil/wtf/wtf" +) + +const ( + defaultTitle = "Feed Reader" +) + +// Settings defines the configuration properties for this module +type Settings struct { + common *cfg.Common + + feeds []string + feedLimit int +} + +// NewSettingsFromYAML creates a new settings instance from a YAML config block +func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *config.Config) *Settings { + settings := &Settings{ + common: cfg.NewCommonSettingsFromModule(name, defaultTitle, ymlConfig, globalConfig), + + feeds: wtf.ToStrs(ymlConfig.UList("feeds")), + feedLimit: ymlConfig.UInt("feedLimit", -1), + } + + return settings +} diff --git a/modules/feedreader/widget.go b/modules/feedreader/widget.go new file mode 100644 index 00000000..ecdc4351 --- /dev/null +++ b/modules/feedreader/widget.go @@ -0,0 +1,165 @@ +package feedreader + +import ( + "fmt" + "sort" + + "github.com/mmcdole/gofeed" + "github.com/rivo/tview" + "github.com/wtfutil/wtf/wtf" +) + +// FeedItem represents an item returned from an RSS or Atom feed +type FeedItem struct { + item *gofeed.Item + viewed bool +} + +// Widget is the container for RSS and Atom data +type Widget struct { + wtf.KeyboardWidget + wtf.ScrollableWidget + + stories []*FeedItem + parser *gofeed.Parser + settings *Settings +} + +// NewWidget creates a new instance of a widget +func NewWidget(app *tview.Application, pages *tview.Pages, settings *Settings) *Widget { + widget := &Widget{ + KeyboardWidget: wtf.NewKeyboardWidget(app, pages, settings.common), + ScrollableWidget: wtf.NewScrollableWidget(app, settings.common, true), + + parser: gofeed.NewParser(), + settings: settings, + } + + widget.SetRenderFunction(widget.Render) + widget.initializeKeyboardControls() + widget.View.SetInputCapture(widget.InputCapture) + + widget.KeyboardWidget.SetView(widget.View) + + return widget +} + +/* -------------------- Exported Functions -------------------- */ + +// Fetch retrieves RSS and Atom feed data +func (widget *Widget) Fetch(feedURLs []string) ([]*FeedItem, error) { + data := []*FeedItem{} + + for _, feedURL := range feedURLs { + feedItems, err := widget.fetchForFeed(feedURL) + if err != nil { + return nil, err + } + + for _, feedItem := range feedItems { + data = append(data, feedItem) + } + } + + data = widget.sort(data) + + return data, nil +} + +// Refresh updates the data in the widget +func (widget *Widget) Refresh() { + feedItems, err := widget.Fetch(widget.settings.feeds) + if err != nil { + widget.Redraw(widget.CommonSettings.Title, err.Error(), true) + } + + widget.stories = feedItems + widget.SetItemCount(len(feedItems)) + + widget.Render() +} + +// Render sets up the widget data for redrawing to the screen +func (widget *Widget) Render() { + if widget.stories == nil { + return + } + + title := widget.CommonSettings.Title + widget.Redraw(title, widget.contentFrom(widget.stories), false) +} + +/* -------------------- Unexported Functions -------------------- */ + +func (widget *Widget) fetchForFeed(feedURL string) ([]*FeedItem, error) { + feed, err := widget.parser.ParseURL(feedURL) + if err != nil { + return nil, err + } + + feedItems := []*FeedItem{} + + for idx, gofeedItem := range feed.Items { + if widget.settings.feedLimit >= 1 && idx >= widget.settings.feedLimit { + // We only want to get the widget.settings.feedLimit latest articles, + // not all of them. To get all, set feedLimit to < 1 + break + } + + feedItem := &FeedItem{ + item: gofeedItem, + viewed: false, + } + + feedItems = append(feedItems, feedItem) + } + + return feedItems, nil +} + +func (widget *Widget) contentFrom(data []*FeedItem) string { + var str string + + for idx, feedItem := range data { + rowColor := widget.RowColor(idx) + + if feedItem.viewed { + // Grays out viewed items in the list, while preserving background highlighting when selected + rowColor = "gray" + if idx == widget.Selected { + rowColor = fmt.Sprintf("gray:%s", widget.settings.common.Colors.HighlightBack) + } + } + + row := fmt.Sprintf( + "[%s]%2d. %s[white]", + rowColor, + idx+1, + feedItem.item.Title, + ) + + str += wtf.HighlightableHelper(widget.View, row, idx, len(feedItem.item.Title)) + } + + return str +} + +// feedItems are sorted by published date +func (widget *Widget) sort(feedItems []*FeedItem) []*FeedItem { + sort.Slice(feedItems, func(i, j int) bool { + return feedItems[i].item.Published < feedItems[j].item.Published + }) + + return feedItems +} + +func (widget *Widget) openStory() { + sel := widget.GetSelected() + + if sel >= 0 && widget.stories != nil && sel < len(widget.stories) { + story := widget.stories[sel] + story.viewed = true + + wtf.OpenFile(story.item.Link) + } +} diff --git a/modules/hackernews/widget.go b/modules/hackernews/widget.go index daa49f9b..1bffb047 100644 --- a/modules/hackernews/widget.go +++ b/modules/hackernews/widget.go @@ -18,7 +18,7 @@ type Widget struct { } func NewWidget(app *tview.Application, pages *tview.Pages, settings *Settings) *Widget { - widget := Widget{ + widget := &Widget{ KeyboardWidget: wtf.NewKeyboardWidget(app, pages, settings.common), ScrollableWidget: wtf.NewScrollableWidget(app, settings.common, true), @@ -31,7 +31,7 @@ func NewWidget(app *tview.Application, pages *tview.Pages, settings *Settings) * widget.KeyboardWidget.SetView(widget.View) - return &widget + return widget } /* -------------------- Exported Functions -------------------- */ @@ -53,9 +53,7 @@ func (widget *Widget) Refresh() { var stories []Story for idx := 0; idx < widget.settings.numberOfStories; idx++ { story, e := GetStory(storyIds[idx]) - if e != nil { - // panic(e) - } else { + if e == nil { stories = append(stories, story) } } @@ -66,21 +64,22 @@ func (widget *Widget) Refresh() { widget.Render() } -/* -------------------- Unexported Functions -------------------- */ - +// Render sets up the widget data for redrawing to the screen func (widget *Widget) Render() { if widget.stories == nil { return } - title := fmt.Sprintf("%s - %sstories", widget.CommonSettings.Title, widget.settings.storyType) + title := fmt.Sprintf("%s - %s stories", widget.CommonSettings.Title, widget.settings.storyType) widget.Redraw(title, widget.contentFrom(widget.stories), false) } +/* -------------------- Unexported Functions -------------------- */ + func (widget *Widget) contentFrom(stories []Story) string { var str string - for idx, story := range stories { + for idx, story := range stories { u, _ := url.Parse(story.URL) row := fmt.Sprintf( diff --git a/modules/hibp/settings.go b/modules/hibp/settings.go index b9b7c269..f5b9d0f3 100644 --- a/modules/hibp/settings.go +++ b/modules/hibp/settings.go @@ -29,7 +29,7 @@ type Settings struct { // NewSettingsFromYAML creates a new settings instance from a YAML config block func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *config.Config) *Settings { - settings := Settings{ + settings := &Settings{ common: cfg.NewCommonSettingsFromModule(name, defaultTitle, ymlConfig, globalConfig), accounts: wtf.ToStrs(ymlConfig.UList("accounts")), @@ -45,7 +45,7 @@ func NewSettingsFromYAML(name string, ymlConfig *config.Config, globalConfig *co settings.common.RefreshInterval = minRefreshInterval } - return &settings + return settings } // HasSince returns TRUE if there's a valid "since" value setting, FALSE if there is not diff --git a/modules/hibp/widget.go b/modules/hibp/widget.go index c33a390c..c744dc4e 100644 --- a/modules/hibp/widget.go +++ b/modules/hibp/widget.go @@ -72,7 +72,7 @@ func (widget *Widget) contentFrom(data []*Status) string { color = widget.settings.colors.pwned } - str = str + fmt.Sprintf(" [%s]%s[white]\n", color, stat.Account) + str += fmt.Sprintf(" [%s]%s[white]\n", color, stat.Account) } return str From 34b7ef1a6023a0651716069e2a1578fd5205d3f2 Mon Sep 17 00:00:00 2001 From: Chris Cummer Date: Thu, 4 Jul 2019 05:46:13 -0700 Subject: [PATCH 2/2] WTF-42 Add help tags to FeedReader settings --- modules/feedreader/settings.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/feedreader/settings.go b/modules/feedreader/settings.go index 154032e3..b6d45a86 100644 --- a/modules/feedreader/settings.go +++ b/modules/feedreader/settings.go @@ -14,8 +14,8 @@ const ( type Settings struct { common *cfg.Common - feeds []string - feedLimit int + feeds []string `help:"An array of RSS and Atom feed URLs"` + feedLimit int `help:"The maximum number of stories to display for each feed"` } // NewSettingsFromYAML creates a new settings instance from a YAML config block