Add twitter lurker (#2)

Add twitter lurker

Signed-off-by: Etzelia <etzelia@hotmail.com>

Reviewed-on: https://git.etztech.xyz/Etzelia/lurk/pulls/2

README.md

@@ -1,6 +1,6 @@
 # lurk
 
-A reddit lurker used to notify a Discord channel via webhook 
+A lurker used to notify a Discord channel via webhook 
 whenever a matching submission is made.
 
 See the [example config](lurk.sample.toml).

config.go

@@ -1,107 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"regexp"
-	"strings"
-
-	"github.com/BurntSushi/toml"
-	"go.jolheiser.com/beaver"
-)
-
-type Config struct {
-	SubReddits []*SubReddit          `toml:"subreddit"`
-	Map        map[string]*SubReddit `toml:"-"`
-
-	// Agent file
-	AppName string `toml:"app_name"`
-	Version string `toml:"version"`
-
-	ClientID     string `toml:"client_id"`
-	ClientSecret string `toml:"client_secret"`
-
-	Username string `toml:"username"`
-	Password string `toml:"password"`
-}
-
-type SubReddit struct {
-	Name             string           `toml:"name"`
-	IconURL          string           `toml:"icon_url"`
-	FlairWhitelist   []string         `toml:"flair_whitelist"`
-	FlairWhitelistRe []*regexp.Regexp `toml:"-"`
-	FlairBlacklist   []string         `toml:"flair_blacklist"`
-	FlairBlacklistRe []*regexp.Regexp `toml:"-"`
-	TitleWhitelist   []string         `toml:"title_whitelist"`
-	TitleWhitelistRe []*regexp.Regexp `toml:"-"`
-	TitleBlacklist   []string         `toml:"title_blacklist"`
-	TitleBlacklistRe []*regexp.Regexp `toml:"-"`
-	TitleLimit       int              `toml:"title_limit"`
-	BodyWhitelist    []string         `toml:"body_whitelist"`
-	BodyWhitelistRe  []*regexp.Regexp `toml:"-"`
-	BodyBlacklist    []string         `toml:"body_blacklist"`
-	BodyBlacklistRe  []*regexp.Regexp `toml:"-"`
-	BodyLimit        int              `toml:"body_limit"`
-	Webhook          string           `toml:"webhook"`
-}
-
-func (c *Config) UserAgent() string {
-	return fmt.Sprintf("%s/%s by /u/%s", c.AppName, c.Version, c.Username)
-}
-
-func (c *Config) SubRedditNames() []string {
-	names := make([]string, len(c.SubReddits))
-	for idx, sub := range c.SubReddits {
-		names[idx] = sub.Name
-	}
-	return names
-}
-
-func (c *Config) load() {
-	for _, sub := range c.SubReddits {
-		c.Map[strings.ToLower(sub.Name)] = sub
-		if sub.TitleLimit == 0 || sub.TitleLimit > 253 {
-			sub.TitleLimit = 253
-		}
-		if sub.BodyLimit == 0 || sub.BodyLimit > 2045 {
-			sub.BodyLimit = 2045
-		}
-		sub.FlairWhitelistRe = make([]*regexp.Regexp, len(sub.FlairWhitelist))
-		for idx, f := range sub.FlairWhitelist {
-			sub.FlairWhitelistRe[idx] = regexp.MustCompile(f)
-		}
-		sub.FlairBlacklistRe = make([]*regexp.Regexp, len(sub.FlairBlacklist))
-		for idx, f := range sub.FlairBlacklist {
-			sub.FlairBlacklistRe[idx] = regexp.MustCompile(f)
-		}
-		sub.TitleWhitelistRe = make([]*regexp.Regexp, len(sub.TitleWhitelist))
-		for idx, t := range sub.TitleWhitelist {
-			sub.TitleWhitelistRe[idx] = regexp.MustCompile(t)
-		}
-		sub.TitleBlacklistRe = make([]*regexp.Regexp, len(sub.TitleBlacklist))
-		for idx, t := range sub.TitleBlacklist {
-			sub.TitleBlacklistRe[idx] = regexp.MustCompile(t)
-		}
-		sub.BodyWhitelistRe = make([]*regexp.Regexp, len(sub.BodyWhitelist))
-		for idx, b := range sub.BodyWhitelist {
-			sub.BodyWhitelistRe[idx] = regexp.MustCompile(b)
-		}
-		sub.BodyBlacklistRe = make([]*regexp.Regexp, len(sub.BodyBlacklist))
-		for idx, b := range sub.BodyBlacklist {
-			sub.BodyBlacklistRe[idx] = regexp.MustCompile(b)
-		}
-	}
-}
-
-func LoadConfig() (*Config, error) {
-	cfg := &Config{
-		Map: make(map[string]*SubReddit),
-	}
-
-	if _, err := toml.DecodeFile(configPath, &cfg); err != nil {
-		return nil, err
-	}
-
-	cfg.load()
-	beaver.Debug(cfg)
-	return cfg, nil
-}

config/config.go

@@ -0,0 +1,66 @@
+package config
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"go.jolheiser.com/beaver"
+)
+
+type Config struct {
+	Reddit  RedditConfig  `toml:"reddit"`
+	Twitter TwitterConfig `toml:"twitter"`
+}
+
+func (c *Config) loadReddit() {
+	for _, sub := range c.Reddit.SubReddits {
+		c.Reddit.Map[strings.ToLower(sub.Name)] = sub
+		if sub.TitleLimit == 0 || sub.TitleLimit > 253 {
+			sub.TitleLimit = 253
+		}
+		if sub.BodyLimit == 0 || sub.BodyLimit > 2045 {
+			sub.BodyLimit = 2045
+		}
+		sub.FlairWhitelistRe = make([]*regexp.Regexp, len(sub.FlairWhitelist))
+		for idx, f := range sub.FlairWhitelist {
+			sub.FlairWhitelistRe[idx] = regexp.MustCompile(f)
+		}
+		sub.FlairBlacklistRe = make([]*regexp.Regexp, len(sub.FlairBlacklist))
+		for idx, f := range sub.FlairBlacklist {
+			sub.FlairBlacklistRe[idx] = regexp.MustCompile(f)
+		}
+		sub.TitleWhitelistRe = make([]*regexp.Regexp, len(sub.TitleWhitelist))
+		for idx, t := range sub.TitleWhitelist {
+			sub.TitleWhitelistRe[idx] = regexp.MustCompile(t)
+		}
+		sub.TitleBlacklistRe = make([]*regexp.Regexp, len(sub.TitleBlacklist))
+		for idx, t := range sub.TitleBlacklist {
+			sub.TitleBlacklistRe[idx] = regexp.MustCompile(t)
+		}
+		sub.BodyWhitelistRe = make([]*regexp.Regexp, len(sub.BodyWhitelist))
+		for idx, b := range sub.BodyWhitelist {
+			sub.BodyWhitelistRe[idx] = regexp.MustCompile(b)
+		}
+		sub.BodyBlacklistRe = make([]*regexp.Regexp, len(sub.BodyBlacklist))
+		for idx, b := range sub.BodyBlacklist {
+			sub.BodyBlacklistRe[idx] = regexp.MustCompile(b)
+		}
+	}
+}
+
+func Load(configPath string) (*Config, error) {
+	cfg := &Config{
+		Reddit: RedditConfig{
+			Map: make(map[string]*SubReddit),
+		},
+	}
+
+	if _, err := toml.DecodeFile(configPath, &cfg); err != nil {
+		return nil, err
+	}
+
+	cfg.loadReddit()
+	beaver.Debug(cfg)
+	return cfg, nil
+}

config/reddit.go

@@ -0,0 +1,53 @@
+package config
+
+import (
+	"fmt"
+	"regexp"
+)
+
+type RedditConfig struct {
+	SubReddits []*SubReddit          `toml:"sub"`
+	Map        map[string]*SubReddit `toml:"-"`
+
+	// Agent file
+	AppName string `toml:"app_name"`
+	Version string `toml:"version"`
+
+	ClientID     string `toml:"client_id"`
+	ClientSecret string `toml:"client_secret"`
+
+	Username string `toml:"username"`
+	Password string `toml:"password"`
+}
+
+type SubReddit struct {
+	Name             string           `toml:"name"`
+	IconURL          string           `toml:"icon_url"`
+	FlairWhitelist   []string         `toml:"flair_whitelist"`
+	FlairWhitelistRe []*regexp.Regexp `toml:"-"`
+	FlairBlacklist   []string         `toml:"flair_blacklist"`
+	FlairBlacklistRe []*regexp.Regexp `toml:"-"`
+	TitleWhitelist   []string         `toml:"title_whitelist"`
+	TitleWhitelistRe []*regexp.Regexp `toml:"-"`
+	TitleBlacklist   []string         `toml:"title_blacklist"`
+	TitleBlacklistRe []*regexp.Regexp `toml:"-"`
+	TitleLimit       int              `toml:"title_limit"`
+	BodyWhitelist    []string         `toml:"body_whitelist"`
+	BodyWhitelistRe  []*regexp.Regexp `toml:"-"`
+	BodyBlacklist    []string         `toml:"body_blacklist"`
+	BodyBlacklistRe  []*regexp.Regexp `toml:"-"`
+	BodyLimit        int              `toml:"body_limit"`
+	Webhook          string           `toml:"webhook"`
+}
+
+func (r *RedditConfig) UserAgent() string {
+	return fmt.Sprintf("%s/%s by /u/%s", r.AppName, r.Version, r.Username)
+}
+
+func (r *RedditConfig) SubRedditNames() []string {
+	names := make([]string, len(r.SubReddits))
+	for idx, sub := range r.SubReddits {
+		names[idx] = sub.Name
+	}
+	return names
+}

config/twitter.go

@@ -0,0 +1,17 @@
+package config
+
+type TwitterConfig struct {
+	ConsumerKey    string `toml:"consumer_key"`
+	ConsumerSecret string `toml:"consumer_secret"`
+	AccessToken    string `toml:"access_token"`
+	AccessSecret   string `toml:"access_secret"`
+
+	Filters []Filter `toml:"filter"`
+}
+
+type Filter struct {
+	Follows   []string `toml:"follows"`
+	Locations []string `toml:"locations"`
+	Tracks    []string `toml:"tracks"`
+	Webhook   string   `toml:"webhook"`
+}

go.mod

@@ -4,6 +4,8 @@ go 1.14
 
 require (
 	github.com/BurntSushi/toml v0.3.1
+	github.com/dghubble/go-twitter v0.0.0-20200725221434-4bc8ad7ad1b4
+	github.com/dghubble/oauth1 v0.6.0
 	github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0
 	go.jolheiser.com/beaver v1.0.2
 )

go.sum

@@ -1,13 +1,27 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY=
+github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dghubble/go-twitter v0.0.0-20200725221434-4bc8ad7ad1b4 h1:I60CX3+rWlBGPXR13jwxxBWB2GhlhZkEEh5o1eDLzJg=
+github.com/dghubble/go-twitter v0.0.0-20200725221434-4bc8ad7ad1b4/go.mod h1:xfg4uS5LEzOj8PgZV7SQYRHbG7jPUnelEiaAVJxmhJE=
+github.com/dghubble/oauth1 v0.6.0 h1:m1yC01Ohc/eF38jwZ8JUjL1a+XHHXtGQgK+MxQbmSx0=
+github.com/dghubble/oauth1 v0.6.0/go.mod h1:8pFdfPkv/jr8mkChVbNVuJ0suiHe278BtWI4Tk1ujxk=
+github.com/dghubble/sling v1.3.0 h1:pZHjCJq4zJvc6qVQ5wN1jo5oNZlNE0+8T/h0XeXBUKU=
+github.com/dghubble/sling v1.3.0/go.mod h1:XXShWaBWKzNLhu2OxikSNFrlsvowtz4kyRuXUG7oQKY=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0 h1:ss0FREpyYvw5V9t5XWWWIvUJnRgWFzdftvCXd8WQNhU=
 github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0/go.mod h1:aAkq4I/q1izZSSwHvzhDn9NA+eGxgTSuibwP3MZRlQY=
 github.com/turnage/redditproto v0.0.0-20151223012412-afedf1b6eddb h1:qR56NGRvs2hTUbkn6QF8bEJzxPIoMw3Np3UigBeJO5A=

handler/reddit.go

@@ -1,4 +1,4 @@
-package main
+package handler
 
 import (
 	"bytes"
@@ -9,6 +9,8 @@ import (
 	"strings"
 	"time"
 
+	"go.etztech.xyz/lurk/config"
+
 	"github.com/turnage/graw/reddit"
 	"go.jolheiser.com/beaver"
 )
@@ -35,14 +37,14 @@ type Author struct {
 	URL  string `json:"url"`
 }
 
-type lurker struct {
+type Reddit struct {
-	config *Config
+	Config *config.Config
 }
 
-func (l *lurker) Post(p *reddit.Post) error {
+func (r *Reddit) Post(p *reddit.Post) error {
-	sub := l.config.Map[strings.ToLower(p.Subreddit)]
+	sub := r.Config.Reddit.Map[strings.ToLower(p.Subreddit)]
 
-	if err := checkPost(l.config, p); err != nil {
+	if err := checkPost(r.Config, p); err != nil {
 		beaver.Debugf("%s: %v", p.Subreddit, err)
 		return nil
 	}
@@ -100,8 +102,8 @@ func (l *lurker) Post(p *reddit.Post) error {
 	return nil
 }
 
-func checkPost(c *Config, p *reddit.Post) error {
+func checkPost(c *config.Config, p *reddit.Post) error {
-	sub := c.Map[strings.ToLower(p.Subreddit)]
+	sub := c.Reddit.Map[strings.ToLower(p.Subreddit)]
 
 	// Check blacklist first
 	// Any match means we ignore

handler/twitter.go

@@ -0,0 +1,60 @@
+package handler
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"go.etztech.xyz/lurk/config"
+
+	"github.com/dghubble/go-twitter/twitter"
+	"go.jolheiser.com/beaver"
+)
+
+type Twitter struct {
+	Filter config.Filter
+	Stream *twitter.Stream
+}
+
+func (t *Twitter) Run() {
+	beaver.Debugf("setting up stream for %v", t.Filter)
+	demux := twitter.NewSwitchDemux()
+	demux.Tweet = func(tweet *twitter.Tweet) {
+		beaver.Debugf("new tweet for %v", t.Filter)
+		if err := t.Discord(tweet); err != nil {
+			beaver.Error(err)
+		}
+	}
+
+	beaver.Debugf("streaming %v", t.Filter)
+	demux.HandleChan(t.Stream.Messages)
+	beaver.Debugf("diconnected from stream: %v", t.Filter)
+}
+
+func (t *Twitter) Discord(tweet *twitter.Tweet) error {
+	e := map[string]string{
+		"content": fmt.Sprintf("https://twitter.com/%d/status/%d", tweet.User.ID, tweet.ID),
+	}
+	data, err := json.Marshal(e)
+	if err != nil {
+		return err
+	}
+	beaver.Debug(string(data))
+
+	if t.Filter.Webhook == "" {
+		return fmt.Errorf("no webhook for %s", t.Filter)
+	}
+
+	payload := bytes.NewBuffer(data)
+	resp, err := httpClient.Post(t.Filter.Webhook, "application/json", payload)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusNoContent {
+		return err
+	}
+
+	return nil
+}

lurk.sample.toml

@@ -1,3 +1,5 @@
+# Reddit information
+[reddit]
     # Reddit Agent Info
     app_name = "Lurk"
     version = "0.1"
@@ -7,40 +9,14 @@ client_id = ""
     client_secret = ""
 
     # Reddit credentials
-username = "24CC-Official"
+    username = ""
     password = ""
 
     # A list of subreddites to monitor
-# Empty fields are just for examples, they can be omitted in a real config
+    # white/blacklist can be omitted in real config if empty
-[[subreddit]]
+    [[reddit.sub]]
-name = "mcservers"
+    name = ""
-icon_url = "https://styles.redditmedia.com/t5_2s3kg/styles/communityIcon_recbaq3ufjv01.png"
+    icon_url = ""
-flair_whitelist = []
-flair_blacklist = []
-title_whitelist = ["wanted"]
-title_blacklist = []
-title_limit = 0
-body_whitelist = []
-body_blacklist = []
-body_limit = 0
-webhook = ""
-
-[[subreddit]]
-name = "MinecraftBuddies"
-icon_url = "https://styles.redditmedia.com/t5_30nfs/styles/communityIcon_qcijaahdybz01.png"
-flair_whitelist = ["JAVA"]
-flair_blacklist = []
-title_whitelist = ["wanted", "vanilla"]
-title_blacklist = []
-title_limit = 0
-body_whitelist = []
-body_blacklist = []
-body_limit = 0
-webhook = ""
-
-[[subreddit]]
-name = "24CarrotCraft"
-icon_url = "https://styles.redditmedia.com/t5_3budc/styles/communityIcon_byu6k6o8omz01.png"
     flair_whitelist = []
     flair_blacklist = []
     title_whitelist = []
@@ -50,3 +26,21 @@ body_whitelist = []
     body_blacklist = []
     body_limit = 0
     webhook = ""
+
+# Twitter information
+[twitter]
+    # Auth
+    consumer_key = ""
+    consumer_secret = ""
+    access_token = ""
+    access_secret = ""
+
+    # A list of filters to watch for
+    # Empty fields can be omitted in a real config
+    [[twitter.filter]]
+    # follows must use a Twitter user's ID
+    # https://tweeterid.com/
+    follows = []
+    locations = []
+    tracks = []
+    webhook = ""

main.go

@@ -2,7 +2,15 @@ package main
 
 import (
 	"flag"
+	"os"
+	"os/signal"
+	"syscall"
 
+	"go.etztech.xyz/lurk/config"
+	"go.etztech.xyz/lurk/handler"
+
+	"github.com/dghubble/go-twitter/twitter"
+	"github.com/dghubble/oauth1"
 	"github.com/turnage/graw"
 	"github.com/turnage/graw/reddit"
 	"go.jolheiser.com/beaver"
@@ -29,31 +37,82 @@ func main() {
 		beaver.Console.Level = beaver.DEBUG
 	}
 
-	cfg, err := LoadConfig()
+	cfg, err := config.Load(configPath)
 	if err != nil {
 		beaver.Fatal(err)
 	}
 
+	// Reddit
+	go lurkReddit(cfg)
+
+	// Twitter
+	go lurkTwitter(cfg)
+
+	beaver.Info("Lurk is ready to start lurking!")
+	ch := make(chan os.Signal)
+	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
+	<-ch
+}
+
+func lurkReddit(cfg *config.Config) {
 	bot, err := reddit.NewBot(reddit.BotConfig{
-		Agent: cfg.UserAgent(),
+		Agent: cfg.Reddit.UserAgent(),
 		App: reddit.App{
-			ID:       cfg.ClientID,
+			ID:       cfg.Reddit.ClientID,
-			Secret:   cfg.ClientSecret,
+			Secret:   cfg.Reddit.ClientSecret,
-			Username: cfg.Username,
+			Username: cfg.Reddit.Username,
-			Password: cfg.Password,
+			Password: cfg.Reddit.Password,
 		},
 	})
 	if err != nil {
 		beaver.Fatal(err)
 	}
 
-	_, wait, err := graw.Run(&lurker{cfg}, bot, graw.Config{Subreddits: cfg.SubRedditNames()})
+	_, wait, err := graw.Run(&handler.Reddit{
+		Config: cfg,
+	}, bot, graw.Config{
+		Subreddits: cfg.Reddit.SubRedditNames(),
+	})
 	if err != nil {
-		beaver.Fatal(err)
+		beaver.Errorf("could not run reddit bot: %v", err)
+		return
 	}
-
-	beaver.Info("Lurk is ready to start lurking!")
 	if err := wait(); err != nil {
 		beaver.Fatal(err)
 	}
 }
+
+func lurkTwitter(cfg *config.Config) {
+	twitterConfig := oauth1.NewConfig(cfg.Twitter.ConsumerKey, cfg.Twitter.ConsumerSecret)
+	token := oauth1.NewToken(cfg.Twitter.AccessToken, cfg.Twitter.AccessSecret)
+
+	httpClient := twitterConfig.Client(oauth1.NoContext, token)
+	client := twitter.NewClient(httpClient)
+
+	// Just to test if we have valid auth
+	_, _, err := client.Timelines.HomeTimeline(&twitter.HomeTimelineParams{
+		Count: 1,
+	})
+	if err != nil {
+		beaver.Fatal(err)
+	}
+
+	for _, filter := range cfg.Twitter.Filters {
+		stream, err := client.Streams.Filter(&twitter.StreamFilterParams{
+			Follow:        filter.Follows,
+			Locations:     filter.Locations,
+			StallWarnings: twitter.Bool(false),
+			Track:         filter.Tracks,
+		})
+		if err != nil {
+			beaver.Fatal(err)
+		}
+
+		lurker := &handler.Twitter{
+			Filter: filter,
+			Stream: stream,
+		}
+
+		go lurker.Run()
+	}
+}