Add twitter lurker

Signed-off-by: Etzelia <etzelia@hotmail.com>

.drone.yml

@@ -1,42 +0,0 @@
----
-kind: pipeline
-name: compliance
-trigger:
-  event:
-    - pull_request
-steps:
-  - name: build
-    pull: always
-    image: golang:1.15
-    commands:
-      - make test
-      - make build
-  - name: check
-    pull: always
-    image: golang:1.15
-    commands:
-      - make vet
-
----
-kind: pipeline
-name: release
-trigger:
-  event:
-    - push
-  branch:
-    - master
-steps:
-  - name: build
-    pull: always
-    image: golang:1.15
-    commands:
-      - make build
-  - name: gitea-release
-    pull: always
-    image: jolheiser/drone-gitea-main:latest
-    settings:
-      token:
-        from_secret: gitea_token
-      base: https://git.etztech.xyz
-      files:
-        - "lurk"

README.md

@@ -1,7 +1,5 @@
 # lurk
 
-### This repository is archived, development has moved to https://gitea.com/jolheiser/lurk
-
 A lurker used to notify a Discord channel via webhook 
 whenever a matching submission is made.
 

config/config.go

@@ -4,7 +4,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/pelletier/go-toml"
+	"github.com/BurntSushi/toml"
 	"go.jolheiser.com/beaver"
 )
 
@@ -22,50 +22,45 @@ func (c *Config) loadReddit() {
 		if sub.BodyLimit == 0 || sub.BodyLimit > 2045 {
 			sub.BodyLimit = 2045
 		}
-		sub.FlairAllowlistRe = make([]*regexp.Regexp, len(sub.FlairAllowlist))
+		sub.FlairWhitelistRe = make([]*regexp.Regexp, len(sub.FlairWhitelist))
-		for idx, f := range sub.FlairAllowlist {
+		for idx, f := range sub.FlairWhitelist {
-			sub.FlairAllowlistRe[idx] = regexp.MustCompile(f)
+			sub.FlairWhitelistRe[idx] = regexp.MustCompile(f)
 		}
-		sub.FlairBlocklistRe = make([]*regexp.Regexp, len(sub.FlairBlocklist))
+		sub.FlairBlacklistRe = make([]*regexp.Regexp, len(sub.FlairBlacklist))
-		for idx, f := range sub.FlairBlocklist {
+		for idx, f := range sub.FlairBlacklist {
-			sub.FlairBlocklistRe[idx] = regexp.MustCompile(f)
+			sub.FlairBlacklistRe[idx] = regexp.MustCompile(f)
 		}
-		sub.TitleAllowlistRe = make([]*regexp.Regexp, len(sub.TitleAllowlist))
+		sub.TitleWhitelistRe = make([]*regexp.Regexp, len(sub.TitleWhitelist))
-		for idx, t := range sub.TitleAllowlist {
+		for idx, t := range sub.TitleWhitelist {
-			sub.TitleAllowlistRe[idx] = regexp.MustCompile(t)
+			sub.TitleWhitelistRe[idx] = regexp.MustCompile(t)
 		}
-		sub.TitleBlocklistRe = make([]*regexp.Regexp, len(sub.TitleBlocklist))
+		sub.TitleBlacklistRe = make([]*regexp.Regexp, len(sub.TitleBlacklist))
-		for idx, t := range sub.TitleBlocklist {
+		for idx, t := range sub.TitleBlacklist {
-			sub.TitleBlocklistRe[idx] = regexp.MustCompile(t)
+			sub.TitleBlacklistRe[idx] = regexp.MustCompile(t)
 		}
-		sub.BodyAllowlistRe = make([]*regexp.Regexp, len(sub.BodyAllowlist))
+		sub.BodyWhitelistRe = make([]*regexp.Regexp, len(sub.BodyWhitelist))
-		for idx, b := range sub.BodyAllowlist {
+		for idx, b := range sub.BodyWhitelist {
-			sub.BodyAllowlistRe[idx] = regexp.MustCompile(b)
+			sub.BodyWhitelistRe[idx] = regexp.MustCompile(b)
 		}
-		sub.BodyBlocklistRe = make([]*regexp.Regexp, len(sub.BodyBlocklist))
+		sub.BodyBlacklistRe = make([]*regexp.Regexp, len(sub.BodyBlacklist))
-		for idx, b := range sub.BodyBlocklist {
+		for idx, b := range sub.BodyBlacklist {
-			sub.BodyBlocklistRe[idx] = regexp.MustCompile(b)
+			sub.BodyBlacklistRe[idx] = regexp.MustCompile(b)
 		}
 	}
 }
 
 func Load(configPath string) (*Config, error) {
-	cfg := Config{
+	cfg := &Config{
 		Reddit: RedditConfig{
 			Map: make(map[string]*SubReddit),
 		},
 	}
 
-	tree, err := toml.LoadFile(configPath)
+	if _, err := toml.DecodeFile(configPath, &cfg); err != nil {
-	if err != nil {
-		return nil, err
-	}
-
-	if err := tree.Unmarshal(&cfg); err != nil {
 		return nil, err
 	}
 
 	cfg.loadReddit()
 	beaver.Debug(cfg)
-	return &cfg, nil
+	return cfg, nil
 }

config/reddit.go

@@ -23,19 +23,19 @@ type RedditConfig struct {
 type SubReddit struct {
 	Name             string           `toml:"name"`
 	IconURL          string           `toml:"icon_url"`
-	FlairAllowlist   []string         `toml:"flair_allowlist"`
+	FlairWhitelist   []string         `toml:"flair_whitelist"`
-	FlairAllowlistRe []*regexp.Regexp `toml:"-"`
+	FlairWhitelistRe []*regexp.Regexp `toml:"-"`
-	FlairBlocklist   []string         `toml:"flair_blocklist"`
+	FlairBlacklist   []string         `toml:"flair_blacklist"`
-	FlairBlocklistRe []*regexp.Regexp `toml:"-"`
+	FlairBlacklistRe []*regexp.Regexp `toml:"-"`
-	TitleAllowlist   []string         `toml:"title_allowlist"`
+	TitleWhitelist   []string         `toml:"title_whitelist"`
-	TitleAllowlistRe []*regexp.Regexp `toml:"-"`
+	TitleWhitelistRe []*regexp.Regexp `toml:"-"`
-	TitleBlocklist   []string         `toml:"title_blocklist"`
+	TitleBlacklist   []string         `toml:"title_blacklist"`
-	TitleBlocklistRe []*regexp.Regexp `toml:"-"`
+	TitleBlacklistRe []*regexp.Regexp `toml:"-"`
 	TitleLimit       int              `toml:"title_limit"`
-	BodyAllowlist    []string         `toml:"body_allowlist"`
+	BodyWhitelist    []string         `toml:"body_whitelist"`
-	BodyAllowlistRe  []*regexp.Regexp `toml:"-"`
+	BodyWhitelistRe  []*regexp.Regexp `toml:"-"`
-	BodyBlocklist    []string         `toml:"body_blocklist"`
+	BodyBlacklist    []string         `toml:"body_blacklist"`
-	BodyBlocklistRe  []*regexp.Regexp `toml:"-"`
+	BodyBlacklistRe  []*regexp.Regexp `toml:"-"`
 	BodyLimit        int              `toml:"body_limit"`
 	Webhook          string           `toml:"webhook"`
 }

config/twitter.go

@@ -11,7 +11,6 @@ type TwitterConfig struct {
 
 type Filter struct {
 	Follows   []string `toml:"follows"`
-	FollowStrict bool     `toml:"follow_strict"`
 	Locations []string `toml:"locations"`
 	Tracks    []string `toml:"tracks"`
 	Webhook   string   `toml:"webhook"`

go.mod

@@ -1,12 +1,11 @@
 module go.etztech.xyz/lurk
 
-go 1.16
+go 1.14
 
 require (
+	github.com/BurntSushi/toml v0.3.1
 	github.com/dghubble/go-twitter v0.0.0-20200725221434-4bc8ad7ad1b4
 	github.com/dghubble/oauth1 v0.6.0
-	github.com/pelletier/go-toml v1.8.1
 	github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0
 	go.jolheiser.com/beaver v1.0.2
-	go.jolheiser.com/disco v0.0.2
 )

go.sum

@@ -1,9 +1,9 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY=
 github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dghubble/go-twitter v0.0.0-20200725221434-4bc8ad7ad1b4 h1:I60CX3+rWlBGPXR13jwxxBWB2GhlhZkEEh5o1eDLzJg=
 github.com/dghubble/go-twitter v0.0.0-20200725221434-4bc8ad7ad1b4/go.mod h1:xfg4uS5LEzOj8PgZV7SQYRHbG7jPUnelEiaAVJxmhJE=
 github.com/dghubble/oauth1 v0.6.0 h1:m1yC01Ohc/eF38jwZ8JUjL1a+XHHXtGQgK+MxQbmSx0=
@@ -19,12 +19,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
-github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0 h1:ss0FREpyYvw5V9t5XWWWIvUJnRgWFzdftvCXd8WQNhU=
 github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0/go.mod h1:aAkq4I/q1izZSSwHvzhDn9NA+eGxgTSuibwP3MZRlQY=
@@ -32,8 +28,6 @@ github.com/turnage/redditproto v0.0.0-20151223012412-afedf1b6eddb h1:qR56NGRvs2h
 github.com/turnage/redditproto v0.0.0-20151223012412-afedf1b6eddb/go.mod h1:GyqJdEoZSNoxKDb7Z2Lu/bX63jtFukwpaTP9ZIS5Ei0=
 go.jolheiser.com/beaver v1.0.2 h1:KA2D6iO8MQhZi1nZYi/Chak/f1Cxfrs6b1XO623+Khk=
 go.jolheiser.com/beaver v1.0.2/go.mod h1:7X4F5+XOGSC3LejTShoBdqtRCnPWcnRgmYGmG3EKW8g=
-go.jolheiser.com/disco v0.0.2 h1:UGYNqO7NQSBGB/OoS9WE5o/jYvmx1G0Bq3qQRM42Bkw=
-go.jolheiser.com/disco v0.0.2/go.mod h1:tY3HkJmMrzXH/bPgDWKHn1DUzDxkemD80OHLgHSA5uQ=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

handler/reddit.go

@@ -1,7 +1,8 @@
 package handler
 
 import (
-	"context"
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"regexp"
@@ -12,11 +13,30 @@ import (
 
 	"github.com/turnage/graw/reddit"
 	"go.jolheiser.com/beaver"
-	"go.jolheiser.com/disco"
 )
 
 var httpClient = &http.Client{Timeout: time.Minute}
 
+type Webhook struct {
+	Username  string  `json:"username"`
+	AvatarURL string  `json:"avatar_url"`
+	Embeds    []Embed `json:"embeds"`
+}
+
+type Embed struct {
+	Title       string    `json:"title"`
+	URL         string    `json:"url"`
+	Description string    `json:"description"`
+	Color       int64     `json:"color"`
+	Timestamp   time.Time `json:"timestamp"`
+	Author      Author    `json:"author"`
+}
+
+type Author struct {
+	Name string `json:"name"`
+	URL  string `json:"url"`
+}
+
 type Reddit struct {
 	Config *config.Config
 }
@@ -38,36 +58,37 @@ func (r *Reddit) Post(p *reddit.Post) error {
 		description = description[:sub.BodyLimit] + "..."
 	}
 
-	e := disco.Webhook{
+	e := Webhook{
 		Username:  "/r/" + p.Subreddit,
 		AvatarURL: sub.IconURL,
-		Embeds: []*disco.Embed{
+		Embeds: []Embed{
 			{
 				Title:       title,
 				URL:         p.URL,
 				Description: description,
-				Color:       0x007D96,
+				Color:       16312092, // Yellow
-				Timestamp:   disco.Now(),
+				Timestamp:   time.Now(),
-				Author: &disco.Author{
+				Author: Author{
 					Name: "/u/" + p.Author,
 					URL:  fmt.Sprintf("https://reddit.com/user/%s", p.Author),
 				},
 			},
 		},
 	}
+	data, err := json.Marshal(e)
+	if err != nil {
+		beaver.Error(err)
+		return nil
+	}
+	beaver.Debug(string(data))
 
 	if sub.Webhook == "" {
 		beaver.Errorf("no webhook for %s", p.Subreddit)
 		return nil
 	}
 
-	req, err := e.Request(context.Background(), sub.Webhook)
+	payload := bytes.NewBuffer(data)
-	if err != nil {
+	resp, err := httpClient.Post(sub.Webhook, "application/json", payload)
-		beaver.Error(err)
-		return nil
-	}
-
-	resp, err := httpClient.Do(req)
 	if err != nil {
 		beaver.Error(err)
 		return nil
@@ -84,29 +105,29 @@ func (r *Reddit) Post(p *reddit.Post) error {
 func checkPost(c *config.Config, p *reddit.Post) error {
 	sub := c.Reddit.Map[strings.ToLower(p.Subreddit)]
 
-	// Check blocklist first
+	// Check blacklist first
 	// Any match means we ignore
-	if matchesAny(p.LinkFlairText, sub.FlairBlocklistRe) {
+	if matchesAny(p.LinkFlairText, sub.FlairBlacklistRe) {
-		return fmt.Errorf("flair matched blocklisted regex: %s", p.LinkFlairText)
+		return fmt.Errorf("flair matched blacklisted regex: %s", p.LinkFlairText)
 	}
-	if matchesAny(p.Title, sub.TitleBlocklistRe) {
+	if matchesAny(p.Title, sub.TitleBlacklistRe) {
-		return fmt.Errorf("title matched blocklisted regex: %s", p.Title)
+		return fmt.Errorf("title matched blacklisted regex: %s", p.Title)
 	}
-	if matchesAny(p.SelfText, sub.BodyBlocklistRe) {
+	if matchesAny(p.SelfText, sub.BodyBlacklistRe) {
-		return fmt.Errorf("body matched blocklisted regex: %s", p.SelfText)
+		return fmt.Errorf("body matched blacklisted regex: %s", p.SelfText)
 	}
 
-	// Check allowlist
+	// Check whitelist
 	// Any match means we pass
-	// If no allowlist, pass
+	// If no whitelist, pass
-	if len(sub.FlairAllowlistRe) > 0 && !matchesAny(p.LinkFlairText, sub.FlairAllowlistRe) {
+	if len(sub.FlairWhitelistRe) > 0 && !matchesAny(p.LinkFlairText, sub.FlairWhitelistRe) {
-		return fmt.Errorf("flair didn't match any allowlisted regex: %s", p.LinkFlairText)
+		return fmt.Errorf("flair didn't match any whitelisted regex: %s", p.LinkFlairText)
 	}
-	if len(sub.TitleAllowlistRe) > 0 && !matchesAny(p.Title, sub.TitleAllowlistRe) {
+	if len(sub.TitleWhitelistRe) > 0 && !matchesAny(p.Title, sub.TitleWhitelistRe) {
-		return fmt.Errorf("title didn't match any allowlisted regex: %s", p.Title)
+		return fmt.Errorf("title didn't match any whitelisted regex: %s", p.Title)
 	}
-	if len(sub.BodyAllowlistRe) > 0 && !matchesAny(p.SelfText, sub.BodyAllowlistRe) {
+	if len(sub.BodyWhitelistRe) > 0 && !matchesAny(p.SelfText, sub.BodyWhitelistRe) {
-		return fmt.Errorf("body didn't match any allowlisted regex: %s", p.SelfText)
+		return fmt.Errorf("body didn't match any whitelisted regex: %s", p.SelfText)
 	}
 
 	return nil

handler/twitter.go

@@ -1,14 +1,15 @@
 package handler
 
 import (
-	"context"
+	"bytes"
+	"encoding/json"
 	"fmt"
+	"net/http"
 
 	"go.etztech.xyz/lurk/config"
 
 	"github.com/dghubble/go-twitter/twitter"
 	"go.jolheiser.com/beaver"
-	"go.jolheiser.com/disco"
 )
 
 type Twitter struct {
@@ -19,39 +20,41 @@ type Twitter struct {
 func (t *Twitter) Run() {
 	beaver.Debugf("setting up stream for %v", t.Filter)
 	demux := twitter.NewSwitchDemux()
-	demux.Tweet = t.Tweet
+	demux.Tweet = func(tweet *twitter.Tweet) {
+		beaver.Debugf("new tweet for %v", t.Filter)
+		if err := t.Discord(tweet); err != nil {
+			beaver.Error(err)
+		}
+	}
 
 	beaver.Debugf("streaming %v", t.Filter)
 	demux.HandleChan(t.Stream.Messages)
-	beaver.Debugf("disconnected from stream: %v", t.Filter)
+	beaver.Debugf("diconnected from stream: %v", t.Filter)
 }
 
-func (t *Twitter) Tweet(tweet *twitter.Tweet) {
+func (t *Twitter) Discord(tweet *twitter.Tweet) error {
-	beaver.Debugf("new tweet for %v", t.Filter)
+	e := map[string]string{
+		"content": fmt.Sprintf("https://twitter.com/%d/status/%d", tweet.User.ID, tweet.ID),
+	}
+	data, err := json.Marshal(e)
+	if err != nil {
+		return err
+	}
+	beaver.Debug(string(data))
 
-	if t.Filter.FollowStrict {
+	if t.Filter.Webhook == "" {
-		if tweet.InReplyToStatusIDStr != "" {
+		return fmt.Errorf("no webhook for %s", t.Filter)
-			beaver.Debug("tweet is a reply")
-			return
-		}
-		var match bool
-		for _, id := range t.Filter.Follows {
-			if id == tweet.User.IDStr {
-				match = true
-				break
-			}
-		}
-		if !match {
-			beaver.Debug("tweet did not match any follow IDs")
-			return
-		}
 	}
 
-	w := &disco.Webhook{
+	payload := bytes.NewBuffer(data)
-		Username: tweet.User.Name,
+	resp, err := httpClient.Post(t.Filter.Webhook, "application/json", payload)
-		Content:  fmt.Sprintf("https://twitter.com/%d/status/%d", tweet.User.ID, tweet.ID),
+	if err != nil {
+		return err
 	}
-	if _, err := w.Send(context.Background(), t.Filter.Webhook); err != nil {
+
-		beaver.Error(err)
+	if resp.StatusCode != http.StatusNoContent {
+		return err
 	}
+
+	return nil
 }

lurk.sample.toml

@@ -13,17 +13,17 @@
     password = ""
 
     # A list of subreddites to monitor
-    # allow/blocklist can be omitted in real config if empty
+    # white/blacklist can be omitted in real config if empty
     [[reddit.sub]]
     name = ""
     icon_url = ""
-    flair_allowlist = []
+    flair_whitelist = []
-    flair_blocklist = []
+    flair_blacklist = []
-    title_allowlist = []
+    title_whitelist = []
-    title_blocklist = []
+    title_blacklist = []
     title_limit = 0
-    body_allowlist = []
+    body_whitelist = []
-    body_blocklist = []
+    body_blacklist = []
     body_limit = 0
     webhook = ""
 
@@ -41,8 +41,6 @@
     # follows must use a Twitter user's ID
     # https://tweeterid.com/
     follows = []
-    # strict mode means only original tweets will be ingested
-    follow_strict = false
     locations = []
     tracks = []
     webhook = ""