Fix format type

Signed-off-by: Etzelia <etzelia@hotmail.com>

Add strict mode

Signed-off-by: Etzelia <etzelia@hotmail.com>

Add Drone tests (#3)

Add Drone tests

Reviewed-on: https://git.etztech.xyz/Etzelia/lurk/pulls/3

Add twitter lurker (#2)

Add twitter lurker

Signed-off-by: Etzelia <etzelia@hotmail.com>

Reviewed-on: https://git.etztech.xyz/Etzelia/lurk/pulls/2

Reviewed-on: https://git.birbmc.com/BirbMC/lurk/pulls/1
Co-Authored-By: Etzelia <etzelia@hotmail.com>
Co-Committed-By: Etzelia <etzelia@hotmail.com>
master
Etzelia 2021-03-16 01:43:24 +00:00
parent 4ff23e52cf
commit b30baea549
12 changed files with 403 additions and 176 deletions

42
.drone.yml 100644
View File

@ -0,0 +1,42 @@
---
kind: pipeline
name: compliance
trigger:
event:
- pull_request
steps:
- name: build
pull: always
image: golang:1.15
commands:
- make test
- make build
- name: check
pull: always
image: golang:1.15
commands:
- make vet
---
kind: pipeline
name: release
trigger:
event:
- push
branch:
- master
steps:
- name: build
pull: always
image: golang:1.15
commands:
- make build
- name: gitea-release
pull: always
image: jolheiser/drone-gitea-main:latest
settings:
token:
from_secret: gitea_token
base: https://git.etztech.xyz
files:
- "lurk"

View File

@ -1,6 +1,6 @@
# lurk
A reddit lurker used to notify a Discord channel via webhook
A lurker used to notify a Discord channel via webhook
whenever a matching submission is made.
See the [example config](lurk.sample.toml).

107
config.go
View File

@ -1,107 +0,0 @@
package main
import (
"fmt"
"regexp"
"strings"
"github.com/BurntSushi/toml"
"go.jolheiser.com/beaver"
)
type Config struct {
SubReddits []*SubReddit `toml:"subreddit"`
Map map[string]*SubReddit `toml:"-"`
// Agent file
AppName string `toml:"app_name"`
Version string `toml:"version"`
ClientID string `toml:"client_id"`
ClientSecret string `toml:"client_secret"`
Username string `toml:"username"`
Password string `toml:"password"`
}
type SubReddit struct {
Name string `toml:"name"`
IconURL string `toml:"icon_url"`
FlairWhitelist []string `toml:"flair_whitelist"`
FlairWhitelistRe []*regexp.Regexp `toml:"-"`
FlairBlacklist []string `toml:"flair_blacklist"`
FlairBlacklistRe []*regexp.Regexp `toml:"-"`
TitleWhitelist []string `toml:"title_whitelist"`
TitleWhitelistRe []*regexp.Regexp `toml:"-"`
TitleBlacklist []string `toml:"title_blacklist"`
TitleBlacklistRe []*regexp.Regexp `toml:"-"`
TitleLimit int `toml:"title_limit"`
BodyWhitelist []string `toml:"body_whitelist"`
BodyWhitelistRe []*regexp.Regexp `toml:"-"`
BodyBlacklist []string `toml:"body_blacklist"`
BodyBlacklistRe []*regexp.Regexp `toml:"-"`
BodyLimit int `toml:"body_limit"`
Webhook string `toml:"webhook"`
}
func (c *Config) UserAgent() string {
return fmt.Sprintf("%s/%s by /u/%s", c.AppName, c.Version, c.Username)
}
func (c *Config) SubRedditNames() []string {
names := make([]string, len(c.SubReddits))
for idx, sub := range c.SubReddits {
names[idx] = sub.Name
}
return names
}
func (c *Config) load() {
for _, sub := range c.SubReddits {
c.Map[strings.ToLower(sub.Name)] = sub
if sub.TitleLimit == 0 || sub.TitleLimit > 253 {
sub.TitleLimit = 253
}
if sub.BodyLimit == 0 || sub.BodyLimit > 2045 {
sub.BodyLimit = 2045
}
sub.FlairWhitelistRe = make([]*regexp.Regexp, len(sub.FlairWhitelist))
for idx, f := range sub.FlairWhitelist {
sub.FlairWhitelistRe[idx] = regexp.MustCompile(f)
}
sub.FlairBlacklistRe = make([]*regexp.Regexp, len(sub.FlairBlacklist))
for idx, f := range sub.FlairBlacklist {
sub.FlairBlacklistRe[idx] = regexp.MustCompile(f)
}
sub.TitleWhitelistRe = make([]*regexp.Regexp, len(sub.TitleWhitelist))
for idx, t := range sub.TitleWhitelist {
sub.TitleWhitelistRe[idx] = regexp.MustCompile(t)
}
sub.TitleBlacklistRe = make([]*regexp.Regexp, len(sub.TitleBlacklist))
for idx, t := range sub.TitleBlacklist {
sub.TitleBlacklistRe[idx] = regexp.MustCompile(t)
}
sub.BodyWhitelistRe = make([]*regexp.Regexp, len(sub.BodyWhitelist))
for idx, b := range sub.BodyWhitelist {
sub.BodyWhitelistRe[idx] = regexp.MustCompile(b)
}
sub.BodyBlacklistRe = make([]*regexp.Regexp, len(sub.BodyBlacklist))
for idx, b := range sub.BodyBlacklist {
sub.BodyBlacklistRe[idx] = regexp.MustCompile(b)
}
}
}
func LoadConfig() (*Config, error) {
cfg := &Config{
Map: make(map[string]*SubReddit),
}
if _, err := toml.DecodeFile(configPath, &cfg); err != nil {
return nil, err
}
cfg.load()
beaver.Debug(cfg)
return cfg, nil
}

71
config/config.go 100644
View File

@ -0,0 +1,71 @@
package config
import (
"regexp"
"strings"
"github.com/pelletier/go-toml"
"go.jolheiser.com/beaver"
)
type Config struct {
Reddit RedditConfig `toml:"reddit"`
Twitter TwitterConfig `toml:"twitter"`
}
func (c *Config) loadReddit() {
for _, sub := range c.Reddit.SubReddits {
c.Reddit.Map[strings.ToLower(sub.Name)] = sub
if sub.TitleLimit == 0 || sub.TitleLimit > 253 {
sub.TitleLimit = 253
}
if sub.BodyLimit == 0 || sub.BodyLimit > 2045 {
sub.BodyLimit = 2045
}
sub.FlairWhitelistRe = make([]*regexp.Regexp, len(sub.FlairWhitelist))
for idx, f := range sub.FlairWhitelist {
sub.FlairWhitelistRe[idx] = regexp.MustCompile(f)
}
sub.FlairBlacklistRe = make([]*regexp.Regexp, len(sub.FlairBlacklist))
for idx, f := range sub.FlairBlacklist {
sub.FlairBlacklistRe[idx] = regexp.MustCompile(f)
}
sub.TitleWhitelistRe = make([]*regexp.Regexp, len(sub.TitleWhitelist))
for idx, t := range sub.TitleWhitelist {
sub.TitleWhitelistRe[idx] = regexp.MustCompile(t)
}
sub.TitleBlacklistRe = make([]*regexp.Regexp, len(sub.TitleBlacklist))
for idx, t := range sub.TitleBlacklist {
sub.TitleBlacklistRe[idx] = regexp.MustCompile(t)
}
sub.BodyWhitelistRe = make([]*regexp.Regexp, len(sub.BodyWhitelist))
for idx, b := range sub.BodyWhitelist {
sub.BodyWhitelistRe[idx] = regexp.MustCompile(b)
}
sub.BodyBlacklistRe = make([]*regexp.Regexp, len(sub.BodyBlacklist))
for idx, b := range sub.BodyBlacklist {
sub.BodyBlacklistRe[idx] = regexp.MustCompile(b)
}
}
}
func Load(configPath string) (*Config, error) {
cfg := Config{
Reddit: RedditConfig{
Map: make(map[string]*SubReddit),
},
}
tree, err := toml.LoadFile(configPath)
if err != nil {
return nil, err
}
if err := tree.Unmarshal(&cfg); err != nil {
return nil, err
}
cfg.loadReddit()
beaver.Debug(cfg)
return &cfg, nil
}

53
config/reddit.go 100644
View File

@ -0,0 +1,53 @@
package config
import (
"fmt"
"regexp"
)
type RedditConfig struct {
SubReddits []*SubReddit `toml:"sub"`
Map map[string]*SubReddit `toml:"-"`
// Agent file
AppName string `toml:"app_name"`
Version string `toml:"version"`
ClientID string `toml:"client_id"`
ClientSecret string `toml:"client_secret"`
Username string `toml:"username"`
Password string `toml:"password"`
}
type SubReddit struct {
Name string `toml:"name"`
IconURL string `toml:"icon_url"`
FlairWhitelist []string `toml:"flair_whitelist"`
FlairWhitelistRe []*regexp.Regexp `toml:"-"`
FlairBlacklist []string `toml:"flair_blacklist"`
FlairBlacklistRe []*regexp.Regexp `toml:"-"`
TitleWhitelist []string `toml:"title_whitelist"`
TitleWhitelistRe []*regexp.Regexp `toml:"-"`
TitleBlacklist []string `toml:"title_blacklist"`
TitleBlacklistRe []*regexp.Regexp `toml:"-"`
TitleLimit int `toml:"title_limit"`
BodyWhitelist []string `toml:"body_whitelist"`
BodyWhitelistRe []*regexp.Regexp `toml:"-"`
BodyBlacklist []string `toml:"body_blacklist"`
BodyBlacklistRe []*regexp.Regexp `toml:"-"`
BodyLimit int `toml:"body_limit"`
Webhook string `toml:"webhook"`
}
func (r *RedditConfig) UserAgent() string {
return fmt.Sprintf("%s/%s by /u/%s", r.AppName, r.Version, r.Username)
}
func (r *RedditConfig) SubRedditNames() []string {
names := make([]string, len(r.SubReddits))
for idx, sub := range r.SubReddits {
names[idx] = sub.Name
}
return names
}

18
config/twitter.go 100644
View File

@ -0,0 +1,18 @@
package config
type TwitterConfig struct {
ConsumerKey string `toml:"consumer_key"`
ConsumerSecret string `toml:"consumer_secret"`
AccessToken string `toml:"access_token"`
AccessSecret string `toml:"access_secret"`
Filters []Filter `toml:"filter"`
}
type Filter struct {
Follows []string `toml:"follows"`
FollowStrict bool `toml:"follow_strict"`
Locations []string `toml:"locations"`
Tracks []string `toml:"tracks"`
Webhook string `toml:"webhook"`
}

4
go.mod
View File

@ -3,7 +3,9 @@ module go.etztech.xyz/lurk
go 1.14
require (
github.com/BurntSushi/toml v0.3.1
github.com/dghubble/go-twitter v0.0.0-20200725221434-4bc8ad7ad1b4
github.com/dghubble/oauth1 v0.6.0
github.com/pelletier/go-toml v1.8.1
github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0
go.jolheiser.com/beaver v1.0.2
)

22
go.sum
View File

@ -1,13 +1,31 @@
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY=
github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dghubble/go-twitter v0.0.0-20200725221434-4bc8ad7ad1b4 h1:I60CX3+rWlBGPXR13jwxxBWB2GhlhZkEEh5o1eDLzJg=
github.com/dghubble/go-twitter v0.0.0-20200725221434-4bc8ad7ad1b4/go.mod h1:xfg4uS5LEzOj8PgZV7SQYRHbG7jPUnelEiaAVJxmhJE=
github.com/dghubble/oauth1 v0.6.0 h1:m1yC01Ohc/eF38jwZ8JUjL1a+XHHXtGQgK+MxQbmSx0=
github.com/dghubble/oauth1 v0.6.0/go.mod h1:8pFdfPkv/jr8mkChVbNVuJ0suiHe278BtWI4Tk1ujxk=
github.com/dghubble/sling v1.3.0 h1:pZHjCJq4zJvc6qVQ5wN1jo5oNZlNE0+8T/h0XeXBUKU=
github.com/dghubble/sling v1.3.0/go.mod h1:XXShWaBWKzNLhu2OxikSNFrlsvowtz4kyRuXUG7oQKY=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0 h1:ss0FREpyYvw5V9t5XWWWIvUJnRgWFzdftvCXd8WQNhU=
github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0/go.mod h1:aAkq4I/q1izZSSwHvzhDn9NA+eGxgTSuibwP3MZRlQY=
github.com/turnage/redditproto v0.0.0-20151223012412-afedf1b6eddb h1:qR56NGRvs2hTUbkn6QF8bEJzxPIoMw3Np3UigBeJO5A=

View File

@ -1,4 +1,4 @@
package main
package handler
import (
"bytes"
@ -9,6 +9,8 @@ import (
"strings"
"time"
"go.etztech.xyz/lurk/config"
"github.com/turnage/graw/reddit"
"go.jolheiser.com/beaver"
)
@ -35,14 +37,14 @@ type Author struct {
URL string `json:"url"`
}
type lurker struct {
config *Config
type Reddit struct {
Config *config.Config
}
func (l *lurker) Post(p *reddit.Post) error {
sub := l.config.Map[strings.ToLower(p.Subreddit)]
func (r *Reddit) Post(p *reddit.Post) error {
sub := r.Config.Reddit.Map[strings.ToLower(p.Subreddit)]
if err := checkPost(l.config, p); err != nil {
if err := checkPost(r.Config, p); err != nil {
beaver.Debugf("%s: %v", p.Subreddit, err)
return nil
}
@ -100,8 +102,8 @@ func (l *lurker) Post(p *reddit.Post) error {
return nil
}
func checkPost(c *Config, p *reddit.Post) error {
sub := c.Map[strings.ToLower(p.Subreddit)]
func checkPost(c *config.Config, p *reddit.Post) error {
sub := c.Reddit.Map[strings.ToLower(p.Subreddit)]
// Check blacklist first
// Any match means we ignore

73
handler/twitter.go 100644
View File

@ -0,0 +1,73 @@
package handler
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"go.etztech.xyz/lurk/config"
"github.com/dghubble/go-twitter/twitter"
"go.jolheiser.com/beaver"
)
type Twitter struct {
Filter config.Filter
Stream *twitter.Stream
}
func (t *Twitter) Run() {
beaver.Debugf("setting up stream for %v", t.Filter)
demux := twitter.NewSwitchDemux()
demux.Tweet = func(tweet *twitter.Tweet) {
beaver.Debugf("new tweet for %v", t.Filter)
if t.Filter.FollowStrict {
var match bool
for _, id := range t.Filter.Follows {
if id == tweet.User.IDStr {
match = true
break
}
}
if !match {
beaver.Debug("tweet did not match any follow IDs")
return
}
}
if err := t.Discord(tweet); err != nil {
beaver.Error(err)
}
}
beaver.Debugf("streaming %v", t.Filter)
demux.HandleChan(t.Stream.Messages)
beaver.Debugf("disconnected from stream: %v", t.Filter)
}
func (t *Twitter) Discord(tweet *twitter.Tweet) error {
e := map[string]string{
"content": fmt.Sprintf("https://twitter.com/%d/status/%d", tweet.User.ID, tweet.ID),
}
data, err := json.Marshal(e)
if err != nil {
return err
}
beaver.Debug(string(data))
if t.Filter.Webhook == "" {
return fmt.Errorf("no webhook for %v", t.Filter)
}
payload := bytes.NewBuffer(data)
resp, err := httpClient.Post(t.Filter.Webhook, "application/json", payload)
if err != nil {
return err
}
if resp.StatusCode != http.StatusNoContent {
return err
}
return nil
}

View File

@ -1,3 +1,5 @@
# Reddit information
[reddit]
# Reddit Agent Info
app_name = "Lurk"
version = "0.1"
@ -7,40 +9,14 @@ client_id = ""
client_secret = ""
# Reddit credentials
username = "24CC-Official"
username = ""
password = ""
# A list of subreddites to monitor
# Empty fields are just for examples, they can be omitted in a real config
[[subreddit]]
name = "mcservers"
icon_url = "https://styles.redditmedia.com/t5_2s3kg/styles/communityIcon_recbaq3ufjv01.png"
flair_whitelist = []
flair_blacklist = []
title_whitelist = ["wanted"]
title_blacklist = []
title_limit = 0
body_whitelist = []
body_blacklist = []
body_limit = 0
webhook = ""
[[subreddit]]
name = "MinecraftBuddies"
icon_url = "https://styles.redditmedia.com/t5_30nfs/styles/communityIcon_qcijaahdybz01.png"
flair_whitelist = ["JAVA"]
flair_blacklist = []
title_whitelist = ["wanted", "vanilla"]
title_blacklist = []
title_limit = 0
body_whitelist = []
body_blacklist = []
body_limit = 0
webhook = ""
[[subreddit]]
name = "24CarrotCraft"
icon_url = "https://styles.redditmedia.com/t5_3budc/styles/communityIcon_byu6k6o8omz01.png"
# white/blacklist can be omitted in real config if empty
[[reddit.sub]]
name = ""
icon_url = ""
flair_whitelist = []
flair_blacklist = []
title_whitelist = []
@ -50,3 +26,23 @@ body_whitelist = []
body_blacklist = []
body_limit = 0
webhook = ""
# Twitter information
[twitter]
# Auth
consumer_key = ""
consumer_secret = ""
access_token = ""
access_secret = ""
# A list of filters to watch for
# Empty fields can be omitted in a real config
[[twitter.filter]]
# follows must use a Twitter user's ID
# https://tweeterid.com/
follows = []
# strict mode means only original tweets will be ingested
follow_strict = false
locations = []
tracks = []
webhook = ""

79
main.go
View File

@ -2,7 +2,15 @@ package main
import (
"flag"
"os"
"os/signal"
"syscall"
"go.etztech.xyz/lurk/config"
"go.etztech.xyz/lurk/handler"
"github.com/dghubble/go-twitter/twitter"
"github.com/dghubble/oauth1"
"github.com/turnage/graw"
"github.com/turnage/graw/reddit"
"go.jolheiser.com/beaver"
@ -29,31 +37,82 @@ func main() {
beaver.Console.Level = beaver.DEBUG
}
cfg, err := LoadConfig()
cfg, err := config.Load(configPath)
if err != nil {
beaver.Fatal(err)
}
// Reddit
go lurkReddit(cfg)
// Twitter
go lurkTwitter(cfg)
beaver.Info("Lurk is ready to start lurking!")
ch := make(chan os.Signal)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
<-ch
}
func lurkReddit(cfg *config.Config) {
bot, err := reddit.NewBot(reddit.BotConfig{
Agent: cfg.UserAgent(),
Agent: cfg.Reddit.UserAgent(),
App: reddit.App{
ID: cfg.ClientID,
Secret: cfg.ClientSecret,
Username: cfg.Username,
Password: cfg.Password,
ID: cfg.Reddit.ClientID,
Secret: cfg.Reddit.ClientSecret,
Username: cfg.Reddit.Username,
Password: cfg.Reddit.Password,
},
})
if err != nil {
beaver.Fatal(err)
}
_, wait, err := graw.Run(&lurker{cfg}, bot, graw.Config{Subreddits: cfg.SubRedditNames()})
_, wait, err := graw.Run(&handler.Reddit{
Config: cfg,
}, bot, graw.Config{
Subreddits: cfg.Reddit.SubRedditNames(),
})
if err != nil {
beaver.Fatal(err)
beaver.Errorf("could not run reddit bot: %v", err)
return
}
beaver.Info("Lurk is ready to start lurking!")
if err := wait(); err != nil {
beaver.Fatal(err)
}
}
func lurkTwitter(cfg *config.Config) {
twitterConfig := oauth1.NewConfig(cfg.Twitter.ConsumerKey, cfg.Twitter.ConsumerSecret)
token := oauth1.NewToken(cfg.Twitter.AccessToken, cfg.Twitter.AccessSecret)
httpClient := twitterConfig.Client(oauth1.NoContext, token)
client := twitter.NewClient(httpClient)
// Just to test if we have valid auth
_, _, err := client.Timelines.HomeTimeline(&twitter.HomeTimelineParams{
Count: 1,
})
if err != nil {
beaver.Fatal(err)
}
for _, filter := range cfg.Twitter.Filters {
stream, err := client.Streams.Filter(&twitter.StreamFilterParams{
Follow: filter.Follows,
Locations: filter.Locations,
StallWarnings: twitter.Bool(false),
Track: filter.Tracks,
})
if err != nil {
beaver.Fatal(err)
}
lurker := &handler.Twitter{
Filter: filter,
Stream: stream,
}
go lurker.Run()
}
}