From 718a2d3cfb8fed76c0107b3f63205af62d3eb437 Mon Sep 17 00:00:00 2001 From: Etzelia Date: Fri, 17 Apr 2020 21:32:09 -0500 Subject: [PATCH] Initial Commit Signed-off-by: Etzelia --- .gitignore | 7 +++ Makefile | 15 +++++ config.go | 107 +++++++++++++++++++++++++++++++++++ go.mod | 9 +++ go.sum | 30 ++++++++++ handler.go | 141 +++++++++++++++++++++++++++++++++++++++++++++++ lurk.sample.toml | 50 +++++++++++++++++ main.go | 59 ++++++++++++++++++++ main_test.go | 38 +++++++++++++ 9 files changed, 456 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 config.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 handler.go create mode 100644 lurk.sample.toml create mode 100644 main.go create mode 100644 main_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..515fddb --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +# GoLand +.idea/ + +# Binaries +/lurk* + +!lurk.sample.toml \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0b355d9 --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +.PHONY: build +build: + go build + +.PHONY: fmt +fmt: + go fmt ./... + +.PHONY: test +test: + go test -race ./... + +.PHONY: vet +vet: + go vet ./... \ No newline at end of file diff --git a/config.go b/config.go new file mode 100644 index 0000000..64e1cb9 --- /dev/null +++ b/config.go @@ -0,0 +1,107 @@ +package main + +import ( + "fmt" + "regexp" + "strings" + + "github.com/BurntSushi/toml" + "go.jolheiser.com/beaver" +) + +type Config struct { + SubReddits []*SubReddit `toml:"subreddit"` + Map map[string]*SubReddit `toml:"-"` + + // Agent file + AppName string `toml:"app_name"` + Version string `toml:"version"` + + ClientID string `toml:"client_id"` + ClientSecret string `toml:"client_secret"` + + Username string `toml:"username"` + Password string `toml:"password"` +} + +type SubReddit struct { + Name string `toml:"name"` + IconURL string `toml:"icon_url"` + FlairWhitelist []string `toml:"flair_whitelist"` + FlairWhitelistRe []*regexp.Regexp `toml:"-"` + FlairBlacklist []string `toml:"flair_blacklist"` + FlairBlacklistRe []*regexp.Regexp `toml:"-"` + TitleWhitelist []string `toml:"title_whitelist"` + TitleWhitelistRe []*regexp.Regexp `toml:"-"` + TitleBlacklist []string `toml:"title_blacklist"` + TitleBlacklistRe []*regexp.Regexp `toml:"-"` + TitleLimit int `toml:"title_limit"` + BodyWhitelist []string `toml:"body_whitelist"` + BodyWhitelistRe []*regexp.Regexp `toml:"-"` + BodyBlacklist []string `toml:"body_blacklist"` + BodyBlacklistRe []*regexp.Regexp `toml:"-"` + BodyLimit int `toml:"body_limit"` + Webhook string `toml:"webhook"` +} + +func (c *Config) UserAgent() string { + return fmt.Sprintf("%s/%s by /u/%s", c.AppName, c.Version, c.Username) +} + +func (c *Config) SubRedditNames() []string { + names := make([]string, len(c.SubReddits)) + for idx, sub := range c.SubReddits { + names[idx] = sub.Name + } + return names +} + +func (c *Config) load() { + for _, sub := range c.SubReddits { + c.Map[strings.ToLower(sub.Name)] = sub + if sub.TitleLimit == 0 || sub.TitleLimit > 253 { + sub.TitleLimit = 253 + } + if sub.BodyLimit == 0 || sub.BodyLimit > 2045 { + sub.BodyLimit = 2045 + } + sub.FlairWhitelistRe = make([]*regexp.Regexp, len(sub.FlairWhitelist)) + for idx, f := range sub.FlairWhitelist { + sub.FlairWhitelistRe[idx] = regexp.MustCompile(f) + } + sub.FlairBlacklistRe = make([]*regexp.Regexp, len(sub.FlairBlacklist)) + for idx, f := range sub.FlairBlacklist { + sub.FlairBlacklistRe[idx] = regexp.MustCompile(f) + } + sub.TitleWhitelistRe = make([]*regexp.Regexp, len(sub.TitleWhitelist)) + for idx, t := range sub.TitleWhitelist { + sub.TitleWhitelistRe[idx] = regexp.MustCompile(t) + } + sub.TitleBlacklistRe = make([]*regexp.Regexp, len(sub.TitleBlacklist)) + for idx, t := range sub.TitleBlacklist { + sub.TitleBlacklistRe[idx] = regexp.MustCompile(t) + } + sub.BodyWhitelistRe = make([]*regexp.Regexp, len(sub.BodyWhitelist)) + for idx, b := range sub.BodyWhitelist { + sub.BodyWhitelistRe[idx] = regexp.MustCompile(b) + } + sub.BodyBlacklistRe = make([]*regexp.Regexp, len(sub.BodyBlacklist)) + for idx, b := range sub.BodyBlacklist { + sub.BodyBlacklistRe[idx] = regexp.MustCompile(b) + } + } +} + +func LoadConfig() (*Config, error) { + cfg := &Config{ + Map: make(map[string]*SubReddit), + } + + if _, err := toml.DecodeFile(configPath, &cfg); err != nil { + return nil, err + } + + cfg.load() + beaver.Debug(cfg) + return cfg, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..57207f8 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module go.etztech.xyz/lurk + +go 1.14 + +require ( + github.com/BurntSushi/toml v0.3.1 + github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0 + go.jolheiser.com/beaver v1.0.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..fdd5fb0 --- /dev/null +++ b/go.sum @@ -0,0 +1,30 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0 h1:ss0FREpyYvw5V9t5XWWWIvUJnRgWFzdftvCXd8WQNhU= +github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0/go.mod h1:aAkq4I/q1izZSSwHvzhDn9NA+eGxgTSuibwP3MZRlQY= +github.com/turnage/redditproto v0.0.0-20151223012412-afedf1b6eddb h1:qR56NGRvs2hTUbkn6QF8bEJzxPIoMw3Np3UigBeJO5A= +github.com/turnage/redditproto v0.0.0-20151223012412-afedf1b6eddb/go.mod h1:GyqJdEoZSNoxKDb7Z2Lu/bX63jtFukwpaTP9ZIS5Ei0= +go.jolheiser.com/beaver v1.0.2 h1:KA2D6iO8MQhZi1nZYi/Chak/f1Cxfrs6b1XO623+Khk= +go.jolheiser.com/beaver v1.0.2/go.mod h1:7X4F5+XOGSC3LejTShoBdqtRCnPWcnRgmYGmG3EKW8g= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/handler.go b/handler.go new file mode 100644 index 0000000..0099d2d --- /dev/null +++ b/handler.go @@ -0,0 +1,141 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "regexp" + "strings" + "time" + + "github.com/turnage/graw/reddit" + "go.jolheiser.com/beaver" +) + +var httpClient = &http.Client{Timeout: time.Minute} + +type Webhook struct { + Username string `json:"username"` + AvatarURL string `json:"avatar_url"` + Embeds []Embed `json:"embeds"` +} + +type Embed struct { + Title string `json:"title"` + URL string `json:"url"` + Description string `json:"description"` + Color int64 `json:"color"` + Timestamp time.Time `json:"timestamp"` + Author Author `json:"author"` +} + +type Author struct { + Name string `json:"name"` + URL string `json:"url"` +} + +type lurker struct { + config *Config +} + +func (l *lurker) Post(p *reddit.Post) error { + sub := l.config.Map[strings.ToLower(p.Subreddit)] + + if err := checkPost(l.config, p); err != nil { + beaver.Debugf("%s: %v", p.Subreddit, err) + return nil + } + + title := p.Title + if len(title) > sub.TitleLimit { + title = title[:sub.TitleLimit] + "..." + } + description := p.SelfText + if len(description) > sub.BodyLimit { + description = description[:sub.BodyLimit] + "..." + } + + e := Webhook{ + Username: "/r/" + p.Subreddit, + AvatarURL: sub.IconURL, + Embeds: []Embed{ + { + Title: title, + URL: p.URL, + Description: description, + Color: 16312092, // Yellow + Timestamp: time.Now(), + Author: Author{ + Name: "/u/" + p.Author, + URL: fmt.Sprintf("https://reddit.com/user/%s", p.Author), + }, + }, + }, + } + data, err := json.Marshal(e) + if err != nil { + beaver.Error(err) + return nil + } + beaver.Debug(string(data)) + + if sub.Webhook == "" { + beaver.Errorf("no webhook for %s", p.Subreddit) + return nil + } + + payload := bytes.NewBuffer(data) + resp, err := httpClient.Post(sub.Webhook, "application/json", payload) + if err != nil { + beaver.Error(err) + return nil + } + + if resp.StatusCode != http.StatusNoContent { + beaver.Error(resp.Status) + return nil + } + + return nil +} + +func checkPost(c *Config, p *reddit.Post) error { + sub := c.Map[strings.ToLower(p.Subreddit)] + + // Check blacklist first + // Any match means we ignore + if matchesAny(p.LinkFlairText, sub.FlairBlacklistRe) { + return fmt.Errorf("flair matched blacklisted regex: %s", p.LinkFlairText) + } + if matchesAny(p.Title, sub.TitleBlacklistRe) { + return fmt.Errorf("title matched blacklisted regex: %s", p.Title) + } + if matchesAny(p.SelfText, sub.BodyBlacklistRe) { + return fmt.Errorf("body matched blacklisted regex: %s", p.SelfText) + } + + // Check whitelist + // Any match means we pass + // If no whitelist, pass + if len(sub.FlairWhitelistRe) > 0 && !matchesAny(p.LinkFlairText, sub.FlairWhitelistRe) { + return fmt.Errorf("flair didn't match any whitelisted regex: %s", p.LinkFlairText) + } + if len(sub.TitleWhitelistRe) > 0 && !matchesAny(p.Title, sub.TitleWhitelistRe) { + return fmt.Errorf("title didn't match any whitelisted regex: %s", p.Title) + } + if len(sub.BodyWhitelistRe) > 0 && !matchesAny(p.SelfText, sub.BodyWhitelistRe) { + return fmt.Errorf("body didn't match any whitelisted regex: %s", p.SelfText) + } + + return nil +} + +func matchesAny(input string, re []*regexp.Regexp) bool { + for _, r := range re { + if r.MatchString(input) { + return true + } + } + return false +} diff --git a/lurk.sample.toml b/lurk.sample.toml new file mode 100644 index 0000000..605a231 --- /dev/null +++ b/lurk.sample.toml @@ -0,0 +1,50 @@ +# Reddit Agent Info +app_name = "Lurk" +version = "0.1" + +client_id = "" +client_secret = "" + +username = "24CC-Official" +password = "" + +# A list of subreddites to monitor +# Empty fields are just for examples, they can be omitted in a real config +[[subreddit]] +name = "mcservers" +icon_url = "https://styles.redditmedia.com/t5_2s3kg/styles/communityIcon_recbaq3ufjv01.png" +flair_whitelist = [] +flair_blacklist = [] +title_whitelist = ["wanted"] +title_blacklist = [] +title_limit = 0 +body_whitelist = [] +body_blacklist = [] +body_limit = 0 +webhook = "" + +[[subreddit]] +name = "MinecraftBuddies" +icon_url = "https://styles.redditmedia.com/t5_30nfs/styles/communityIcon_qcijaahdybz01.png" +flair_whitelist = ["JAVA"] +flair_blacklist = [] +title_whitelist = ["wanted", "vanilla"] +title_blacklist = [] +title_limit = 0 +body_whitelist = [] +body_blacklist = [] +body_limit = 0 +webhook = "" + +[[subreddit]] +name = "24CarrotCraft" +icon_url = "https://styles.redditmedia.com/t5_3budc/styles/communityIcon_byu6k6o8omz01.png" +flair_whitelist = [] +flair_blacklist = [] +title_whitelist = [] +title_blacklist = [] +title_limit = 0 +body_whitelist = [] +body_blacklist = [] +body_limit = 0 +webhook = "" \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..d257b23 --- /dev/null +++ b/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "flag" + + "github.com/turnage/graw" + "github.com/turnage/graw/reddit" + "go.jolheiser.com/beaver" +) + +var ( + configPath string + debug bool +) + +func main() { + flag.StringVar(&configPath, "config", "lurk.toml", "Path to lurk's config file") + flag.BoolVar(&debug, "debug", false, "Turn on debug mode") + flag.Parse() + + beaver.Console.Format = beaver.FormatOptions{ + TimePrefix: true, + StackPrefix: true, + StackLimit: 15, + LevelPrefix: true, + LevelColor: true, + } + if debug { + beaver.Console.Level = beaver.DEBUG + } + + cfg, err := LoadConfig() + if err != nil { + beaver.Fatal(err) + } + + bot, err := reddit.NewBot(reddit.BotConfig{ + Agent: cfg.UserAgent(), + App: reddit.App{ + ID: cfg.ClientID, + Secret: cfg.ClientSecret, + Username: cfg.Username, + Password: cfg.Password, + }, + }) + if err != nil { + beaver.Fatal(err) + } + + _, wait, err := graw.Run(&lurker{cfg}, bot, graw.Config{Subreddits: cfg.SubRedditNames()}) + if err != nil { + beaver.Fatal(err) + } + + beaver.Info("Lurk is ready to start lurking!") + if err := wait(); err != nil { + beaver.Fatal(err) + } +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..32358ac --- /dev/null +++ b/main_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "os" + "regexp" + "testing" +) + +func TestMain(m *testing.M) { + os.Exit(m.Run()) +} + +func TestMatchesAny(t *testing.T) { + tt := []struct { + Regex string + Input string + Match bool + }{ + {Regex: `JAVA`, Input: "java", Match: false}, + {Regex: `(?i)JAVA`, Input: "java", Match: true}, + {Regex: `white`, Input: "whitelist", Match: true}, + {Regex: `\bwhite\b`, Input: "whitelist", Match: false}, + {Regex: `\bwhite\b`, Input: "white list", Match: true}, + {Regex: `\bwhite\b`, Input: "a white list", Match: true}, + {Regex: `\swhite\s`, Input: "whitelist", Match: false}, + {Regex: `\swhite\s`, Input: "white list", Match: false}, + {Regex: `\swhite\s`, Input: "a white list", Match: true}, + } + + for _, tc := range tt { + t.Run(tc.Regex, func(t *testing.T) { + r := regexp.MustCompile(tc.Regex) + if r.MatchString(tc.Input) != tc.Match { + t.Fail() + } + }) + } +}