Minecraft
/
lurk
6
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: Minecraft:main
Branches Tags
Minecraft:main
Minecraft:master
Minecraft:strict
Minecraft:twitter
..
compare: Minecraft:twitter
Branches Tags
Minecraft:main
Minecraft:master
Minecraft:strict
Minecraft:twitter

1 Commits
main ... twitter

Author SHA1 Message Date
Etzelia 31680a4efe
Add twitter lurker 
Signed-off-by: Etzelia <etzelia@hotmail.com>
 2020-08-09 17:03:47 -05:00
10 changed files with 129 additions and 164 deletions
Show Stats Expand all files Collapse all files

42
.drone.yml
View File

@ -1,42 +0,0 @@
---
kind: pipeline
name: compliance
trigger:
event:
- pull_request
steps:
- name: build
pull: always
image: golang:1.15
commands:
- make test
- make build
- name: check
pull: always
image: golang:1.15
commands:
- make vet
---
kind: pipeline
name: release
trigger:
event:
- push
branch:
- master
steps:
- name: build
pull: always
image: golang:1.15
commands:
- make build
- name: gitea-release
pull: always
image: jolheiser/drone-gitea-main:latest
settings:
token:
from_secret: gitea_token
base: https://git.etztech.xyz
files:
- "lurk"

2
README.md
View File

@ -1,7 +1,5 @@
# lurk
### This repository is archived, development has moved to https://gitea.com/jolheiser/lurk
A lurker used to notify a Discord channel via webhook
whenever a matching submission is made.

49
config/config.go
View File

@ -4,7 +4,7 @@ import (
"regexp"
"strings"
"github.com/pelletier/go-toml"
"github.com/BurntSushi/toml"
"go.jolheiser.com/beaver"
)
@ -22,50 +22,45 @@ func (c *Config) loadReddit() {
if sub.BodyLimit == 0 || sub.BodyLimit > 2045 {
sub.BodyLimit = 2045
}
sub.FlairAllowlistRe = make([]*regexp.Regexp, len(sub.FlairAllowlist))
for idx, f := range sub.FlairAllowlist {
sub.FlairAllowlistRe[idx] = regexp.MustCompile(f)
sub.FlairWhitelistRe = make([]*regexp.Regexp, len(sub.FlairWhitelist))
for idx, f := range sub.FlairWhitelist {
sub.FlairWhitelistRe[idx] = regexp.MustCompile(f)
}
sub.FlairBlocklistRe = make([]*regexp.Regexp, len(sub.FlairBlocklist))
for idx, f := range sub.FlairBlocklist {
sub.FlairBlocklistRe[idx] = regexp.MustCompile(f)
sub.FlairBlacklistRe = make([]*regexp.Regexp, len(sub.FlairBlacklist))
for idx, f := range sub.FlairBlacklist {
sub.FlairBlacklistRe[idx] = regexp.MustCompile(f)
}
sub.TitleAllowlistRe = make([]*regexp.Regexp, len(sub.TitleAllowlist))
for idx, t := range sub.TitleAllowlist {
sub.TitleAllowlistRe[idx] = regexp.MustCompile(t)
sub.TitleWhitelistRe = make([]*regexp.Regexp, len(sub.TitleWhitelist))
for idx, t := range sub.TitleWhitelist {
sub.TitleWhitelistRe[idx] = regexp.MustCompile(t)
}
sub.TitleBlocklistRe = make([]*regexp.Regexp, len(sub.TitleBlocklist))
for idx, t := range sub.TitleBlocklist {
sub.TitleBlocklistRe[idx] = regexp.MustCompile(t)
sub.TitleBlacklistRe = make([]*regexp.Regexp, len(sub.TitleBlacklist))
for idx, t := range sub.TitleBlacklist {
sub.TitleBlacklistRe[idx] = regexp.MustCompile(t)
}
sub.BodyAllowlistRe = make([]*regexp.Regexp, len(sub.BodyAllowlist))
for idx, b := range sub.BodyAllowlist {
sub.BodyAllowlistRe[idx] = regexp.MustCompile(b)
sub.BodyWhitelistRe = make([]*regexp.Regexp, len(sub.BodyWhitelist))
for idx, b := range sub.BodyWhitelist {
sub.BodyWhitelistRe[idx] = regexp.MustCompile(b)
}
sub.BodyBlocklistRe = make([]*regexp.Regexp, len(sub.BodyBlocklist))
for idx, b := range sub.BodyBlocklist {
sub.BodyBlocklistRe[idx] = regexp.MustCompile(b)
sub.BodyBlacklistRe = make([]*regexp.Regexp, len(sub.BodyBlacklist))
for idx, b := range sub.BodyBlacklist {
sub.BodyBlacklistRe[idx] = regexp.MustCompile(b)
}
}
}
func Load(configPath string) (*Config, error) {
cfg := Config{
cfg := &Config{
Reddit: RedditConfig{
Map: make(map[string]*SubReddit),
},
}
tree, err := toml.LoadFile(configPath)
if err != nil {
return nil, err
}
if err := tree.Unmarshal(&cfg); err != nil {
if _, err := toml.DecodeFile(configPath, &cfg); err != nil {
return nil, err
}
cfg.loadReddit()
beaver.Debug(cfg)
return &cfg, nil
return cfg, nil
}

24
config/reddit.go
View File

@ -23,19 +23,19 @@ type RedditConfig struct {
type SubReddit struct {
Name string `toml:"name"`
IconURL string `toml:"icon_url"`
FlairAllowlist []string `toml:"flair_allowlist"`
FlairAllowlistRe []*regexp.Regexp `toml:"-"`
FlairBlocklist []string `toml:"flair_blocklist"`
FlairBlocklistRe []*regexp.Regexp `toml:"-"`
TitleAllowlist []string `toml:"title_allowlist"`
TitleAllowlistRe []*regexp.Regexp `toml:"-"`
TitleBlocklist []string `toml:"title_blocklist"`
TitleBlocklistRe []*regexp.Regexp `toml:"-"`
FlairWhitelist []string `toml:"flair_whitelist"`
FlairWhitelistRe []*regexp.Regexp `toml:"-"`
FlairBlacklist []string `toml:"flair_blacklist"`
FlairBlacklistRe []*regexp.Regexp `toml:"-"`
TitleWhitelist []string `toml:"title_whitelist"`
TitleWhitelistRe []*regexp.Regexp `toml:"-"`
TitleBlacklist []string `toml:"title_blacklist"`
TitleBlacklistRe []*regexp.Regexp `toml:"-"`
TitleLimit int `toml:"title_limit"`
BodyAllowlist []string `toml:"body_allowlist"`
BodyAllowlistRe []*regexp.Regexp `toml:"-"`
BodyBlocklist []string `toml:"body_blocklist"`
BodyBlocklistRe []*regexp.Regexp `toml:"-"`
BodyWhitelist []string `toml:"body_whitelist"`
BodyWhitelistRe []*regexp.Regexp `toml:"-"`
BodyBlacklist []string `toml:"body_blacklist"`
BodyBlacklistRe []*regexp.Regexp `toml:"-"`
BodyLimit int `toml:"body_limit"`
Webhook string `toml:"webhook"`
}

9
config/twitter.go
View File

@ -10,9 +10,8 @@ type TwitterConfig struct {
}
type Filter struct {
Follows []string `toml:"follows"`
FollowStrict bool `toml:"follow_strict"`
Locations []string `toml:"locations"`
Tracks []string `toml:"tracks"`
Webhook string `toml:"webhook"`
Follows []string `toml:"follows"`
Locations []string `toml:"locations"`
Tracks []string `toml:"tracks"`
Webhook string `toml:"webhook"`
}

5
go.mod
View File

@ -1,12 +1,11 @@
module go.etztech.xyz/lurk
go 1.16
go 1.14
require (
github.com/BurntSushi/toml v0.3.1
github.com/dghubble/go-twitter v0.0.0-20200725221434-4bc8ad7ad1b4
github.com/dghubble/oauth1 v0.6.0
github.com/pelletier/go-toml v1.8.1
github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0
go.jolheiser.com/beaver v1.0.2
go.jolheiser.com/disco v0.0.2
)

10
go.sum
View File

@ -1,9 +1,9 @@
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY=
github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dghubble/go-twitter v0.0.0-20200725221434-4bc8ad7ad1b4 h1:I60CX3+rWlBGPXR13jwxxBWB2GhlhZkEEh5o1eDLzJg=
github.com/dghubble/go-twitter v0.0.0-20200725221434-4bc8ad7ad1b4/go.mod h1:xfg4uS5LEzOj8PgZV7SQYRHbG7jPUnelEiaAVJxmhJE=
github.com/dghubble/oauth1 v0.6.0 h1:m1yC01Ohc/eF38jwZ8JUjL1a+XHHXtGQgK+MxQbmSx0=
@ -19,12 +19,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0 h1:ss0FREpyYvw5V9t5XWWWIvUJnRgWFzdftvCXd8WQNhU=
github.com/turnage/graw v0.0.0-20200404033202-65715eea1cd0/go.mod h1:aAkq4I/q1izZSSwHvzhDn9NA+eGxgTSuibwP3MZRlQY=
@ -32,8 +28,6 @@ github.com/turnage/redditproto v0.0.0-20151223012412-afedf1b6eddb h1:qR56NGRvs2h
github.com/turnage/redditproto v0.0.0-20151223012412-afedf1b6eddb/go.mod h1:GyqJdEoZSNoxKDb7Z2Lu/bX63jtFukwpaTP9ZIS5Ei0=
go.jolheiser.com/beaver v1.0.2 h1:KA2D6iO8MQhZi1nZYi/Chak/f1Cxfrs6b1XO623+Khk=
go.jolheiser.com/beaver v1.0.2/go.mod h1:7X4F5+XOGSC3LejTShoBdqtRCnPWcnRgmYGmG3EKW8g=
go.jolheiser.com/disco v0.0.2 h1:UGYNqO7NQSBGB/OoS9WE5o/jYvmx1G0Bq3qQRM42Bkw=
go.jolheiser.com/disco v0.0.2/go.mod h1:tY3HkJmMrzXH/bPgDWKHn1DUzDxkemD80OHLgHSA5uQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

79
handler/reddit.go
View File

@ -1,7 +1,8 @@
package handler
import (
"context"
"bytes"
"encoding/json"
"fmt"
"net/http"
"regexp"
@ -12,11 +13,30 @@ import (
"github.com/turnage/graw/reddit"
"go.jolheiser.com/beaver"
"go.jolheiser.com/disco"
)
var httpClient = &http.Client{Timeout: time.Minute}
type Webhook struct {
Username string `json:"username"`
AvatarURL string `json:"avatar_url"`
Embeds []Embed `json:"embeds"`
}
type Embed struct {
Title string `json:"title"`
URL string `json:"url"`
Description string `json:"description"`
Color int64 `json:"color"`
Timestamp time.Time `json:"timestamp"`
Author Author `json:"author"`
}
type Author struct {
Name string `json:"name"`
URL string `json:"url"`
}
type Reddit struct {
Config *config.Config
}
@ -38,36 +58,37 @@ func (r *Reddit) Post(p *reddit.Post) error {
description = description[:sub.BodyLimit] + "..."
}
e := disco.Webhook{
e := Webhook{
Username: "/r/" + p.Subreddit,
AvatarURL: sub.IconURL,
Embeds: []*disco.Embed{
Embeds: []Embed{
{
Title: title,
URL: p.URL,
Description: description,
Color: 0x007D96,
Timestamp: disco.Now(),
Author: &disco.Author{
Color: 16312092, // Yellow
Timestamp: time.Now(),
Author: Author{
Name: "/u/" + p.Author,
URL: fmt.Sprintf("https://reddit.com/user/%s", p.Author),
},
},
},
}
data, err := json.Marshal(e)
if err != nil {
beaver.Error(err)
return nil
}
beaver.Debug(string(data))
if sub.Webhook == "" {
beaver.Errorf("no webhook for %s", p.Subreddit)
return nil
}
req, err := e.Request(context.Background(), sub.Webhook)
if err != nil {
beaver.Error(err)
return nil
}
resp, err := httpClient.Do(req)
payload := bytes.NewBuffer(data)
resp, err := httpClient.Post(sub.Webhook, "application/json", payload)
if err != nil {
beaver.Error(err)
return nil
@ -84,29 +105,29 @@ func (r *Reddit) Post(p *reddit.Post) error {
func checkPost(c *config.Config, p *reddit.Post) error {
sub := c.Reddit.Map[strings.ToLower(p.Subreddit)]
// Check blocklist first
// Check blacklist first
// Any match means we ignore
if matchesAny(p.LinkFlairText, sub.FlairBlocklistRe) {
return fmt.Errorf("flair matched blocklisted regex: %s", p.LinkFlairText)
if matchesAny(p.LinkFlairText, sub.FlairBlacklistRe) {
return fmt.Errorf("flair matched blacklisted regex: %s", p.LinkFlairText)
}
if matchesAny(p.Title, sub.TitleBlocklistRe) {
return fmt.Errorf("title matched blocklisted regex: %s", p.Title)
if matchesAny(p.Title, sub.TitleBlacklistRe) {
return fmt.Errorf("title matched blacklisted regex: %s", p.Title)
}
if matchesAny(p.SelfText, sub.BodyBlocklistRe) {
return fmt.Errorf("body matched blocklisted regex: %s", p.SelfText)
if matchesAny(p.SelfText, sub.BodyBlacklistRe) {
return fmt.Errorf("body matched blacklisted regex: %s", p.SelfText)
}
// Check allowlist
// Check whitelist
// Any match means we pass
// If no allowlist, pass
if len(sub.FlairAllowlistRe) > 0 && !matchesAny(p.LinkFlairText, sub.FlairAllowlistRe) {
return fmt.Errorf("flair didn't match any allowlisted regex: %s", p.LinkFlairText)
// If no whitelist, pass
if len(sub.FlairWhitelistRe) > 0 && !matchesAny(p.LinkFlairText, sub.FlairWhitelistRe) {
return fmt.Errorf("flair didn't match any whitelisted regex: %s", p.LinkFlairText)
}
if len(sub.TitleAllowlistRe) > 0 && !matchesAny(p.Title, sub.TitleAllowlistRe) {
return fmt.Errorf("title didn't match any allowlisted regex: %s", p.Title)
if len(sub.TitleWhitelistRe) > 0 && !matchesAny(p.Title, sub.TitleWhitelistRe) {
return fmt.Errorf("title didn't match any whitelisted regex: %s", p.Title)
}
if len(sub.BodyAllowlistRe) > 0 && !matchesAny(p.SelfText, sub.BodyAllowlistRe) {
return fmt.Errorf("body didn't match any allowlisted regex: %s", p.SelfText)
if len(sub.BodyWhitelistRe) > 0 && !matchesAny(p.SelfText, sub.BodyWhitelistRe) {
return fmt.Errorf("body didn't match any whitelisted regex: %s", p.SelfText)
}
return nil

57
handler/twitter.go
View File

@ -1,14 +1,15 @@
package handler
import (
"context"
"bytes"
"encoding/json"
"fmt"
"net/http"
"go.etztech.xyz/lurk/config"
"github.com/dghubble/go-twitter/twitter"
"go.jolheiser.com/beaver"
"go.jolheiser.com/disco"
)
type Twitter struct {
@ -19,39 +20,41 @@ type Twitter struct {
func (t *Twitter) Run() {
beaver.Debugf("setting up stream for %v", t.Filter)
demux := twitter.NewSwitchDemux()
demux.Tweet = t.Tweet
demux.Tweet = func(tweet *twitter.Tweet) {
beaver.Debugf("new tweet for %v", t.Filter)
if err := t.Discord(tweet); err != nil {
beaver.Error(err)
}
}
beaver.Debugf("streaming %v", t.Filter)
demux.HandleChan(t.Stream.Messages)
beaver.Debugf("disconnected from stream: %v", t.Filter)
beaver.Debugf("diconnected from stream: %v", t.Filter)
}
func (t *Twitter) Tweet(tweet *twitter.Tweet) {
beaver.Debugf("new tweet for %v", t.Filter)
func (t *Twitter) Discord(tweet *twitter.Tweet) error {
e := map[string]string{
"content": fmt.Sprintf("https://twitter.com/%d/status/%d", tweet.User.ID, tweet.ID),
}
data, err := json.Marshal(e)
if err != nil {
return err
}
beaver.Debug(string(data))
if t.Filter.FollowStrict {
if tweet.InReplyToStatusIDStr != "" {
beaver.Debug("tweet is a reply")
return
}
var match bool
for _, id := range t.Filter.Follows {
if id == tweet.User.IDStr {
match = true
break
}
}
if !match {
beaver.Debug("tweet did not match any follow IDs")
return
}
if t.Filter.Webhook == "" {
return fmt.Errorf("no webhook for %s", t.Filter)
}
w := &disco.Webhook{
Username: tweet.User.Name,
Content: fmt.Sprintf("https://twitter.com/%d/status/%d", tweet.User.ID, tweet.ID),
payload := bytes.NewBuffer(data)
resp, err := httpClient.Post(t.Filter.Webhook, "application/json", payload)
if err != nil {
return err
}
if _, err := w.Send(context.Background(), t.Filter.Webhook); err != nil {
beaver.Error(err)
if resp.StatusCode != http.StatusNoContent {
return err
}
return nil
}

16
lurk.sample.toml
View File

@ -13,17 +13,17 @@
password = ""
# A list of subreddites to monitor
# allow/blocklist can be omitted in real config if empty
# white/blacklist can be omitted in real config if empty
[[reddit.sub]]
name = ""
icon_url = ""
flair_allowlist = []
flair_blocklist = []
title_allowlist = []
title_blocklist = []
flair_whitelist = []
flair_blacklist = []
title_whitelist = []
title_blacklist = []
title_limit = 0
body_allowlist = []
body_blocklist = []
body_whitelist = []
body_blacklist = []
body_limit = 0
webhook = ""
@ -41,8 +41,6 @@
# follows must use a Twitter user's ID
# https://tweeterid.com/
follows = []
# strict mode means only original tweets will be ingested
follow_strict = false
locations = []
tracks = []
webhook = ""