Refactor, add lib, add some tests

Signed-off-by: jolheiser <john.olheiser@gmail.com>
main
jolheiser 2022-01-01 23:48:20 -06:00
parent d1ca6b1c49
commit 15d69e947d
Signed by: jolheiser
GPG Key ID: B853ADA5DA7BBF7A
21 changed files with 197 additions and 29 deletions

3
.gitignore vendored
View File

@ -1,3 +1,4 @@
.idea/ .idea/
.cabinet/ .cabinet/
/cabinet* /cabinet
/cabinet.exe

116
cabinet.go 100644
View File

@ -0,0 +1,116 @@
package cabinet
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/url"
)
type Client struct {
BaseURL string
HTTP *http.Client
Token string
}
func New(baseURL string, opts ...ClientOption) *Client {
c := &Client{
BaseURL: baseURL,
HTTP: http.DefaultClient,
}
for _, opt := range opts {
opt(c)
}
return c
}
type ClientOption func(*Client)
func WithHTTPClient(client *http.Client) ClientOption {
return func(c *Client) {
c.HTTP = client
}
}
func WithToken(token string) ClientOption {
return func(c *Client) {
c.Token = token
}
}
func (c *Client) Redirect(u string) (string, *http.Response, error) {
vals := url.Values{
"url": []string{u},
}
if c.Token != "" {
vals["token"] = []string{c.Token}
}
resp, err := c.HTTP.PostForm(fmt.Sprintf("%s/r", c.BaseURL), vals)
if err != nil {
return "", nil, err
}
if resp.StatusCode != 200 {
return "", resp, fmt.Errorf("non-200 status code: %s", resp.Status)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", resp, err
}
resp.Body = io.NopCloser(bytes.NewReader(body))
return string(body), resp, nil
}
func (c *Client) File(name string, f io.Reader) (string, *http.Response, error) {
var buf bytes.Buffer
mp := multipart.NewWriter(&buf)
token, err := mp.CreateFormField("token")
if err != nil {
return "", nil, err
}
if _, err := token.Write([]byte(c.Token)); err != nil {
return "", nil, err
}
file, err := mp.CreateFormFile("file", name)
if err != nil {
return "", nil, err
}
if _, err := io.Copy(file, f); err != nil {
return "", nil, err
}
if err := mp.Close(); err != nil {
return "", nil, err
}
req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/f", c.BaseURL), &buf)
if err != nil {
return "", nil, err
}
req.Header.Set("Content-Type", mp.FormDataContentType())
resp, err := c.HTTP.Do(req)
if err != nil {
return "", nil, err
}
if resp.StatusCode != 200 {
return "", resp, fmt.Errorf("non-200 status code: %s", resp.Status)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", resp, err
}
resp.Body = io.NopCloser(bytes.NewReader(body))
return string(body), resp, nil
}

View File

@ -7,13 +7,14 @@ import (
"os" "os"
"time" "time"
"go.jolheiser.com/cabinet/internal/gc"
router2 "go.jolheiser.com/cabinet/internal/router"
workspace2 "go.jolheiser.com/cabinet/internal/workspace"
"github.com/AlecAivazis/survey/v2" "github.com/AlecAivazis/survey/v2"
"github.com/rs/zerolog" "github.com/rs/zerolog"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"go.jolheiser.com/cabinet/gc"
"go.jolheiser.com/cabinet/router"
"go.jolheiser.com/cabinet/workspace"
) )
type serveOpts struct { type serveOpts struct {
@ -44,14 +45,14 @@ var serveCmd = func(opts *serveOpts) func(context.Context, []string) error {
opts.domain = fmt.Sprintf("http://localhost:%d", opts.port) opts.domain = fmt.Sprintf("http://localhost:%d", opts.port)
} }
ws, err := workspace.New(opts.workspacePath) ws, err := workspace2.New(opts.workspacePath)
if err != nil { if err != nil {
log.Fatal().Err(err).Msg("could not open workspace") log.Fatal().Err(err).Msg("could not open workspace")
} }
go gc.Start(ws, opts.maxDiskSize, opts.gcInterval) go gc.Start(ws, opts.maxDiskSize, opts.gcInterval)
r := router.New(opts.domain, ws, router.NewLimit(opts.requestPerMinute, opts.sizePerMinute, opts.burstSize, opts.memPerRequest)) r := router2.New(opts.domain, ws, router2.NewLimit(opts.requestPerMinute, opts.sizePerMinute, opts.burstSize, opts.memPerRequest))
portStr := fmt.Sprintf(":%d", opts.port) portStr := fmt.Sprintf(":%d", opts.port)
log.Info().Msgf("Listening at http://localhost%s", portStr) log.Info().Msgf("Listening at http://localhost%s", portStr)
@ -64,7 +65,7 @@ var serveCmd = func(opts *serveOpts) func(context.Context, []string) error {
type tokenOpts struct { type tokenOpts struct {
token string token string
perm workspace.TokenPermission perm workspace2.TokenPermission
desc string desc string
workspacePath string workspacePath string
delete bool delete bool
@ -72,7 +73,7 @@ type tokenOpts struct {
var tokenCmd = func(opts *tokenOpts) func(context.Context, []string) error { var tokenCmd = func(opts *tokenOpts) func(context.Context, []string) error {
return func(_ context.Context, _ []string) error { return func(_ context.Context, _ []string) error {
ws, err := workspace.New(opts.workspacePath) ws, err := workspace2.New(opts.workspacePath)
if err != nil { if err != nil {
log.Fatal().Err(err).Msg("could not open workspace") log.Fatal().Err(err).Msg("could not open workspace")
} }
@ -84,7 +85,7 @@ var tokenCmd = func(opts *tokenOpts) func(context.Context, []string) error {
} }
} }
func addToken(ws *workspace.Workspace, opts *tokenOpts) error { func addToken(ws *workspace2.Workspace, opts *tokenOpts) error {
token := opts.token token := opts.token
if token == "" { if token == "" {
r, err := randToken() r, err := randToken()
@ -118,7 +119,7 @@ func addToken(ws *workspace.Workspace, opts *tokenOpts) error {
if err != nil { if err != nil {
return err return err
} }
perm, err = workspace.ParseTokenPermission(p) perm, err = workspace2.ParseTokenPermission(p)
if err != nil { if err != nil {
return err return err
} }
@ -136,7 +137,7 @@ func addToken(ws *workspace.Workspace, opts *tokenOpts) error {
} }
} }
if err := ws.AddToken(workspace.Token{ if err := ws.AddToken(workspace2.Token{
Key: token, Key: token,
Permission: perm, Permission: perm,
Description: desc, Description: desc,
@ -148,7 +149,7 @@ func addToken(ws *workspace.Workspace, opts *tokenOpts) error {
return nil return nil
} }
func deleteTokens(ws *workspace.Workspace, opts *tokenOpts) error { func deleteTokens(ws *workspace2.Workspace, opts *tokenOpts) error {
if opts.token != "" { if opts.token != "" {
if err := ws.DeleteTokens(opts.token); err != nil { if err := ws.DeleteTokens(opts.token); err != nil {
return err return err

View File

@ -13,12 +13,13 @@ import (
"strings" "strings"
"time" "time"
"go.jolheiser.com/cabinet/internal/workspace"
"github.com/peterbourgon/ff/v3" "github.com/peterbourgon/ff/v3"
"github.com/peterbourgon/ff/v3/ffcli" "github.com/peterbourgon/ff/v3/ffcli"
"github.com/peterbourgon/ff/v3/fftoml" "github.com/peterbourgon/ff/v3/fftoml"
"github.com/rs/zerolog" "github.com/rs/zerolog"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"go.jolheiser.com/cabinet/workspace"
) )
func main() { func main() {
@ -39,7 +40,7 @@ func main() {
serveFS.Func("max-disk-size", "Max size of all disk space usage", fileSizeParse(&serveOpts.maxDiskSize)) serveFS.Func("max-disk-size", "Max size of all disk space usage", fileSizeParse(&serveOpts.maxDiskSize))
serveFS.StringVar(&serveOpts.workspacePath, "workspace", ".cabinet", "Workspace for DB, files, tokens, etc.") serveFS.StringVar(&serveOpts.workspacePath, "workspace", ".cabinet", "Workspace for DB, files, tokens, etc.")
serveFS.DurationVar(&serveOpts.gcInterval, "gc-interval", time.Hour, "GC interval for cleaning up files") serveFS.DurationVar(&serveOpts.gcInterval, "gc-interval", time.Hour, "GC interval for cleaning up files")
serveFS.IntVar(&serveOpts.requestPerMinute, "request-per-minute", 12, "Request limit per-second") serveFS.IntVar(&serveOpts.requestPerMinute, "request-per-minute", 12, "Request limit per-minute")
serveFS.Func("size-per-second", "File size limit per-second", fileSizeParse(&serveOpts.sizePerMinute)) serveFS.Func("size-per-second", "File size limit per-second", fileSizeParse(&serveOpts.sizePerMinute))
serveFS.Func("burst-size", "Burst size for files", fileSizeParse(&serveOpts.burstSize)) serveFS.Func("burst-size", "Burst size for files", fileSizeParse(&serveOpts.burstSize))
serveFS.Func("mem-per-request", "Memory per request before storing on disk temporarily", fileSizeParse(&serveOpts.memPerRequest)) serveFS.Func("mem-per-request", "Memory per request before storing on disk temporarily", fileSizeParse(&serveOpts.memPerRequest))

3
go.mod
View File

@ -1,11 +1,12 @@
module go.jolheiser.com/cabinet module go.jolheiser.com/cabinet
go 1.17 go 1.18
require ( require (
github.com/AlecAivazis/survey/v2 v2.3.2 github.com/AlecAivazis/survey/v2 v2.3.2
github.com/bwmarrin/snowflake v0.3.0 github.com/bwmarrin/snowflake v0.3.0
github.com/go-chi/chi/v5 v5.0.7 github.com/go-chi/chi/v5 v5.0.7
github.com/matryer/is v1.4.0
github.com/peterbourgon/ff/v3 v3.1.2 github.com/peterbourgon/ff/v3 v3.1.2
github.com/rs/zerolog v1.26.1 github.com/rs/zerolog v1.26.1
go.etcd.io/bbolt v1.3.6 go.etcd.io/bbolt v1.3.6

2
go.sum
View File

@ -18,6 +18,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kr/pty v1.1.4 h1:5Myjjh3JY/NaAi4IsUbHADytDyl1VE1Y9PXDlL+P/VQ= github.com/kr/pty v1.1.4 h1:5Myjjh3JY/NaAi4IsUbHADytDyl1VE1Y9PXDlL+P/VQ=
github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE=
github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=

View File

@ -29,10 +29,6 @@ func Start(g GC, maxSize int, interval time.Duration) {
size += int(info.Size()) size += int(info.Size())
} }
if size < maxSize {
continue
}
sort.Sort(sorter(infos)) sort.Sort(sorter(infos))
for size > maxSize { for size > maxSize {
fi := infos[0] fi := infos[0]

View File

@ -82,7 +82,7 @@ func (l *Limit) hasHitRequestLimit(remoteAddr string) bool {
if l.request[ip] == nil { if l.request[ip] == nil {
l.request[ip] = rate.NewLimiter(rate.Limit(l.rpm/60), l.rpm) l.request[ip] = rate.NewLimiter(rate.Limit(l.rpm/60), l.rpm)
l.size[ip] = rate.NewLimiter(rate.Limit(l.spm), l.burst) l.size[ip] = rate.NewLimiter(rate.Limit(l.spm/60), l.burst)
} }
return !l.request[ip].Allow() return !l.request[ip].Allow()

View File

@ -7,12 +7,12 @@ import (
"net/url" "net/url"
"strings" "strings"
"github.com/go-chi/chi/v5/middleware" "go.jolheiser.com/cabinet/internal/static"
"github.com/rs/zerolog/log" "go.jolheiser.com/cabinet/internal/workspace"
"go.jolheiser.com/cabinet/static"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"go.jolheiser.com/cabinet/workspace" "github.com/go-chi/chi/v5/middleware"
"github.com/rs/zerolog/log"
) )
type Cabinet interface { type Cabinet interface {

View File

@ -0,0 +1,43 @@
package router
import (
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/matryer/is"
"go.jolheiser.com/cabinet"
"go.jolheiser.com/cabinet/internal/workspace/mock"
)
func TestRouter(t *testing.T) {
assert := is.New(t)
m := mock.New()
server := httptest.NewUnstartedServer(nil)
server.Start()
defer server.Close()
r := New(server.URL, m, NewLimit(12, 5*1024*1024, 100*1024*1024, 1024*1024))
server.Config.Handler = r
c := cabinet.New(server.URL, cabinet.WithHTTPClient(server.Client()))
// Redirect
redir := "https://duckduckgo.com"
u, _, err := c.Redirect(redir)
assert.NoErr(err) // Creating a redirect should succeed
resp, err := http.Get(u)
assert.NoErr(err)
assert.Equal(redir, resp.Request.URL.String()) // The redirect should match what was given
file := "foobar"
f, _, err := c.File("test.txt", strings.NewReader(file))
assert.NoErr(err) // Creating a file should succeed
resp, err = http.Get(f)
assert.NoErr(err)
b, err := io.ReadAll(resp.Body)
assert.NoErr(err)
assert.Equal(file, string(b)) // The file should match what was given
}

View File

@ -3,7 +3,7 @@ package router
import ( import (
"net/http" "net/http"
"go.jolheiser.com/cabinet/workspace" "go.jolheiser.com/cabinet/internal/workspace"
) )
func tokenMiddleware(c Cabinet, perm workspace.TokenPermission) func(handler http.Handler) http.Handler { func tokenMiddleware(c Cabinet, perm workspace.TokenPermission) func(handler http.Handler) http.Handler {

View File

@ -4,7 +4,7 @@ import (
"os" "os"
"time" "time"
"go.jolheiser.com/cabinet/workspace" "go.jolheiser.com/cabinet/internal/workspace"
) )
type File struct { type File struct {

View File

@ -8,7 +8,7 @@ import (
"strings" "strings"
"time" "time"
"go.jolheiser.com/cabinet/workspace" "go.jolheiser.com/cabinet/internal/workspace"
) )
type Mock struct { type Mock struct {
@ -18,7 +18,8 @@ type Mock struct {
func New() *Mock { func New() *Mock {
return &Mock{ return &Mock{
files: make(map[string]*File), files: make(map[string]*File),
tokens: make(map[string]workspace.Token),
} }
} }
@ -72,6 +73,13 @@ func (m Mock) IsProtected() (bool, error) {
return len(m.tokens) > 0, nil return len(m.tokens) > 0, nil
} }
func (m Mock) Token(key string) (workspace.Token, error) {
if token, ok := m.tokens[key]; ok {
return token, nil
}
return workspace.Token{}, fmt.Errorf("no token found for %s", key)
}
func (m Mock) AddToken(token workspace.Token) error { func (m Mock) AddToken(token workspace.Token) error {
m.tokens[token.Key] = token m.tokens[token.Key] = token
return nil return nil

View File

@ -1 +0,0 @@
package router