From 15d69e947de68c0c6b810555c9b04355d297beb5 Mon Sep 17 00:00:00 2001 From: jolheiser Date: Sat, 1 Jan 2022 23:48:20 -0600 Subject: [PATCH] Refactor, add lib, add some tests Signed-off-by: jolheiser --- .gitignore | 3 +- cabinet.go | 116 ++++++++++++++++++ command.go => cmd/cabinet/command.go | 23 ++-- main.go => cmd/cabinet/main.go | 5 +- go.mod | 3 +- go.sum | 2 + {gc => internal/gc}/gc.go | 4 - {gc => internal/gc}/sort.go | 0 {router => internal/router}/rate.go | 2 +- {router => internal/router}/router.go | 8 +- internal/router/router_test.go | 43 +++++++ {router => internal/router}/token.go | 2 +- {static => internal/static}/index.tmpl | 0 {static => internal/static}/sakura.css | 0 {static => internal/static}/static.go | 0 {workspace => internal/workspace}/meta.go | 0 .../workspace}/mock/file.go | 2 +- .../workspace}/mock/mock.go | 12 +- {workspace => internal/workspace}/token.go | 0 .../workspace}/workspace.go | 0 router/logger.go | 1 - 21 files changed, 197 insertions(+), 29 deletions(-) create mode 100644 cabinet.go rename command.go => cmd/cabinet/command.go (85%) rename main.go => cmd/cabinet/main.go (98%) rename {gc => internal/gc}/gc.go (95%) rename {gc => internal/gc}/sort.go (100%) rename {router => internal/router}/rate.go (97%) rename {router => internal/router}/router.go (97%) create mode 100644 internal/router/router_test.go rename {router => internal/router}/token.go (95%) rename {static => internal/static}/index.tmpl (100%) rename {static => internal/static}/sakura.css (100%) rename {static => internal/static}/static.go (100%) rename {workspace => internal/workspace}/meta.go (100%) rename {workspace => internal/workspace}/mock/file.go (90%) rename {workspace => internal/workspace}/mock/mock.go (83%) rename {workspace => internal/workspace}/token.go (100%) rename {workspace => internal/workspace}/workspace.go (100%) delete mode 100644 router/logger.go diff --git a/.gitignore b/.gitignore index 669554e..b5fab61 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea/ .cabinet/ -/cabinet* \ No newline at end of file +/cabinet +/cabinet.exe \ No newline at end of file diff --git a/cabinet.go b/cabinet.go new file mode 100644 index 0000000..d2015e4 --- /dev/null +++ b/cabinet.go @@ -0,0 +1,116 @@ +package cabinet + +import ( + "bytes" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/url" +) + +type Client struct { + BaseURL string + HTTP *http.Client + Token string +} + +func New(baseURL string, opts ...ClientOption) *Client { + c := &Client{ + BaseURL: baseURL, + HTTP: http.DefaultClient, + } + for _, opt := range opts { + opt(c) + } + return c +} + +type ClientOption func(*Client) + +func WithHTTPClient(client *http.Client) ClientOption { + return func(c *Client) { + c.HTTP = client + } +} + +func WithToken(token string) ClientOption { + return func(c *Client) { + c.Token = token + } +} + +func (c *Client) Redirect(u string) (string, *http.Response, error) { + vals := url.Values{ + "url": []string{u}, + } + if c.Token != "" { + vals["token"] = []string{c.Token} + } + + resp, err := c.HTTP.PostForm(fmt.Sprintf("%s/r", c.BaseURL), vals) + if err != nil { + return "", nil, err + } + + if resp.StatusCode != 200 { + return "", resp, fmt.Errorf("non-200 status code: %s", resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", resp, err + } + resp.Body = io.NopCloser(bytes.NewReader(body)) + + return string(body), resp, nil +} + +func (c *Client) File(name string, f io.Reader) (string, *http.Response, error) { + var buf bytes.Buffer + mp := multipart.NewWriter(&buf) + + token, err := mp.CreateFormField("token") + if err != nil { + return "", nil, err + } + if _, err := token.Write([]byte(c.Token)); err != nil { + return "", nil, err + } + + file, err := mp.CreateFormFile("file", name) + if err != nil { + return "", nil, err + } + + if _, err := io.Copy(file, f); err != nil { + return "", nil, err + } + + if err := mp.Close(); err != nil { + return "", nil, err + } + + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/f", c.BaseURL), &buf) + if err != nil { + return "", nil, err + } + req.Header.Set("Content-Type", mp.FormDataContentType()) + + resp, err := c.HTTP.Do(req) + if err != nil { + return "", nil, err + } + + if resp.StatusCode != 200 { + return "", resp, fmt.Errorf("non-200 status code: %s", resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", resp, err + } + resp.Body = io.NopCloser(bytes.NewReader(body)) + + return string(body), resp, nil +} diff --git a/command.go b/cmd/cabinet/command.go similarity index 85% rename from command.go rename to cmd/cabinet/command.go index 7c4be5d..e1a7720 100644 --- a/command.go +++ b/cmd/cabinet/command.go @@ -7,13 +7,14 @@ import ( "os" "time" + "go.jolheiser.com/cabinet/internal/gc" + router2 "go.jolheiser.com/cabinet/internal/router" + workspace2 "go.jolheiser.com/cabinet/internal/workspace" + "github.com/AlecAivazis/survey/v2" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "go.jolheiser.com/cabinet/gc" - "go.jolheiser.com/cabinet/router" - "go.jolheiser.com/cabinet/workspace" ) type serveOpts struct { @@ -44,14 +45,14 @@ var serveCmd = func(opts *serveOpts) func(context.Context, []string) error { opts.domain = fmt.Sprintf("http://localhost:%d", opts.port) } - ws, err := workspace.New(opts.workspacePath) + ws, err := workspace2.New(opts.workspacePath) if err != nil { log.Fatal().Err(err).Msg("could not open workspace") } go gc.Start(ws, opts.maxDiskSize, opts.gcInterval) - r := router.New(opts.domain, ws, router.NewLimit(opts.requestPerMinute, opts.sizePerMinute, opts.burstSize, opts.memPerRequest)) + r := router2.New(opts.domain, ws, router2.NewLimit(opts.requestPerMinute, opts.sizePerMinute, opts.burstSize, opts.memPerRequest)) portStr := fmt.Sprintf(":%d", opts.port) log.Info().Msgf("Listening at http://localhost%s", portStr) @@ -64,7 +65,7 @@ var serveCmd = func(opts *serveOpts) func(context.Context, []string) error { type tokenOpts struct { token string - perm workspace.TokenPermission + perm workspace2.TokenPermission desc string workspacePath string delete bool @@ -72,7 +73,7 @@ type tokenOpts struct { var tokenCmd = func(opts *tokenOpts) func(context.Context, []string) error { return func(_ context.Context, _ []string) error { - ws, err := workspace.New(opts.workspacePath) + ws, err := workspace2.New(opts.workspacePath) if err != nil { log.Fatal().Err(err).Msg("could not open workspace") } @@ -84,7 +85,7 @@ var tokenCmd = func(opts *tokenOpts) func(context.Context, []string) error { } } -func addToken(ws *workspace.Workspace, opts *tokenOpts) error { +func addToken(ws *workspace2.Workspace, opts *tokenOpts) error { token := opts.token if token == "" { r, err := randToken() @@ -118,7 +119,7 @@ func addToken(ws *workspace.Workspace, opts *tokenOpts) error { if err != nil { return err } - perm, err = workspace.ParseTokenPermission(p) + perm, err = workspace2.ParseTokenPermission(p) if err != nil { return err } @@ -136,7 +137,7 @@ func addToken(ws *workspace.Workspace, opts *tokenOpts) error { } } - if err := ws.AddToken(workspace.Token{ + if err := ws.AddToken(workspace2.Token{ Key: token, Permission: perm, Description: desc, @@ -148,7 +149,7 @@ func addToken(ws *workspace.Workspace, opts *tokenOpts) error { return nil } -func deleteTokens(ws *workspace.Workspace, opts *tokenOpts) error { +func deleteTokens(ws *workspace2.Workspace, opts *tokenOpts) error { if opts.token != "" { if err := ws.DeleteTokens(opts.token); err != nil { return err diff --git a/main.go b/cmd/cabinet/main.go similarity index 98% rename from main.go rename to cmd/cabinet/main.go index 3e380c9..d2b9084 100644 --- a/main.go +++ b/cmd/cabinet/main.go @@ -13,12 +13,13 @@ import ( "strings" "time" + "go.jolheiser.com/cabinet/internal/workspace" + "github.com/peterbourgon/ff/v3" "github.com/peterbourgon/ff/v3/ffcli" "github.com/peterbourgon/ff/v3/fftoml" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "go.jolheiser.com/cabinet/workspace" ) func main() { @@ -39,7 +40,7 @@ func main() { serveFS.Func("max-disk-size", "Max size of all disk space usage", fileSizeParse(&serveOpts.maxDiskSize)) serveFS.StringVar(&serveOpts.workspacePath, "workspace", ".cabinet", "Workspace for DB, files, tokens, etc.") serveFS.DurationVar(&serveOpts.gcInterval, "gc-interval", time.Hour, "GC interval for cleaning up files") - serveFS.IntVar(&serveOpts.requestPerMinute, "request-per-minute", 12, "Request limit per-second") + serveFS.IntVar(&serveOpts.requestPerMinute, "request-per-minute", 12, "Request limit per-minute") serveFS.Func("size-per-second", "File size limit per-second", fileSizeParse(&serveOpts.sizePerMinute)) serveFS.Func("burst-size", "Burst size for files", fileSizeParse(&serveOpts.burstSize)) serveFS.Func("mem-per-request", "Memory per request before storing on disk temporarily", fileSizeParse(&serveOpts.memPerRequest)) diff --git a/go.mod b/go.mod index 15368dc..b447109 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,12 @@ module go.jolheiser.com/cabinet -go 1.17 +go 1.18 require ( github.com/AlecAivazis/survey/v2 v2.3.2 github.com/bwmarrin/snowflake v0.3.0 github.com/go-chi/chi/v5 v5.0.7 + github.com/matryer/is v1.4.0 github.com/peterbourgon/ff/v3 v3.1.2 github.com/rs/zerolog v1.26.1 go.etcd.io/bbolt v1.3.6 diff --git a/go.sum b/go.sum index 32010e2..7dd4fd0 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kr/pty v1.1.4 h1:5Myjjh3JY/NaAi4IsUbHADytDyl1VE1Y9PXDlL+P/VQ= github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= diff --git a/gc/gc.go b/internal/gc/gc.go similarity index 95% rename from gc/gc.go rename to internal/gc/gc.go index 3f2c1c3..3add336 100644 --- a/gc/gc.go +++ b/internal/gc/gc.go @@ -29,10 +29,6 @@ func Start(g GC, maxSize int, interval time.Duration) { size += int(info.Size()) } - if size < maxSize { - continue - } - sort.Sort(sorter(infos)) for size > maxSize { fi := infos[0] diff --git a/gc/sort.go b/internal/gc/sort.go similarity index 100% rename from gc/sort.go rename to internal/gc/sort.go diff --git a/router/rate.go b/internal/router/rate.go similarity index 97% rename from router/rate.go rename to internal/router/rate.go index 54757a2..f7b7b47 100644 --- a/router/rate.go +++ b/internal/router/rate.go @@ -82,7 +82,7 @@ func (l *Limit) hasHitRequestLimit(remoteAddr string) bool { if l.request[ip] == nil { l.request[ip] = rate.NewLimiter(rate.Limit(l.rpm/60), l.rpm) - l.size[ip] = rate.NewLimiter(rate.Limit(l.spm), l.burst) + l.size[ip] = rate.NewLimiter(rate.Limit(l.spm/60), l.burst) } return !l.request[ip].Allow() diff --git a/router/router.go b/internal/router/router.go similarity index 97% rename from router/router.go rename to internal/router/router.go index b4daeb7..8108bcf 100644 --- a/router/router.go +++ b/internal/router/router.go @@ -7,12 +7,12 @@ import ( "net/url" "strings" - "github.com/go-chi/chi/v5/middleware" - "github.com/rs/zerolog/log" - "go.jolheiser.com/cabinet/static" + "go.jolheiser.com/cabinet/internal/static" + "go.jolheiser.com/cabinet/internal/workspace" "github.com/go-chi/chi/v5" - "go.jolheiser.com/cabinet/workspace" + "github.com/go-chi/chi/v5/middleware" + "github.com/rs/zerolog/log" ) type Cabinet interface { diff --git a/internal/router/router_test.go b/internal/router/router_test.go new file mode 100644 index 0000000..e550c66 --- /dev/null +++ b/internal/router/router_test.go @@ -0,0 +1,43 @@ +package router + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/matryer/is" + "go.jolheiser.com/cabinet" + "go.jolheiser.com/cabinet/internal/workspace/mock" +) + +func TestRouter(t *testing.T) { + assert := is.New(t) + + m := mock.New() + server := httptest.NewUnstartedServer(nil) + server.Start() + defer server.Close() + r := New(server.URL, m, NewLimit(12, 5*1024*1024, 100*1024*1024, 1024*1024)) + server.Config.Handler = r + + c := cabinet.New(server.URL, cabinet.WithHTTPClient(server.Client())) + + // Redirect + redir := "https://duckduckgo.com" + u, _, err := c.Redirect(redir) + assert.NoErr(err) // Creating a redirect should succeed + resp, err := http.Get(u) + assert.NoErr(err) + assert.Equal(redir, resp.Request.URL.String()) // The redirect should match what was given + + file := "foobar" + f, _, err := c.File("test.txt", strings.NewReader(file)) + assert.NoErr(err) // Creating a file should succeed + resp, err = http.Get(f) + assert.NoErr(err) + b, err := io.ReadAll(resp.Body) + assert.NoErr(err) + assert.Equal(file, string(b)) // The file should match what was given +} diff --git a/router/token.go b/internal/router/token.go similarity index 95% rename from router/token.go rename to internal/router/token.go index 108eaf9..41237de 100644 --- a/router/token.go +++ b/internal/router/token.go @@ -3,7 +3,7 @@ package router import ( "net/http" - "go.jolheiser.com/cabinet/workspace" + "go.jolheiser.com/cabinet/internal/workspace" ) func tokenMiddleware(c Cabinet, perm workspace.TokenPermission) func(handler http.Handler) http.Handler { diff --git a/static/index.tmpl b/internal/static/index.tmpl similarity index 100% rename from static/index.tmpl rename to internal/static/index.tmpl diff --git a/static/sakura.css b/internal/static/sakura.css similarity index 100% rename from static/sakura.css rename to internal/static/sakura.css diff --git a/static/static.go b/internal/static/static.go similarity index 100% rename from static/static.go rename to internal/static/static.go diff --git a/workspace/meta.go b/internal/workspace/meta.go similarity index 100% rename from workspace/meta.go rename to internal/workspace/meta.go diff --git a/workspace/mock/file.go b/internal/workspace/mock/file.go similarity index 90% rename from workspace/mock/file.go rename to internal/workspace/mock/file.go index 519dc47..bf26e22 100644 --- a/workspace/mock/file.go +++ b/internal/workspace/mock/file.go @@ -4,7 +4,7 @@ import ( "os" "time" - "go.jolheiser.com/cabinet/workspace" + "go.jolheiser.com/cabinet/internal/workspace" ) type File struct { diff --git a/workspace/mock/mock.go b/internal/workspace/mock/mock.go similarity index 83% rename from workspace/mock/mock.go rename to internal/workspace/mock/mock.go index b8c2b97..1944fe9 100644 --- a/workspace/mock/mock.go +++ b/internal/workspace/mock/mock.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "go.jolheiser.com/cabinet/workspace" + "go.jolheiser.com/cabinet/internal/workspace" ) type Mock struct { @@ -18,7 +18,8 @@ type Mock struct { func New() *Mock { return &Mock{ - files: make(map[string]*File), + files: make(map[string]*File), + tokens: make(map[string]workspace.Token), } } @@ -72,6 +73,13 @@ func (m Mock) IsProtected() (bool, error) { return len(m.tokens) > 0, nil } +func (m Mock) Token(key string) (workspace.Token, error) { + if token, ok := m.tokens[key]; ok { + return token, nil + } + return workspace.Token{}, fmt.Errorf("no token found for %s", key) +} + func (m Mock) AddToken(token workspace.Token) error { m.tokens[token.Key] = token return nil diff --git a/workspace/token.go b/internal/workspace/token.go similarity index 100% rename from workspace/token.go rename to internal/workspace/token.go diff --git a/workspace/workspace.go b/internal/workspace/workspace.go similarity index 100% rename from workspace/workspace.go rename to internal/workspace/workspace.go diff --git a/router/logger.go b/router/logger.go deleted file mode 100644 index 7ef135b..0000000 --- a/router/logger.go +++ /dev/null @@ -1 +0,0 @@ -package router