Add tags

README.md

@@ -70,6 +70,23 @@ Overrides are available via config or by setting an environment variable `VANITY
 
 To run Vanity in config-only mode for packages, set `--service` to `off`.
 
+## Manual Mode
+
+To run Vanity without automatic updating, use `--manual`.
+
+## Topic Lists
+
+By setting `--topics`, anyone visiting the index page will see packages grouped by their topics.
+
+Regardless of the setting, you can switch beteween list-view and topic-view with the provided button
+or changing the URL between `?format=list` and `?format=topics`.
+
+## API
+
+In order to preserve namespaces for packages, Vanity's API uses the URL `/_/{endpoint}`
+
+Vanity currently supports `/_/status` and `/_/update`, to get some status information and update the package cache respectively.
+
 ## License
 
 [MIT](LICENSE)

api/package.go

@@ -6,12 +6,13 @@ import (
 )
 
 type Package struct {
-	Name        string `toml:"name"`
+	Name        string   `toml:"name"`
-	Description string `toml:"description"`
+	Description string   `toml:"description"`
-	Branch      string `toml:"branch"`
+	Branch      string   `toml:"branch"`
-	WebURL      string `toml:"web_url"`
+	WebURL      string   `toml:"web_url"`
-	CloneHTTP   string `toml:"clone_http"`
+	CloneHTTP   string   `toml:"clone_http"`
-	CloneSSH    string `toml:"clone_ssh"`
+	CloneSSH    string   `toml:"clone_ssh"`
+	Topics      []string `toml:"topics"`
 
 	Private bool `toml:"-"`
 	Fork    bool `toml:"-"`

flags/flags.go

@@ -36,6 +36,7 @@ var (
 	Override  = make(map[string]string)
 	Interval  time.Duration
 	Manual    bool
+	Topics    bool
 	Debug     bool
 
 	ConfigPackages []*api.Package
@@ -142,6 +143,12 @@ var Flags = []cli.Flag{
 		EnvVars:     []string{"VANITY_MANUAL"},
 		Destination: &Manual,
 	},
+	&cli.BoolFlag{
+		Name:        "topics",
+		Usage:       "Group projects by topic by default",
+		EnvVars:     []string{"VANITY_TOPICS"},
+		Destination: &Topics,
+	},
 	&cli.BoolFlag{
 		Name:        "debug",
 		Usage:       "Debug logging",

go.mod

@@ -6,6 +6,7 @@ require (
 	code.gitea.io/sdk/gitea v0.13.2
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/go-chi/chi v4.1.2+incompatible
+	github.com/go-chi/cors v1.1.1 // indirect
 	github.com/google/go-github/v32 v32.1.0
 	github.com/pelletier/go-toml v1.8.1
 	github.com/urfave/cli/v2 v2.2.0

go.sum

@@ -53,6 +53,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
 github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-chi/cors v1.1.1 h1:eHuqxsIw89iXcWnWUN8R72JMibABJTN/4IOYI5WERvw=
+github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=

router/api.go

@@ -0,0 +1,71 @@
+package router
+
+import (
+	"encoding/json"
+	"github.com/go-chi/chi"
+	"github.com/go-chi/cors"
+	"go.jolheiser.com/beaver"
+	"go.jolheiser.com/vanity/api"
+	"go.jolheiser.com/vanity/flags"
+	"net/http"
+	"runtime"
+	"time"
+)
+
+func apiRoutes() *chi.Mux {
+	r := chi.NewRouter()
+	r.Use(cors.AllowAll().Handler)
+
+	r.Get("/status", doAPIStatus)
+	r.Get("/update", doAPIUpdate)
+
+	return r
+}
+
+func doAPIStatus(res http.ResponseWriter, _ *http.Request) {
+	res.Header().Set("Content-Type", "application/json")
+
+	var nextUpdate *string
+	if !lastUpdate.IsZero() {
+		nu := lastUpdate.Add(flags.Interval).Format(time.RFC3339)
+		nextUpdate = &nu
+	}
+
+	resp := map[string]interface{}{
+		"vanity_version": api.Version,
+		"go_version":     runtime.Version(),
+		"num_packages":   len(cache.Packages),
+		"next_update":    nextUpdate,
+	}
+
+	data, err := json.Marshal(&resp)
+	if err != nil {
+		beaver.Errorf("could not marshal API status: %v", err)
+		data = []byte("{}")
+	}
+
+	if _, err = res.Write(data); err != nil {
+		beaver.Errorf("could not write response: %v", err)
+	}
+}
+
+func doAPIUpdate(res http.ResponseWriter, _ *http.Request) {
+	res.Header().Set("Content-Type", "application/json")
+
+	resp := map[string]bool{
+		"updated": false,
+	}
+	if canUpdate {
+		updateCache()
+		resp["updated"] = true
+	}
+
+	payload, err := json.Marshal(resp)
+	if err != nil {
+		beaver.Errorf("could not marshal payload: %v", err)
+	}
+
+	if _, err = res.Write(payload); err != nil {
+		beaver.Errorf("could not write response: %v", err)
+	}
+}

router/cache.go

@@ -11,30 +11,46 @@ import (
 )
 
 var cache = &packageCache{
-	packages: make(map[string]*api.Package),
+	Packages: make(map[string]*api.Package),
+}
+
+type packageList map[string]*api.Package
+
+func (p packageList) Names() []string {
+	names := make([]string, len(p))
+	idx := 0
+	for name := range p {
+		names[idx] = name
+		idx++
+	}
+	return names
+}
+
+func (p packageList) Topics() map[string][]*api.Package {
+	topics := make(map[string][]*api.Package, 0)
+	for _, pkg := range p {
+		for _, t := range pkg.Topics {
+			if tt, ok := topics[t]; ok {
+				topics[t] = append(tt, pkg)
+			} else {
+				topics[t] = []*api.Package{pkg}
+			}
+		}
+	}
+	return topics
 }
 
 type packageCache struct {
-	packages map[string]*api.Package
+	Packages packageList
 	sync.Mutex
 }
 
 func (c *packageCache) Update(packages map[string]*api.Package) {
 	c.Lock()
-	c.packages = packages
+	c.Packages = packages
 	c.Unlock()
 }
 
-func (c *packageCache) Names() []string {
-	names := make([]string, len(c.packages))
-	idx := 0
-	for name := range c.packages {
-		names[idx] = name
-		idx++
-	}
-	return names
-}
-
 func updateCache() {
 	packages, err := svc.Packages()
 	if err != nil {

router/cron.go

@@ -9,8 +9,9 @@ import (
 )
 
 var (
-	svc       service.Service
+	svc        service.Service
-	canUpdate bool
+	lastUpdate time.Time
+	canUpdate  bool
 )
 
 func cronStart() {
@@ -21,7 +22,8 @@ func cronStart() {
 		if !flags.Manual && canUpdate {
 			beaver.Debug("Running package update...")
 			updateCache()
-			beaver.Debugf("Finished package update: %s", cache.Names())
+			beaver.Debugf("Finished package update: %s", cache.Packages.Names())
+			lastUpdate = time.Now()
 		}
 		canUpdate = true
 	}

router/router.go

@@ -1,7 +1,6 @@
 package router
 
 import (
-	"encoding/json"
 	"fmt"
 	"html/template"
 	"net/http"
@@ -31,24 +30,34 @@ func Init() (*chi.Mux, error) {
 	r.Use(middleware.Timeout(30 * time.Second))
 
 	r.Get("/", doIndex)
-	r.Head("/", doUpdate)
 	r.Get("/*", doVanity)
+	r.Mount("/_", apiRoutes())
 
 	svc = service.New()
 
 	beaver.Info("Warming up cache...")
 	updateCache()
-	beaver.Infof("Finished warming up cache: %s", cache.Names())
+	beaver.Infof("Finished warming up cache: %s", cache.Packages.Names())
 	go cronStart()
 
 	beaver.Infof("Running vanity server at http://localhost:%d", flags.Port)
 	return r, nil
 }
 
-func doIndex(res http.ResponseWriter, _ *http.Request) {
+func doIndex(res http.ResponseWriter, req *http.Request) {
+	format := "list"
+	if flags.Topics {
+		format = "topics"
+	}
+	q := req.URL.Query().Get("format")
+	if q != "" {
+		format = q
+	}
+
 	if err := tmpl.Lookup("index.tmpl").Execute(res, map[string]interface{}{
-		"Packages": cache.packages,
+		"Packages": cache.Packages,
-		"Index": true,
+		"Index":    true,
+		"Format":   format,
 	}); err != nil {
 		beaver.Errorf("could not write response: %v", err)
 	}
@@ -56,7 +65,7 @@ func doIndex(res http.ResponseWriter, _ *http.Request) {
 
 func doVanity(res http.ResponseWriter, req *http.Request) {
 	key := chi.URLParam(req, "*")
-	pkg, ok := cache.packages[strings.Split(key, "/")[0]]
+	pkg, ok := cache.Packages[strings.Split(key, "/")[0]]
 	if !ok {
 		http.NotFound(res, req)
 		return
@@ -66,29 +75,8 @@ func doVanity(res http.ResponseWriter, req *http.Request) {
 		"Package":  pkg,
 		"Module":   pkg.Module(flags.Domain),
 		"GoSource": fmt.Sprintf("%s %s %s %s", pkg.Module(flags.Domain), pkg.CloneHTTP, svc.GoDir(pkg), svc.GoFile(pkg)),
-		"Index": false,
+		"Index":    false,
 	}); err != nil {
 		beaver.Errorf("could not write response: %v", err)
 	}
 }
-
-func doUpdate(res http.ResponseWriter, _ *http.Request) {
-	res.Header().Set("Content-Type", "application/json")
-
-	resp := map[string]bool{
-		"updated": false,
-	}
-	if canUpdate {
-		updateCache()
-		resp["updated"] = true
-	}
-
-	payload, err := json.Marshal(resp)
-	if err != nil {
-		beaver.Errorf("could not marshal payload: %v", err)
-	}
-
-	if _, err = res.Write(payload); err != nil {
-		beaver.Errorf("could not write response: %v", err)
-	}
-}

router/templates/foot.tmpl

@@ -12,8 +12,8 @@
     updateImports.addEventListener('click', () => {
         updateImports.disabled = true;
         updateImports.innerHTML = 'Updating...';
-        fetch('{{if .Index}}.{{else}}../{{end}}', {
+        fetch('{{if .Index}}.{{else}}../{{end}}_/update', {
-            method: 'HEAD'
+            method: 'GET'
         }).then(() => {
             location.reload();
         }).catch(() => {

router/templates/index.tmpl

@@ -1,8 +1,29 @@
 {{template "head.tmpl" .}}
 	<h3>Imports:</h3>
-	<ul>
+	{{if eq .Format "list"}}
-		{{range $path, $package := .Packages}}
+		<ul>
-			<li><a href="{{$package.Name}}">{{$package.Name}}</a></li>
+			{{range $path, $package := .Packages}}
+				<li><a href="{{$package.Name}}">{{$package.Name}}</a></li>
+			{{end}}
+		</ul>
+	{{else}}
+		{{range $topic, $packages := .Packages.Topics}}
+			<details>
+				<summary>{{$topic}}</summary>
+				<ul>
+					{{range $package := $packages}}
+						<li><a href="{{$package.Name}}">{{$package.Name}}</a></li>
+					{{end}}
+				</ul>
+			</details>
 		{{end}}
-	</ul>
+	{{end}}
+	<br/>
+	<form method="get">
+		{{if eq .Format "list"}}
+			<button type="submit" name="format" value="topics">See Topics</button>
+		{{else}}
+			<button type="submit" name="format" value="list"> See List</button>
+		{{end}}
+	</form>
 {{template "foot.tmpl" .}}

service/gitea.go

@@ -28,6 +28,12 @@ type Gitea struct {
 
 func (g Gitea) Packages() (map[string]*api.Package, error) {
 	packages := make(map[string]*api.Package)
+	topicOpts := gitea.ListRepoTopicsOptions{
+		ListOptions: gitea.ListOptions{
+			Page:     1,
+			PageSize: 50,
+		},
+	}
 	page := 0
 	for {
 		opts := gitea.ListReposOptions{
@@ -43,7 +49,7 @@ func (g Gitea) Packages() (map[string]*api.Package, error) {
 		}
 
 		for _, repo := range repos {
-			packages[repo.Name] = &api.Package{
+			pkg := &api.Package{
 				Name:        repo.Name,
 				Description: repo.Description,
 				Branch:      repo.DefaultBranch,
@@ -55,6 +61,15 @@ func (g Gitea) Packages() (map[string]*api.Package, error) {
 				Mirror:      repo.Mirror,
 				Archive:     repo.Archived,
 			}
+
+			// Get tags
+			topics, _, err := g.client.ListRepoTopics(flags.Namespace, repo.Name, topicOpts)
+			if err != nil {
+				beaver.Warnf("could not get topics for %s: %v", repo.Name, err)
+			}
+			pkg.Topics = topics
+
+			packages[repo.Name] = pkg
 		}
 
 		page++

service/github.go

@@ -57,6 +57,7 @@ func (g GitHub) Packages() (map[string]*api.Package, error) {
 				Fork:        repo.GetFork(),
 				Mirror:      false,
 				Archive:     repo.GetArchived(),
+				Topics:      repo.Topics,
 			}
 		}
 

service/gitlab.go

@@ -55,6 +55,7 @@ func (g GitLab) Packages() (map[string]*api.Package, error) {
 				Fork:        repo.ForkedFromProject != nil,
 				Mirror:      repo.Mirror,
 				Archive:     repo.Archived,
+				Topics:      repo.TagList,
 			}
 		}