feat(multisite): supports multiple tenants to a single deployment instance

This change adds support for managing multiple sites from a single deployment
of the pages application. Routes are configured using a configuration file
rather than looking for CNAMEs in all the associated repositories. For an
example configuration, see examples/multisite/pages.toml.

Resolves #4

examples/multisite/pages.json

@@ -0,0 +1,14 @@
+{
+  "sites": {
+    "mjpitz.com": {
+      "url": "https://github.com/mjpitz/mjpitz.git",
+      "branch": "gh-pages",
+      "sync_interval": 300000000000
+    },
+    "go.pitz.tech": {
+      "url": "https://github.com/mjpitz/myago.git",
+      "branch": "site",
+      "sync_interval": 1500000000000
+    }
+  }
+}

examples/multisite/pages.toml

@@ -0,0 +1,9 @@
+[sites."mjpitz.com"]
+  url = "https://github.com/mjpitz/mjpitz.git"
+  branch = "gh-pages"
+  syncInterval = "5m"
+
+[sites."go.pitz.tech"]
+  url = "https://github.com/mjpitz/myago.git"
+  branch = "site"
+  syncInterval = "15m"

examples/multisite/pages.yaml

@@ -0,0 +1,11 @@
+# sync interval not currently working for YAML
+sites:
+  mjpitz.com:
+    url: "https://github.com/mjpitz/mjpitz.git"
+    branch: "gh-pages"
+    syncInterval: 300000000000 # 5m
+
+  go.pitz.tech:
+    url: "https://github.com/mjpitz/myago.git"
+    branch: "site"
+    syncInterval: 300000000000 # 5m

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
+	github.com/jonboulle/clockwork v0.2.2
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.2
 	go.uber.org/zap v1.19.1
@@ -31,16 +32,18 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
-	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
@@ -53,4 +56,5 @@ require (
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/protobuf v1.26.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )

go.sum

@@ -207,6 +207,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
+github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -252,10 +254,15 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/urfave/cli/v2 v2.8.1 h1:CGuYNZF9IKZY/rfBe3lJpccSoIY1ytfvmgQT90cNOl4=
 github.com/urfave/cli/v2 v2.8.1/go.mod h1:Z41J9TPoffeoqP0Iza0YbAhGvymRdZAd2uPmZ5JxRdY=
+github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
+github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
+github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
+github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
 github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=

internal/commands/host.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"mime"
 	"net/http"
-	"path"
 	"time"
 
 	"github.com/urfave/cli/v2"
@@ -30,13 +29,15 @@ import (
 	"code.pitz.tech/mya/pages/internal"
 	"code.pitz.tech/mya/pages/internal/git"
 
+	"github.com/mjpitz/myago/config"
 	"github.com/mjpitz/myago/flagset"
 	"github.com/mjpitz/myago/zaputil"
 )
 
 type HostConfig struct {
 	internal.ServerConfig
-	Git git.Config `json:"git"`
+	Git      git.Config `json:"git"`
+	SiteFile string     `json:"site_file" usage:"configure multiple sites using a single file"`
 }
 
 var (
@@ -63,62 +64,34 @@ var (
 			_ = mime.AddExtensionType(".yml", "application/yaml")
 			_ = mime.AddExtensionType(".json", "application/json")
 
-			gitService, err := git.NewService(hostConfig.Git)
-			if err != nil {
-				return err
-			}
-
-			err = gitService.Load(ctx.Context)
-			if err != nil {
-				return err
-			}
-
 			server, err := internal.NewServer(ctx.Context, hostConfig.ServerConfig)
 			if err != nil {
 				return err
 			}
 
-			{
+			endpointConfig := git.EndpointConfig{
-				server.AdminMux.
+				Sites: make(map[string]*git.Config),
-					HandleFunc("/sync", func(w http.ResponseWriter, r *http.Request) {
-						err = gitService.Sync(r.Context())
-						if err != nil {
-							log.Error(err.Error())
-							http.Error(w, "", http.StatusInternalServerError)
-							return
-						}
-					}).
-					Methods(http.MethodPost)
 			}
 
-			httpfs := http.FileServer(git.HTTP(gitService.FS))
+			if hostConfig.SiteFile == "" {
-			server.PublicMux.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				endpointConfig.Sites["*"] = &hostConfig.Git
-				values := r.URL.Query()
+			} else {
-
+				err = config.Load(ctx.Context, &endpointConfig, hostConfig.SiteFile)
-				if values.Get("go-get") == "1" {
+				if err != nil {
-					// peak for go-get requests
+					return err
-					_, name := path.Split(r.URL.Path)
-					index := path.Join(r.URL.Path, "index.html")
-
-					// if index.html exists, then use that
-					info, err := gitService.FS.Stat(index)
-					if err == nil {
-						file, err := gitService.FS.Open(index)
-						if err != nil {
-							http.Error(w, "", http.StatusInternalServerError)
-							return
-						}
-
-						defer file.Close()
-
-						http.ServeContent(w, r, name, info.ModTime(), file)
-						return
-					}
 				}
+			}
 
-				httpfs.ServeHTTP(w, r)
+			endpoint, err := git.NewEndpoint(ctx.Context, endpointConfig)
-				return
+			if err != nil {
-			}).Methods(http.MethodGet)
+				return err
+			}
+			defer endpoint.Close()
+
+			{ // git endpoints
+				server.AdminMux.HandleFunc("/sync", endpoint.Sync).Methods(http.MethodPost)
+				server.PublicMux.PathPrefix("/").HandlerFunc(endpoint.Lookup).Methods(http.MethodGet)
+			}
 
 			log.Info("serving",
 				zap.String("public", hostConfig.Public.Address),
@@ -127,24 +100,16 @@ var (
 			group, c := errgroup.WithContext(ctx.Context)
 			group.Go(server.ListenAndServe)
 			group.Go(func() error {
-				timer := time.NewTimer(hostConfig.Git.SyncInterval)
+				return endpoint.SyncLoop(ctx.Context)
-				defer timer.Stop()
-
-				for {
-					select {
-					case <-c.Done():
-						return nil
-					case <-timer.C:
-						_ = gitService.Sync(ctx.Context)
-
-						timer.Reset(hostConfig.Git.SyncInterval)
-					}
-				}
 			})
 
 			<-c.Done()
 
-			_ = server.Shutdown(context.Background())
+			shutdownTimeout := 30 * time.Second
+			timeout, cancelTimeout := context.WithTimeout(context.Background(), shutdownTimeout)
+			defer cancelTimeout()
+
+			_ = server.Shutdown(timeout)
 			_ = group.Wait()
 
 			err = c.Err()

internal/git/endpoint.go

@@ -0,0 +1,184 @@
+package git
+
+import (
+	"context"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/jonboulle/clockwork"
+	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/mjpitz/myago/clocks"
+	"github.com/mjpitz/myago/zaputil"
+)
+
+type EndpointConfig struct {
+	Sites map[string]*Config `json:"sites"`
+}
+
+func NewEndpoint(ctx context.Context, multi EndpointConfig) (endpoint *Endpoint, err error) {
+	clock := clocks.Extract(ctx)
+	log := zaputil.Extract(ctx)
+
+	endpoint = &Endpoint{
+		sites: make(map[string]*entry),
+	}
+
+	for domain, cfg := range multi.Sites {
+		log.Info("loading site",
+			zap.String("domain", domain),
+			zap.String("tag", cfg.Tag),
+			zap.String("branch", cfg.Branch),
+			zap.Duration("sync_interval", cfg.SyncInterval),
+		)
+
+		endpoint.sites[domain] = &entry{
+			config: *cfg,
+		}
+
+		endpoint.sites[domain].service, err = NewService(*cfg)
+		if err != nil {
+			return nil, err
+		}
+
+		err = endpoint.sites[domain].service.Load(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	for domain, cfg := range multi.Sites {
+		endpoint.sites[domain].ticker = clock.NewTicker(cfg.SyncInterval)
+	}
+
+	return endpoint, nil
+}
+
+type entry struct {
+	config  Config
+	service *Service
+	ticker  clockwork.Ticker
+}
+
+type Endpoint struct {
+	sites map[string]*entry
+}
+
+func (e *Endpoint) lookupSite(r *http.Request) *entry {
+	if len(e.sites) == 1 && e.sites["*"] != nil {
+		return e.sites["*"]
+	}
+
+	url, err := url.Parse(r.RequestURI)
+	if err != nil {
+		return nil
+	}
+
+	domain := url.Hostname()
+	switch {
+	case r.Header.Get("Host") != "":
+		domain = r.Header.Get("Host")
+	case r.Header.Get("X-Forwarded-Host") != "":
+		domain = r.Header.Get("X-Forwarded-Host")
+	}
+
+	return e.sites[domain]
+}
+
+func (e *Endpoint) Sync(w http.ResponseWriter, r *http.Request) {
+	entry := e.lookupSite(r)
+
+	if entry == nil {
+		http.Error(w, "", http.StatusBadRequest)
+		return
+	}
+
+	err := entry.service.Sync(r.Context())
+	if err != nil {
+		http.Error(w, "", http.StatusInternalServerError)
+		return
+	}
+}
+
+func (e *Endpoint) Lookup(w http.ResponseWriter, r *http.Request) {
+	entry := e.lookupSite(r)
+
+	if entry == nil {
+		http.Error(w, "", http.StatusNotFound)
+		return
+	}
+
+	// Lookup file
+	values := r.URL.Query()
+
+	if values.Get("go-get") == "1" {
+		// peak for go-get requests
+		_, name := path.Split(r.URL.Path)
+		index := path.Join(r.URL.Path, "index.html")
+
+		// if index.html exists, then use that
+		info, err := entry.service.FS.Stat(index)
+		if err == nil {
+			file, err := entry.service.FS.Open(index)
+			if err != nil {
+				http.Error(w, "", http.StatusInternalServerError)
+				return
+			}
+
+			defer file.Close()
+
+			http.ServeContent(w, r, name, info.ModTime(), file)
+			return
+		}
+	}
+
+	http.FileServer(HTTP(entry.service.FS)).ServeHTTP(w, r)
+}
+
+func (e *Endpoint) SyncLoop(ctx context.Context) error {
+	clock := clocks.Extract(ctx)
+
+	timer := clock.NewTicker(time.Second)
+	defer timer.Stop()
+
+	keys := make([]string, 0, len(e.sites))
+	for key := range e.sites {
+		keys = append(keys, key)
+	}
+
+	group := &errgroup.Group{}
+
+	for {
+		for i := 0; i < len(keys); i++ {
+			select {
+			case <-ctx.Done():
+				// context cancelled / hit deadline
+				return ctx.Err()
+
+			case <-e.sites[keys[i]].ticker.Chan():
+				service := e.sites[keys[i]].service
+
+				// ticker expired, sync the site, check the next
+				// sync happens in a background thread to avoid contention on this loop
+				group.Go(func() error {
+					return service.Sync(ctx)
+				})
+
+			case <-timer.Chan():
+				// times up, check the next
+				continue
+			}
+		}
+	}
+}
+
+func (e *Endpoint) Close() error {
+	for _, site := range e.sites {
+		site.ticker.Stop()
+	}
+
+	return nil
+}