From 29de39b130f98aa2c6547ec06c002bf626241ecb Mon Sep 17 00:00:00 2001 From: Mya Pitzeruse Date: Tue, 25 Oct 2022 10:19:25 -0500 Subject: [PATCH] feat(multisite): supports multiple tenants to a single deployment instance This change adds support for managing multiple sites from a single deployment of the pages application. Routes are configured using a configuration file rather than looking for CNAMEs in all the associated repositories. For an example configuration, see examples/multisite/pages.toml. Resolves https://code.pitz.tech/mya/pages/issues/4 --- examples/multisite/pages.json | 14 +++ examples/multisite/pages.toml | 9 ++ examples/multisite/pages.yaml | 11 ++ go.mod | 6 +- go.sum | 7 ++ internal/commands/host.go | 91 ++++++----------- internal/git/endpoint.go | 184 ++++++++++++++++++++++++++++++++++ 7 files changed, 258 insertions(+), 64 deletions(-) create mode 100644 examples/multisite/pages.json create mode 100644 examples/multisite/pages.toml create mode 100644 examples/multisite/pages.yaml create mode 100644 internal/git/endpoint.go diff --git a/examples/multisite/pages.json b/examples/multisite/pages.json new file mode 100644 index 0000000..6f733b3 --- /dev/null +++ b/examples/multisite/pages.json @@ -0,0 +1,14 @@ +{ + "sites": { + "mjpitz.com": { + "url": "https://github.com/mjpitz/mjpitz.git", + "branch": "gh-pages", + "sync_interval": 300000000000 + }, + "go.pitz.tech": { + "url": "https://github.com/mjpitz/myago.git", + "branch": "site", + "sync_interval": 1500000000000 + } + } +} diff --git a/examples/multisite/pages.toml b/examples/multisite/pages.toml new file mode 100644 index 0000000..17ce466 --- /dev/null +++ b/examples/multisite/pages.toml @@ -0,0 +1,9 @@ +[sites."mjpitz.com"] + url = "https://github.com/mjpitz/mjpitz.git" + branch = "gh-pages" + syncInterval = "5m" + +[sites."go.pitz.tech"] + url = "https://github.com/mjpitz/myago.git" + branch = "site" + syncInterval = "15m" diff --git a/examples/multisite/pages.yaml b/examples/multisite/pages.yaml new file mode 100644 index 0000000..06a9665 --- /dev/null +++ b/examples/multisite/pages.yaml @@ -0,0 +1,11 @@ +# sync interval not currently working for YAML +sites: + mjpitz.com: + url: "https://github.com/mjpitz/mjpitz.git" + branch: "gh-pages" + syncInterval: 300000000000 # 5m + + go.pitz.tech: + url: "https://github.com/mjpitz/myago.git" + branch: "site" + syncInterval: 300000000000 # 5m diff --git a/go.mod b/go.mod index f139d8d..1a14574 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 + github.com/jonboulle/clockwork v0.2.2 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.12.2 go.uber.org/zap v1.19.1 @@ -31,16 +32,18 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/jonboulle/clockwork v0.2.2 // indirect github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/pelletier/go-toml v1.9.4 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/spf13/afero v1.6.0 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/atomic v1.9.0 // indirect @@ -53,4 +56,5 @@ require ( google.golang.org/appengine v1.6.6 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 0b2ab0f..b17dee1 100644 --- a/go.sum +++ b/go.sum @@ -207,6 +207,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -252,10 +254,15 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/urfave/cli/v2 v2.8.1 h1:CGuYNZF9IKZY/rfBe3lJpccSoIY1ytfvmgQT90cNOl4= github.com/urfave/cli/v2 v2.8.1/go.mod h1:Z41J9TPoffeoqP0Iza0YbAhGvymRdZAd2uPmZ5JxRdY= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= diff --git a/internal/commands/host.go b/internal/commands/host.go index 2bfaeb0..c0727a3 100644 --- a/internal/commands/host.go +++ b/internal/commands/host.go @@ -20,7 +20,6 @@ import ( "context" "mime" "net/http" - "path" "time" "github.com/urfave/cli/v2" @@ -30,13 +29,15 @@ import ( "code.pitz.tech/mya/pages/internal" "code.pitz.tech/mya/pages/internal/git" + "github.com/mjpitz/myago/config" "github.com/mjpitz/myago/flagset" "github.com/mjpitz/myago/zaputil" ) type HostConfig struct { internal.ServerConfig - Git git.Config `json:"git"` + Git git.Config `json:"git"` + SiteFile string `json:"site_file" usage:"configure multiple sites using a single file"` } var ( @@ -63,62 +64,34 @@ var ( _ = mime.AddExtensionType(".yml", "application/yaml") _ = mime.AddExtensionType(".json", "application/json") - gitService, err := git.NewService(hostConfig.Git) - if err != nil { - return err - } - - err = gitService.Load(ctx.Context) - if err != nil { - return err - } - server, err := internal.NewServer(ctx.Context, hostConfig.ServerConfig) if err != nil { return err } - { - server.AdminMux. - HandleFunc("/sync", func(w http.ResponseWriter, r *http.Request) { - err = gitService.Sync(r.Context()) - if err != nil { - log.Error(err.Error()) - http.Error(w, "", http.StatusInternalServerError) - return - } - }). - Methods(http.MethodPost) + endpointConfig := git.EndpointConfig{ + Sites: make(map[string]*git.Config), } - httpfs := http.FileServer(git.HTTP(gitService.FS)) - server.PublicMux.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - values := r.URL.Query() - - if values.Get("go-get") == "1" { - // peak for go-get requests - _, name := path.Split(r.URL.Path) - index := path.Join(r.URL.Path, "index.html") - - // if index.html exists, then use that - info, err := gitService.FS.Stat(index) - if err == nil { - file, err := gitService.FS.Open(index) - if err != nil { - http.Error(w, "", http.StatusInternalServerError) - return - } - - defer file.Close() - - http.ServeContent(w, r, name, info.ModTime(), file) - return - } + if hostConfig.SiteFile == "" { + endpointConfig.Sites["*"] = &hostConfig.Git + } else { + err = config.Load(ctx.Context, &endpointConfig, hostConfig.SiteFile) + if err != nil { + return err } + } - httpfs.ServeHTTP(w, r) - return - }).Methods(http.MethodGet) + endpoint, err := git.NewEndpoint(ctx.Context, endpointConfig) + if err != nil { + return err + } + defer endpoint.Close() + + { // git endpoints + server.AdminMux.HandleFunc("/sync", endpoint.Sync).Methods(http.MethodPost) + server.PublicMux.PathPrefix("/").HandlerFunc(endpoint.Lookup).Methods(http.MethodGet) + } log.Info("serving", zap.String("public", hostConfig.Public.Address), @@ -127,24 +100,16 @@ var ( group, c := errgroup.WithContext(ctx.Context) group.Go(server.ListenAndServe) group.Go(func() error { - timer := time.NewTimer(hostConfig.Git.SyncInterval) - defer timer.Stop() - - for { - select { - case <-c.Done(): - return nil - case <-timer.C: - _ = gitService.Sync(ctx.Context) - - timer.Reset(hostConfig.Git.SyncInterval) - } - } + return endpoint.SyncLoop(ctx.Context) }) <-c.Done() - _ = server.Shutdown(context.Background()) + shutdownTimeout := 30 * time.Second + timeout, cancelTimeout := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancelTimeout() + + _ = server.Shutdown(timeout) _ = group.Wait() err = c.Err() diff --git a/internal/git/endpoint.go b/internal/git/endpoint.go new file mode 100644 index 0000000..8d6debf --- /dev/null +++ b/internal/git/endpoint.go @@ -0,0 +1,184 @@ +package git + +import ( + "context" + "net/http" + "net/url" + "path" + "time" + + "github.com/jonboulle/clockwork" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + + "github.com/mjpitz/myago/clocks" + "github.com/mjpitz/myago/zaputil" +) + +type EndpointConfig struct { + Sites map[string]*Config `json:"sites"` +} + +func NewEndpoint(ctx context.Context, multi EndpointConfig) (endpoint *Endpoint, err error) { + clock := clocks.Extract(ctx) + log := zaputil.Extract(ctx) + + endpoint = &Endpoint{ + sites: make(map[string]*entry), + } + + for domain, cfg := range multi.Sites { + log.Info("loading site", + zap.String("domain", domain), + zap.String("tag", cfg.Tag), + zap.String("branch", cfg.Branch), + zap.Duration("sync_interval", cfg.SyncInterval), + ) + + endpoint.sites[domain] = &entry{ + config: *cfg, + } + + endpoint.sites[domain].service, err = NewService(*cfg) + if err != nil { + return nil, err + } + + err = endpoint.sites[domain].service.Load(ctx) + if err != nil { + return nil, err + } + } + + for domain, cfg := range multi.Sites { + endpoint.sites[domain].ticker = clock.NewTicker(cfg.SyncInterval) + } + + return endpoint, nil +} + +type entry struct { + config Config + service *Service + ticker clockwork.Ticker +} + +type Endpoint struct { + sites map[string]*entry +} + +func (e *Endpoint) lookupSite(r *http.Request) *entry { + if len(e.sites) == 1 && e.sites["*"] != nil { + return e.sites["*"] + } + + url, err := url.Parse(r.RequestURI) + if err != nil { + return nil + } + + domain := url.Hostname() + switch { + case r.Header.Get("Host") != "": + domain = r.Header.Get("Host") + case r.Header.Get("X-Forwarded-Host") != "": + domain = r.Header.Get("X-Forwarded-Host") + } + + return e.sites[domain] +} + +func (e *Endpoint) Sync(w http.ResponseWriter, r *http.Request) { + entry := e.lookupSite(r) + + if entry == nil { + http.Error(w, "", http.StatusBadRequest) + return + } + + err := entry.service.Sync(r.Context()) + if err != nil { + http.Error(w, "", http.StatusInternalServerError) + return + } +} + +func (e *Endpoint) Lookup(w http.ResponseWriter, r *http.Request) { + entry := e.lookupSite(r) + + if entry == nil { + http.Error(w, "", http.StatusNotFound) + return + } + + // Lookup file + values := r.URL.Query() + + if values.Get("go-get") == "1" { + // peak for go-get requests + _, name := path.Split(r.URL.Path) + index := path.Join(r.URL.Path, "index.html") + + // if index.html exists, then use that + info, err := entry.service.FS.Stat(index) + if err == nil { + file, err := entry.service.FS.Open(index) + if err != nil { + http.Error(w, "", http.StatusInternalServerError) + return + } + + defer file.Close() + + http.ServeContent(w, r, name, info.ModTime(), file) + return + } + } + + http.FileServer(HTTP(entry.service.FS)).ServeHTTP(w, r) +} + +func (e *Endpoint) SyncLoop(ctx context.Context) error { + clock := clocks.Extract(ctx) + + timer := clock.NewTicker(time.Second) + defer timer.Stop() + + keys := make([]string, 0, len(e.sites)) + for key := range e.sites { + keys = append(keys, key) + } + + group := &errgroup.Group{} + + for { + for i := 0; i < len(keys); i++ { + select { + case <-ctx.Done(): + // context cancelled / hit deadline + return ctx.Err() + + case <-e.sites[keys[i]].ticker.Chan(): + service := e.sites[keys[i]].service + + // ticker expired, sync the site, check the next + // sync happens in a background thread to avoid contention on this loop + group.Go(func() error { + return service.Sync(ctx) + }) + + case <-timer.Chan(): + // times up, check the next + continue + } + } + } +} + +func (e *Endpoint) Close() error { + for _, site := range e.sites { + site.ticker.Stop() + } + + return nil +}