mya/pages
1
Fork 0
Code Issues 5 Pull Requests Projects Releases Wiki Activity

feat(multisite): supports multiple tenants to a single deployment instance 

This change adds support for managing multiple sites from a single deployment
of the pages application. Routes are configured using a configuration file
rather than looking for CNAMEs in all the associated repositories. For an
example configuration, see examples/multisite/pages.toml.

Resolves #4
This commit is contained in:
Mya Pitzeruse 2022-10-25 10:19:25 -05:00
parent 16e60f38f7
commit 29de39b130
No known key found for this signature in database
GPG Key ID: C3ECFA648DAD27FA
7 changed files with 258 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
examples/multisite/pages.json Normal file
View File

@ -0,0 +1,14 @@
{
"sites": {
"mjpitz.com": {
"url": "https://github.com/mjpitz/mjpitz.git",
"branch": "gh-pages",
"sync_interval": 300000000000
},
"go.pitz.tech": {
"url": "https://github.com/mjpitz/myago.git",
"branch": "site",
"sync_interval": 1500000000000
}
}
}

9
examples/multisite/pages.toml Normal file
View File

@ -0,0 +1,9 @@
[sites."mjpitz.com"]
url = "https://github.com/mjpitz/mjpitz.git"
branch = "gh-pages"
syncInterval = "5m"
[sites."go.pitz.tech"]
url = "https://github.com/mjpitz/myago.git"
branch = "site"
syncInterval = "15m"

11
examples/multisite/pages.yaml Normal file
View File

@ -0,0 +1,11 @@
# sync interval not currently working for YAML
sites:
mjpitz.com:
url: "https://github.com/mjpitz/mjpitz.git"
branch: "gh-pages"
syncInterval: 300000000000 # 5m
go.pitz.tech:
url: "https://github.com/mjpitz/myago.git"
branch: "site"
syncInterval: 300000000000 # 5m

6
go.mod
View File

@ -13,6 +13,7 @@ require (
github.com/go-git/go-git/v5 v5.4.2
github.com/gorilla/mux v1.8.0
github.com/gorilla/websocket v1.5.0
github.com/jonboulle/clockwork v0.2.2
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.12.2
go.uber.org/zap v1.19.1
@ -31,16 +32,18 @@ require (
github.com/golang/protobuf v1.5.2 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.1.0 // indirect
github.com/spf13/afero v1.6.0 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xanzy/ssh-agent v0.3.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.uber.org/atomic v1.9.0 // indirect
@ -53,4 +56,5 @@ require (
google.golang.org/appengine v1.6.6 // indirect
google.golang.org/protobuf v1.26.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

7
go.sum
View File

@ -207,6 +207,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -252,10 +254,15 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/urfave/cli/v2 v2.8.1 h1:CGuYNZF9IKZY/rfBe3lJpccSoIY1ytfvmgQT90cNOl4=
github.com/urfave/cli/v2 v2.8.1/go.mod h1:Z41J9TPoffeoqP0Iza0YbAhGvymRdZAd2uPmZ5JxRdY=
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=

91
internal/commands/host.go
View File

@ -20,7 +20,6 @@ import (
"context"
"mime"
"net/http"
"path"
"time"
"github.com/urfave/cli/v2"
@ -30,13 +29,15 @@ import (
"code.pitz.tech/mya/pages/internal"
"code.pitz.tech/mya/pages/internal/git"
"github.com/mjpitz/myago/config"
"github.com/mjpitz/myago/flagset"
"github.com/mjpitz/myago/zaputil"
)
type HostConfig struct {
internal.ServerConfig
Git git.Config `json:"git"`
Git git.Config `json:"git"`
SiteFile string `json:"site_file" usage:"configure multiple sites using a single file"`
}
var (
@ -63,62 +64,34 @@ var (
_ = mime.AddExtensionType(".yml", "application/yaml")
_ = mime.AddExtensionType(".json", "application/json")
gitService, err := git.NewService(hostConfig.Git)
if err != nil {
return err
}
err = gitService.Load(ctx.Context)
if err != nil {
return err
}
server, err := internal.NewServer(ctx.Context, hostConfig.ServerConfig)
if err != nil {
return err
}
{
server.AdminMux.
HandleFunc("/sync", func(w http.ResponseWriter, r *http.Request) {
err = gitService.Sync(r.Context())
if err != nil {
log.Error(err.Error())
http.Error(w, "", http.StatusInternalServerError)
return
}
}).
Methods(http.MethodPost)
endpointConfig := git.EndpointConfig{
Sites: make(map[string]*git.Config),
}
httpfs := http.FileServer(git.HTTP(gitService.FS))
server.PublicMux.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
values := r.URL.Query()
if values.Get("go-get") == "1" {
// peak for go-get requests
_, name := path.Split(r.URL.Path)
index := path.Join(r.URL.Path, "index.html")
// if index.html exists, then use that
info, err := gitService.FS.Stat(index)
if err == nil {
file, err := gitService.FS.Open(index)
if err != nil {
http.Error(w, "", http.StatusInternalServerError)
return
}
defer file.Close()
http.ServeContent(w, r, name, info.ModTime(), file)
return
}
if hostConfig.SiteFile == "" {
endpointConfig.Sites["*"] = &hostConfig.Git
} else {
err = config.Load(ctx.Context, &endpointConfig, hostConfig.SiteFile)
if err != nil {
return err
}
}
httpfs.ServeHTTP(w, r)
return
}).Methods(http.MethodGet)
endpoint, err := git.NewEndpoint(ctx.Context, endpointConfig)
if err != nil {
return err
}
defer endpoint.Close()
{ // git endpoints
server.AdminMux.HandleFunc("/sync", endpoint.Sync).Methods(http.MethodPost)
server.PublicMux.PathPrefix("/").HandlerFunc(endpoint.Lookup).Methods(http.MethodGet)
}
log.Info("serving",
zap.String("public", hostConfig.Public.Address),
@ -127,24 +100,16 @@ var (
group, c := errgroup.WithContext(ctx.Context)
group.Go(server.ListenAndServe)
group.Go(func() error {
timer := time.NewTimer(hostConfig.Git.SyncInterval)
defer timer.Stop()
for {
select {
case <-c.Done():
return nil
case <-timer.C:
_ = gitService.Sync(ctx.Context)
timer.Reset(hostConfig.Git.SyncInterval)
}
}
return endpoint.SyncLoop(ctx.Context)
})
<-c.Done()
_ = server.Shutdown(context.Background())
shutdownTimeout := 30 * time.Second
timeout, cancelTimeout := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancelTimeout()
_ = server.Shutdown(timeout)
_ = group.Wait()
err = c.Err()

184
internal/git/endpoint.go Normal file
View File

@ -0,0 +1,184 @@
package git
import (
"context"
"net/http"
"net/url"
"path"
"time"
"github.com/jonboulle/clockwork"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
"github.com/mjpitz/myago/clocks"
"github.com/mjpitz/myago/zaputil"
)
type EndpointConfig struct {
Sites map[string]*Config `json:"sites"`
}
func NewEndpoint(ctx context.Context, multi EndpointConfig) (endpoint *Endpoint, err error) {
clock := clocks.Extract(ctx)
log := zaputil.Extract(ctx)
endpoint = &Endpoint{
sites: make(map[string]*entry),
}
for domain, cfg := range multi.Sites {
log.Info("loading site",
zap.String("domain", domain),
zap.String("tag", cfg.Tag),
zap.String("branch", cfg.Branch),
zap.Duration("sync_interval", cfg.SyncInterval),
)
endpoint.sites[domain] = &entry{
config: *cfg,
}
endpoint.sites[domain].service, err = NewService(*cfg)
if err != nil {
return nil, err
}
err = endpoint.sites[domain].service.Load(ctx)
if err != nil {
return nil, err
}
}
for domain, cfg := range multi.Sites {
endpoint.sites[domain].ticker = clock.NewTicker(cfg.SyncInterval)
}
return endpoint, nil
}
type entry struct {
config Config
service *Service
ticker clockwork.Ticker
}
type Endpoint struct {
sites map[string]*entry
}
func (e *Endpoint) lookupSite(r *http.Request) *entry {
if len(e.sites) == 1 && e.sites["*"] != nil {
return e.sites["*"]
}
url, err := url.Parse(r.RequestURI)
if err != nil {
return nil
}
domain := url.Hostname()
switch {
case r.Header.Get("Host") != "":
domain = r.Header.Get("Host")
case r.Header.Get("X-Forwarded-Host") != "":
domain = r.Header.Get("X-Forwarded-Host")
}
return e.sites[domain]
}
func (e *Endpoint) Sync(w http.ResponseWriter, r *http.Request) {
entry := e.lookupSite(r)
if entry == nil {
http.Error(w, "", http.StatusBadRequest)
return
}
err := entry.service.Sync(r.Context())
if err != nil {
http.Error(w, "", http.StatusInternalServerError)
return
}
}
func (e *Endpoint) Lookup(w http.ResponseWriter, r *http.Request) {
entry := e.lookupSite(r)
if entry == nil {
http.Error(w, "", http.StatusNotFound)
return
}
// Lookup file
values := r.URL.Query()
if values.Get("go-get") == "1" {
// peak for go-get requests
_, name := path.Split(r.URL.Path)
index := path.Join(r.URL.Path, "index.html")
// if index.html exists, then use that
info, err := entry.service.FS.Stat(index)
if err == nil {
file, err := entry.service.FS.Open(index)
if err != nil {
http.Error(w, "", http.StatusInternalServerError)
return
}
defer file.Close()
http.ServeContent(w, r, name, info.ModTime(), file)
return
}
}
http.FileServer(HTTP(entry.service.FS)).ServeHTTP(w, r)
}
func (e *Endpoint) SyncLoop(ctx context.Context) error {
clock := clocks.Extract(ctx)
timer := clock.NewTicker(time.Second)
defer timer.Stop()
keys := make([]string, 0, len(e.sites))
for key := range e.sites {
keys = append(keys, key)
}
group := &errgroup.Group{}
for {
for i := 0; i < len(keys); i++ {
select {
case <-ctx.Done():
// context cancelled / hit deadline
return ctx.Err()
case <-e.sites[keys[i]].ticker.Chan():
service := e.sites[keys[i]].service
// ticker expired, sync the site, check the next
// sync happens in a background thread to avoid contention on this loop
group.Go(func() error {
return service.Sync(ctx)
})
case <-timer.Chan():
// times up, check the next
continue
}
}
}
}
func (e *Endpoint) Close() error {
for _, site := range e.sites {
site.ticker.Stop()
}
return nil
}