mya/pages
1
Fork 0
Code Issues 5 Pull Requests Projects Releases Wiki Activity

feat(session): support session tracking 

This change refactors a handful of components to make them reusable
across pageview and session tracking endpoints.

Resolves https://github.com/mjpitz/pages/issues/2
This commit is contained in:
Mya Pitzeruse 2022-06-08 02:14:27 -05:00
parent 9a745569c4
commit 02d115f682
No known key found for this signature in database
GPG Key ID: C3ECFA648DAD27FA
18 changed files with 1595 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
README.md
View File

@ -47,19 +47,30 @@ system.
The number of page views for a given path and their associated referrer.
```text
# HELP pages_page_view_count page views
# HELP pages_page_view_count the number of times a given page has been viewed and by what referrer
# TYPE pages_page_view_count counter
pages_page_view_count{country="",path="/charts/",referrer="http://localhost:8080/blog/"} 1
```
### pages_page_session_duration
### pages_page_session_seconds
How long a user is actively engaged with the page.
https://github.com/mjpitz/pages/issues/2
Histogram of how long users spend on a page.
```text
# HELP pages_page_session_duration time spent on a given page in seconds
# TYPE pages_page_session_duration histogram
pages_page_session_duration{country="",path="/charts/"}
# HELP pages_page_session_seconds how long someone spent on a given page
# TYPE pages_page_session_seconds histogram
pages_page_session_seconds_bucket{country="",path="/",le="0.005"} 0
pages_page_session_seconds_bucket{country="",path="/",le="0.01"} 0
pages_page_session_seconds_bucket{country="",path="/",le="0.025"} 0
pages_page_session_seconds_bucket{country="",path="/",le="0.05"} 0
pages_page_session_seconds_bucket{country="",path="/",le="0.1"} 0
pages_page_session_seconds_bucket{country="",path="/",le="0.25"} 0
pages_page_session_seconds_bucket{country="",path="/",le="0.5"} 0
pages_page_session_seconds_bucket{country="",path="/",le="1"} 0
pages_page_session_seconds_bucket{country="",path="/",le="2.5"} 1
pages_page_session_seconds_bucket{country="",path="/",le="5"} 1
pages_page_session_seconds_bucket{country="",path="/",le="10"} 1
pages_page_session_seconds_bucket{country="",path="/",le="+Inf"} 1
pages_page_session_seconds_sum{country="",path="/"} 1.855976549
pages_page_session_seconds_count{country="",path="/"} 1
```

1
go.mod
View File

@ -11,6 +11,7 @@ require (
github.com/go-git/go-billy/v5 v5.3.1
github.com/go-git/go-git/v5 v5.4.2
github.com/gorilla/mux v1.8.0
github.com/gorilla/websocket v1.5.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.12.2
go.uber.org/zap v1.19.1

2
go.sum
View File

@ -171,6 +171,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=

62
internal/excludes/excludes.go Normal file
View File

@ -0,0 +1,62 @@
// Copyright (C) 2022 The pages authors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package excludes
import (
"path"
"regexp"
"strings"
)
// Exclusion defines an abstraction for matching paths.
type Exclusion func(s string) bool
// AnyExclusion returns a matcher who returns true if any of the provided matchers match the string.
func AnyExclusion(exclusions ...Exclusion) Exclusion {
return func(s string) bool {
for _, exclusion := range exclusions {
if exclusion(s) {
return true
}
}
return false
}
}
// AssetExclusion returns a matcher who returns true when an asset file is requested.
func AssetExclusion() Exclusion {
return func(s string) bool {
return path.Ext(s) != ""
}
}
// PrefixExclusion returns a matcher who returns true if the string matches the provided prefix.
func PrefixExclusion(prefix string) Exclusion {
return func(s string) bool {
return strings.HasPrefix(s, prefix)
}
}
// RegexExclusion returns a matcher who returns true if the string matches the provided regular expression pattern.
func RegexExclusion(pattern string) Exclusion {
exp := regexp.MustCompile(pattern)
return func(s string) bool {
return exp.MatchString(s)
}
}

70
internal/geoip/middleware.go Normal file
View File

@ -0,0 +1,70 @@
// Copyright (C) 2022 The pages authors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package geoip
import (
"context"
"net"
"net/http"
"github.com/gorilla/mux"
"github.com/mjpitz/myago"
)
var key = myago.ContextKey("geoip.info")
type Info struct {
CountryCode string
}
func Extract(ctx context.Context) Info {
val := ctx.Value(key)
v, ok := val.(Info)
if val == nil || !ok {
return Info{}
}
return v
}
type Interface interface {
Lookup(ip string) Info
}
type Empty struct{}
func (e Empty) Lookup(ip string) Info {
return Info{}
}
func Middleware(geoip Interface) mux.MiddlewareFunc {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
clientIP, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
http.Error(w, "", http.StatusBadRequest)
return
}
ctx := context.WithValue(r.Context(), key, geoip.Lookup(clientIP))
next.ServeHTTP(w, r.WithContext(ctx))
})
}
}

12
internal/metrics/metrics.go
View File

@ -31,8 +31,18 @@ var (
Namespace: namespace,
Subsystem: page,
Name: "view_count",
Help: "page views",
Help: "the number of times a given page has been viewed and by what referrer",
},
[]string{"path", "referrer", "country"},
)
PageSessionDuration = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: page,
Name: "session_seconds",
Help: "how long someone spent on a given page",
},
[]string{"path", "country"},
)
)

79
internal/middleware.go → internal/pageviews/middleware.go
View File

@ -14,73 +14,40 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package internal
package pageviews
import (
"net"
"net/http"
"path"
"regexp"
"strings"
"github.com/gorilla/mux"
"github.com/mjpitz/pages/internal/excludes"
"github.com/mjpitz/pages/internal/geoip"
"github.com/mjpitz/pages/internal/metrics"
)
// GeoIP provides an abstraction for looking up a country code from an IP address.
type GeoIP interface {
Lookup(ip string) (countryCode string)
type opt struct {
excludes []excludes.Exclusion
}
type emptyGeoIP struct{}
// Option provides a way to configure elements of the Middleware.
type Option func(*opt)
func (e emptyGeoIP) Lookup(ip string) (countryCode string) {
return ""
}
// Matcher defines an abstraction for matching paths.
type Matcher func(s string) bool
// AnyMatcher returns a matcher who returns true if any of the provided matchers match the string.
func AnyMatcher(matchers ...Matcher) Matcher {
return func(s string) bool {
for _, matcher := range matchers {
if matcher(s) {
return true
}
}
return false
}
}
// AssetMatcher returns a matcher who returns true when an asset file is requested.
func AssetMatcher() Matcher {
return func(s string) bool {
return path.Ext(s) != ""
}
}
// PrefixMatcher returns a matcher who returns true if the string matches the provided prefix.
func PrefixMatcher(prefix string) Matcher {
return func(s string) bool {
return strings.HasPrefix(s, prefix)
}
}
// RegexMatcher returns a matcher who returns true if the string matches the provided regular expression pattern.
func RegexMatcher(pattern string) Matcher {
exp := regexp.MustCompile(pattern)
return func(s string) bool {
return exp.MatchString(s)
// Exclusions appends the provided rules to the excludes list. Any path that matches an exclusion will not be measured.
func Exclusions(exclusions ...excludes.Exclusion) Option {
return func(o *opt) {
o.excludes = append(o.excludes, exclusions...)
}
}
// Middleware produces an HTTP middleware function that reports page views.
func Middleware(geoIP GeoIP, excludes ...Matcher) mux.MiddlewareFunc {
exclude := AnyMatcher(excludes...)
func Middleware(opts ...Option) mux.MiddlewareFunc {
o := opt{}
for _, opt := range opts {
opt(&o)
}
exclude := excludes.AnyExclusion(o.excludes...)
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -89,18 +56,12 @@ func Middleware(geoIP GeoIP, excludes ...Matcher) mux.MiddlewareFunc {
return
}
clientIP, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
http.Error(w, "", http.StatusBadRequest)
return
}
countryCode := geoIP.Lookup(clientIP)
info := geoip.Extract(r.Context())
d := &writer{w, http.StatusOK}
defer func() {
if d.statusCode != http.StatusNotFound {
metrics.PageViewCount.WithLabelValues(r.URL.Path, r.Referer(), countryCode).Inc()
metrics.PageViewCount.WithLabelValues(r.URL.Path, r.Referer(), info.CountryCode).Inc()
}
}()

39
internal/server.go
View File

@ -20,6 +20,7 @@ import (
"context"
"net"
"net/http"
"path"
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus"
@ -31,11 +32,16 @@ import (
httpauth "github.com/mjpitz/myago/auth/http"
"github.com/mjpitz/myago/headers"
"github.com/mjpitz/myago/livetls"
"github.com/mjpitz/pages/internal/excludes"
"github.com/mjpitz/pages/internal/geoip"
"github.com/mjpitz/pages/internal/pageviews"
"github.com/mjpitz/pages/internal/session"
"github.com/mjpitz/pages/internal/web"
)
// AdminConfig encapsulates configuration for the administrative process.
// AdminConfig encapsulates configuration for the administrative endpoints.
type AdminConfig struct {
Prefix string `json:"prefix" usage:"configure the prefix to use for admin endpoints" default:"/_admin"`
Prefix string `json:"prefix" usage:"configure the prefix to use for admin endpoints" default:"/_admin" hidden:"true"`
Username string `json:"username" usage:"specify the username used to authenticate requests with the admin endpoints" default:"admin"`
Password string `json:"password" usage:"specify the password used to authenticate requests with the admin endpoints"`
}
@ -48,6 +54,7 @@ type BindConfig struct {
// ServerConfig defines configuration for a public and private interface.
type ServerConfig struct {
Admin AdminConfig `json:"admin"`
Session session.Config `json:"session"`
TLS livetls.Config `json:"tls"`
Public BindConfig `json:"public"`
Private BindConfig `json:"private"`
@ -63,17 +70,26 @@ func NewServer(ctx context.Context, config ServerConfig) (*Server, error) {
private := mux.NewRouter()
private.Handle("/metrics", promhttp.Handler())
exclusions := []excludes.Exclusion{
excludes.AssetExclusion(),
excludes.PrefixExclusion(config.Admin.Prefix),
excludes.PrefixExclusion(config.Session.Prefix),
}
public := mux.NewRouter()
public.Use(
func(next http.Handler) http.Handler { return headers.HTTP(next) },
Middleware(
emptyGeoIP{},
AssetMatcher(),
PrefixMatcher(config.Admin.Prefix),
geoip.Middleware(geoip.Empty{}),
pageviews.Middleware(
pageviews.Exclusions(exclusions...),
),
session.Middleware(
session.Exclusions(exclusions...),
session.JavaScriptPath(path.Join(config.Session.Prefix, "pages.js")),
),
)
admin := public.Path(config.Admin.Prefix).Subrouter()
admin := public.PathPrefix(config.Admin.Prefix).Subrouter()
if config.Admin.Password != "" {
authenticate := basicauth.Static(config.Admin.Username, config.Admin.Password)
@ -84,6 +100,15 @@ func NewServer(ctx context.Context, config ServerConfig) (*Server, error) {
})
}
{
var handler http.Handler = session.Handler()
handler = http.StripPrefix(config.Session.Prefix, handler)
session := public.PathPrefix(config.Session.Prefix).Subrouter()
session.HandleFunc("/pages.js", web.Handler()).Methods(http.MethodGet)
session.Handle("/", handler)
}
return &Server{
AdminMux: admin,

22
internal/session/config.go Normal file
View File

@ -0,0 +1,22 @@
// Copyright (C) 2022 The pages authors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package session
// Config encapsulates configuration for the session endpoints.
type Config struct {
Prefix string `json:"prefix" usage:"configure the prefix to use for recording sessions" default:"/_session" hidden:"true"`
}

73
internal/session/handler.go Normal file
View File

@ -0,0 +1,73 @@
// Copyright (C) 2022 The pages authors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package session
import (
"net/http"
"time"
"github.com/gorilla/websocket"
"github.com/mjpitz/pages/internal/geoip"
"github.com/mjpitz/pages/internal/metrics"
)
type Request struct {
ID string
FullName string
}
func Handler() *Handle {
return &Handle{
upgrader: websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
},
}
}
type Handle struct {
upgrader websocket.Upgrader
}
func (h *Handle) ServeHTTP(w http.ResponseWriter, r *http.Request) {
conn, err := h.upgrader.Upgrade(w, r, nil)
defer func() { _ = conn.Close() }()
if err != nil {
// log
return
}
path := r.URL.Path
geoInfo := geoip.Extract(r.Context())
start := time.Now()
defer func() {
metrics.PageSessionDuration.WithLabelValues(path, geoInfo.CountryCode).Observe(time.Since(start).Seconds())
}()
for {
req := Request{}
err = conn.ReadJSON(&req)
if err != nil {
// log
return
}
}
}

124
internal/session/middleware.go Normal file
View File

@ -0,0 +1,124 @@
// Copyright (C) 2022 The pages authors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package session
import (
"bytes"
"fmt"
"net/http"
"strconv"
"strings"
"github.com/gorilla/mux"
"github.com/mjpitz/pages/internal/excludes"
)
func script(path string) string {
return fmt.Sprintf(`<script async type='text/javascript' src='%s'></script>`, path)
}
type opt struct {
jsPath string
excludes []excludes.Exclusion
}
// Option provides a way to configure elements of the Middleware.
type Option func(*opt)
// JavaScriptPath configures the URL for the JS snippet.
func JavaScriptPath(path string) Option {
return func(o *opt) {
o.jsPath = path
}
}
// Exclusions appends the provided rules to the excludes list. Any path that matches an exclusion will not be measured.
func Exclusions(exclusions ...excludes.Exclusion) Option {
return func(o *opt) {
o.excludes = append(o.excludes, exclusions...)
}
}
// Middleware injects a JavaScript snippet that establishes a session with the server.
func Middleware(opts ...Option) mux.MiddlewareFunc {
o := opt{}
for _, opt := range opts {
opt(&o)
}
exclude := excludes.AnyExclusion(o.excludes...)
injection := script(o.jsPath)
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if exclude(r.URL.Path) {
next.ServeHTTP(w, r)
return
}
buffer := &bufferedResponseWriter{
header: http.Header{},
status: http.StatusOK,
buffer: bytes.NewBuffer(nil),
}
next.ServeHTTP(buffer, r)
for key := range buffer.header {
w.Header().Set(key, buffer.header.Get(key))
}
contents := buffer.buffer.Bytes()
if strings.Contains(buffer.header.Get("Content-Type"), "text/html") {
contents = bytes.TrimSpace(contents)
contents = bytes.TrimSuffix(contents, []byte("</html>"))
contents = bytes.TrimSpace(contents)
contents = bytes.TrimSuffix(contents, []byte("</body>"))
contents = append(contents, []byte(injection)...)
contents = append(contents, []byte("</body></html>")...)
}
w.Header().Set("Content-Length", strconv.Itoa(len(contents)))
w.WriteHeader(buffer.status)
_, _ = w.Write(contents)
})
}
}
type bufferedResponseWriter struct {
header http.Header
status int
buffer *bytes.Buffer
}
func (w *bufferedResponseWriter) Header() http.Header {
return w.header
}
func (w *bufferedResponseWriter) WriteHeader(statusCode int) {
w.status = statusCode
}
func (w *bufferedResponseWriter) Write(data []byte) (n int, err error) {
return w.buffer.Write(data)
}
var _ http.ResponseWriter = &bufferedResponseWriter{}

2
internal/web/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
dist/
node_modules/

1041
internal/web/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

15
internal/web/package.json Normal file
View File

@ -0,0 +1,15 @@
{
"name": "@mjpitz/pages-web",
"version": "0.0.0",
"private": "true",
"scripts": {
"build": "webpack build"
},
"devDependencies": {
"webpack": "^5.73.0",
"webpack-cli": "^4.9.2"
},
"dependencies": {
"ulid": "^2.3.0"
}
}

33
internal/web/src/index.js Normal file
View File

@ -0,0 +1,33 @@
/**
* Copyright (C) 2022 The pages authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
import {ulid} from 'ulid';
const sessionPrefix = "/_session";
const domain = window.location.host;
async function main() {
const ws = new WebSocket(`ws://${domain}${sessionPrefix}${window.location.pathname}`);
const message = `{"ID":"${ulid()}","FullName":"ping"}`
let id = null;
ws.onopen = () => id = setInterval(() => ws.send(message), 5000);
ws.onclose = () => clearInterval(id);
}
main().catch(err => console.error(err))

38
internal/web/web.go Normal file
View File

@ -0,0 +1,38 @@
// Copyright (C) 2022 The pages authors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package web
import (
_ "embed"
"net/http"
"strings"
"time"
)
//go:generate npm install
//go:generate npm run build
//go:embed dist/pages.js
var pagesjs string
func Handler() http.HandlerFunc {
start := time.Now()
return func(w http.ResponseWriter, r *http.Request) {
http.ServeContent(w, r, "pages.js", start, strings.NewReader(pagesjs))
}
}

28
internal/web/webpack.config.js Normal file
View File

@ -0,0 +1,28 @@
/**
* Copyright (C) 2022 The pages authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
const path = require('path');
module.exports = {
entry: './src/index.js',
output: {
path: path.resolve(__dirname, 'dist'),
filename: 'pages.js',
},
mode: process.env.PAGES_ENV || 'production',
}

2
scripts/dist-go.sh
View File

@ -20,6 +20,8 @@ set -e -o pipefail
go mod download
go mod verify
go generate ./...
if [[ -z "${VERSION}" ]]; then
goreleaser --snapshot --skip-publish --rm-dist
else