first commit

This commit is contained in:
libroot 2026-11-19 04:40:59 +02:00
commit 8c4dcae475
Signed by: libroot
GPG key ID: 2AC68C029EBE9ACE
89 changed files with 24623 additions and 0 deletions

312
tests/bench_test.go Normal file
View file

@ -0,0 +1,312 @@
package main
import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"runtime"
"sort"
"strconv"
"sync"
"testing"
"time"
)
// Page benchmarks against a REAL corpus - the fixture is four documents, so
// it cannot show what 3,000 do. Point $SNOWCHIVE_BENCH_ROOT at an archive
// root (the directory holding data/ and project/) and run:
// tests/run.sh -run XXX -bench . -benchtime 40x
//
// What they are for: every page but /search is built from data indexed at
// load, so a request should cost what its OUTPUT costs (template execution)
// and nothing more. A page whose time or allocations grow with the CORPUS
// rather than with the page has picked up a scan.
var (
benchOnce sync.Once
benchMux http.Handler
)
func benchCorpus(tb testing.TB) http.Handler {
root := os.Getenv("SNOWCHIVE_BENCH_ROOT")
if root == "" {
tb.Skip("set SNOWCHIVE_BENCH_ROOT to an archive root to run the page benchmarks")
}
benchOnce.Do(func() {
setRoot(root)
devMode = false
apiEnabled = true
initAssets()
if err := parseTemplates(); err != nil {
tb.Fatal(err)
}
loadData()
benchMux = withBaseHeaders(newMux())
})
return benchMux
}
func benchPage(b *testing.B, path string) {
h := benchCorpus(b)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
r := httptest.NewRequest("GET", path, nil)
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
if w.Code != 200 {
b.Fatalf("%s: %d", path, w.Code)
}
}
}
func BenchmarkPageHome(b *testing.B) { benchPage(b, "/") }
func BenchmarkPageBrowse(b *testing.B) { benchPage(b, "/browse") }
func BenchmarkPageArchiveTab(b *testing.B) { benchPage(b, "/browse?collection=sidtoday-archive") }
func BenchmarkPageCategoryList(b *testing.B) { benchPage(b, "/browse/program") }
func BenchmarkPageCategoryValue(b *testing.B) { benchPage(b, "/browse/agency/NSA") }
func BenchmarkPageDoc(b *testing.B) {
h := benchCorpus(b)
if len(corpus().archive.Docs) == 0 {
b.Skip("empty corpus")
}
benchPageWith(b, h, "/doc/"+corpus().archive.Docs[0].ID)
}
func BenchmarkPageExploreIDs(b *testing.B) { benchPage(b, "/explore/ids") }
func BenchmarkPageGlossary(b *testing.B) { benchPage(b, "/glossary") }
func BenchmarkPageSitemap(b *testing.B) { benchPage(b, "/sitemap.xml") }
func BenchmarkPageFacetsAPI(b *testing.B) { benchPage(b, "/api/facets") }
func BenchmarkPageSearch(b *testing.B) { benchPage(b, "/search?q=nsa&do_search=1") }
func benchPageWith(b *testing.B, h http.Handler, path string) {
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
r := httptest.NewRequest("GET", path, nil)
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
if w.Code != 200 {
b.Fatalf("%s: %d", path, w.Code)
}
}
}
func BenchmarkPageCategoryTag(b *testing.B) { benchPage(b, "/browse/tag") }
func BenchmarkPageCategoryPeople(b *testing.B) { benchPage(b, "/browse/people") }
func BenchmarkPageCategoryTagSorted(b *testing.B) { benchPage(b, "/browse/tag?sort=count") }
func BenchmarkPageExploreMetric(b *testing.B) { benchPage(b, "/explore/pages") }
func BenchmarkPageCodewords(b *testing.B) { benchPage(b, "/explore/terms") }
func BenchmarkPageOutlets(b *testing.B) { benchPage(b, "/explore/outlets") }
func BenchmarkPageClassPatterns(b *testing.B) { benchPage(b, "/explore/classification") }
func BenchmarkPageArticles(b *testing.B) { benchPage(b, "/articles") }
func BenchmarkPageChangelog(b *testing.B) { benchPage(b, "/changelog") }
func BenchmarkPageAbout(b *testing.B) { benchPage(b, "/about") }
func BenchmarkPageAPIDocs(b *testing.B) { benchPage(b, "/api/docs") }
func BenchmarkPageExploreIndex(b *testing.B) { benchPage(b, "/explore") }
func BenchmarkPageContactSheet(b *testing.B) {
h := benchCorpus(b)
id := ""
for i := range corpus().archive.Docs {
if len(corpus().archive.Docs[i].Images) > 40 {
id = corpus().archive.Docs[i].ID
break
}
}
if id == "" {
b.Skip("no document with many pages")
}
benchPageWith(b, h, "/doc/"+id+"/pages")
}
// Pathological documents: the biggest thing in the corpus is what breaks
// first. What IS the biggest thing?
func TestProbeExtremes(t *testing.T) {
if os.Getenv("SNOWCHIVE_BENCH_ROOT") == "" {
t.Skip("needs a real corpus")
}
benchCorpus(t)
var maxImgs, maxText, maxPubs, maxVersions *Document
for i := range corpus().archive.Docs {
d := &corpus().archive.Docs[i]
if maxImgs == nil || len(d.Images) > len(maxImgs.Images) {
maxImgs = d
}
if maxText == nil || len(d.FullText) > len(maxText.FullText) {
maxText = d
}
if maxPubs == nil || len(d.Publication) > len(maxPubs.Publication) {
maxPubs = d
}
if maxVersions == nil || len(d.Versions) > len(maxVersions.Versions) {
maxVersions = d
}
}
h := benchCorpus(t)
timeIt := func(p string) time.Duration {
start := time.Now()
r := httptest.NewRequest("GET", p, nil)
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
return time.Since(start)
}
fmt.Printf("most pages: %d %s doc %s, contact sheet %s\n", len(maxImgs.Images), trunc30(maxImgs.ID),
timeIt("/doc/"+maxImgs.ID).Round(time.Millisecond), timeIt("/doc/"+maxImgs.ID+"/pages").Round(time.Millisecond))
fmt.Printf("most text: %d KB %s doc page %s\n", len(maxText.FullText)/1024, trunc30(maxText.ID),
timeIt("/doc/"+maxText.ID).Round(time.Millisecond))
fmt.Printf("most pubs: %d %s\n", len(maxPubs.Publication), trunc30(maxPubs.ID))
fmt.Printf("most copies: %d %s\n", len(maxVersions.Versions), trunc30(maxVersions.ID))
}
// Distribution, not the median: which document pages are slow, and why?
func TestProbeDocPageDistribution(t *testing.T) {
if os.Getenv("SNOWCHIVE_BENCH_ROOT") == "" {
t.Skip("needs a real corpus")
}
h := benchCorpus(t)
totalImgs := 0
for i := range corpus().archive.Docs {
totalImgs += len(corpus().archive.Docs[i].Images)
}
var ds []time.Duration
for i := 0; i < len(corpus().archive.Docs); i += 7 { // a sample, every 7th
d := &corpus().archive.Docs[i]
start := time.Now()
r := httptest.NewRequest("GET", "/doc/"+d.ID, nil)
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
ds = append(ds, time.Since(start))
}
sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] })
p := func(q float64) time.Duration { return ds[int(float64(len(ds)-1)*q)] }
fmt.Printf("%d page images in the corpus (%d docs sampled)\n", totalImgs, len(ds))
fmt.Printf("doc page: p50 %s p90 %s p99 %s max %s\n",
p(0.5).Round(time.Microsecond), p(0.9).Round(time.Microsecond),
p(0.99).Round(time.Millisecond), ds[len(ds)-1].Round(time.Millisecond))
}
func TestProbeWorstDocSplit(t *testing.T) {
if os.Getenv("SNOWCHIVE_BENCH_ROOT") == "" {
t.Skip("needs a real corpus")
}
h := benchCorpus(t)
var worst *Document
for i := range corpus().archive.Docs {
if worst == nil || len(corpus().archive.Docs[i].Images) > len(worst.Images) {
worst = &corpus().archive.Docs[i]
}
}
run := func(path string, cookies ...*http.Cookie) (time.Duration, int) {
var best time.Duration
var size int
for i := 0; i < 5; i++ {
r := httptest.NewRequest("GET", path, nil)
for _, c := range cookies {
r.AddCookie(c)
}
w := httptest.NewRecorder()
start := time.Now()
h.ServeHTTP(w, r)
d := time.Since(start)
if i == 0 || d < best {
best = d
}
size = w.Body.Len()
}
return best, size
}
full, fullSize := run("/doc/" + worst.ID)
textOnly, textSize := run("/doc/"+worst.ID, &http.Cookie{Name: "pref_text_only", Value: "1"})
sheet, sheetSize := run("/doc/" + worst.ID + "/pages")
fmt.Printf("worst doc: %d images, %d KB transcript\n", len(worst.Images), len(worst.FullText)/1024)
fmt.Printf(" full page %6s %d KB\n", full.Round(time.Microsecond), fullSize/1024)
fmt.Printf(" text-only %6s %d KB (no image grid, no lightbox)\n", textOnly.Round(time.Microsecond), textSize/1024)
fmt.Printf(" contact sheet %6s %d KB\n", sheet.Round(time.Microsecond), sheetSize/1024)
}
func trunc30(s string) string {
if len(s) > 30 {
return s[:30]
}
return s
}
// TestSoak hammers the server for a while and watches its memory. The claim
// it checks: nothing accumulates between requests - no cache, no growing
// map, no goroutine left behind. Run it against a real corpus:
//
// SNOWCHIVE_BENCH_ROOT=/path/to/archive tests/run.sh -run TestSoak -timeout 30m
//
// SNOWCHIVE_SOAK_REQUESTS sets the count (default 20,000).
func TestSoak(t *testing.T) {
h := benchCorpus(t)
n := 20000
if v := os.Getenv("SNOWCHIVE_SOAK_REQUESTS"); v != "" {
if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 {
n = parsed
}
}
paths := []string{"/", "/browse", "/browse?collection=foia", "/browse/program",
"/browse/tag?sort=count", "/browse/agency/NSA", "/search?q=nsa&do_search=1",
"/search?q=%22upstream+collection%22&do_search=1", "/explore/ids", "/glossary",
"/articles", "/api/facets", "/api/search?q=prism", "/sitemap.xml"}
cookies := []*http.Cookie{
{Name: "pref_theme", Value: "dark"},
{Name: "pref_per_page", Value: "100"},
{Name: "pref_text_only", Value: "1"},
}
read := func() (heap, sys uint64, goroutines int) {
runtime.GC()
var m runtime.MemStats
runtime.ReadMemStats(&m)
return m.HeapAlloc, m.Sys, runtime.NumGoroutine()
}
// Warm up first: the pre-rendered bodies and the first template
// executions allocate once and stay.
for i := 0; i < 200; i++ {
r := httptest.NewRequest("GET", paths[i%len(paths)], nil)
h.ServeHTTP(httptest.NewRecorder(), r)
}
heap0, sys0, gor0 := read()
start := time.Now()
var wg sync.WaitGroup
const workers = 8
for w := 0; w < workers; w++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
for i := id; i < n; i += workers {
r := httptest.NewRequest("GET", paths[i%len(paths)], nil)
if i%4 == 0 {
r.AddCookie(cookies[i%len(cookies)])
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, r)
if rec.Code != 200 {
t.Errorf("%s: %d", paths[i%len(paths)], rec.Code)
return
}
}
}(w)
}
wg.Wait()
elapsed := time.Since(start)
heap1, sys1, gor1 := read()
mb := func(b uint64) float64 { return float64(b) / (1 << 20) }
fmt.Printf("%d requests in %s (%.0f req/s, %d workers)\n",
n, elapsed.Round(time.Millisecond), float64(n)/elapsed.Seconds(), workers)
fmt.Printf("heap %.1f MB -> %.1f MB (%+.1f)\n", mb(heap0), mb(heap1), mb(heap1)-mb(heap0))
fmt.Printf("sys %.1f MB -> %.1f MB (%+.1f)\n", mb(sys0), mb(sys1), mb(sys1)-mb(sys0))
fmt.Printf("goroutines %d -> %d\n", gor0, gor1)
if gor1 > gor0+2 {
t.Errorf("goroutines grew from %d to %d", gor0, gor1)
}
// The heap after a GC should be the corpus and the pre-rendered pages,
// not a function of how many requests have been served.
if grown := mb(heap1) - mb(heap0); grown > 32 {
t.Errorf("heap grew by %.1f MB over %d requests - something is accumulating", grown, n)
}
}