773 lines
24 KiB
Go
773 lines
24 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"embed"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"html/template"
|
|
"io/fs"
|
|
"log"
|
|
"maps"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"os/signal"
|
|
"path"
|
|
"path/filepath"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
// devMode is set by the -dev flag. In dev mode templates and static files
|
|
// are read from ./templates and ./static on every request (so edits show
|
|
// without a rebuild), caching is disabled and error details are shown in
|
|
// responses. Never run production with -dev: the error detail leaks
|
|
// filesystem paths and template internals.
|
|
var devMode bool
|
|
|
|
// The templates and static assets are compiled into the binary, so a
|
|
// deployment is one executable plus the data and project directories - nothing depends
|
|
// on the working directory the process happens to start in. -dev swaps in
|
|
// the on-disk copies (see initAssets).
|
|
//
|
|
//go:embed templates static
|
|
var embeddedAssets embed.FS
|
|
|
|
var (
|
|
templateFS fs.FS // templates/*.html
|
|
staticFS fs.FS // static/*
|
|
)
|
|
|
|
// staticHashes holds a content hash per embedded asset, computed once at
|
|
// startup. It is both the assets' ETag (embed.FS files have a zero modtime,
|
|
// so net/http sends no Last-Modified and an unvalidated asset would be
|
|
// re-downloaded on every view) and their version stamp: base.html links
|
|
// them as ?v=<hash> (the asset func), and handleStatic caches exactly that
|
|
// URL forever. A changed file gets a new hash and so a new URL, whatever
|
|
// the build was stamped with.
|
|
var staticHashes map[string]string
|
|
|
|
func buildStaticHashes() {
|
|
staticHashes = map[string]string{}
|
|
err := fs.WalkDir(staticFS, ".", func(p string, d fs.DirEntry, err error) error {
|
|
if err != nil || d.IsDir() {
|
|
return err
|
|
}
|
|
b, err := fs.ReadFile(staticFS, p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sum := sha256.Sum256(b)
|
|
staticHashes[p] = hex.EncodeToString(sum[:8])
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
log.Fatalf("embedded static: %v", err)
|
|
}
|
|
}
|
|
|
|
func initAssets() {
|
|
if devMode {
|
|
templateFS = os.DirFS("templates")
|
|
staticFS = os.DirFS("static")
|
|
staticHashes = nil
|
|
return
|
|
}
|
|
var err error
|
|
if templateFS, err = fs.Sub(embeddedAssets, "templates"); err != nil {
|
|
log.Fatalf("embedded templates: %v", err)
|
|
}
|
|
if staticFS, err = fs.Sub(embeddedAssets, "static"); err != nil {
|
|
log.Fatalf("embedded static: %v", err)
|
|
}
|
|
buildStaticHashes()
|
|
}
|
|
|
|
type sortOption struct{ Key, Label string }
|
|
|
|
var sortOptions = func() []sortOption {
|
|
opts := []sortOption{{"relevance", "Relevance"}}
|
|
for _, s := range docSorts {
|
|
if s.InBar {
|
|
opts = append(opts, sortOption{s.Key, s.Label})
|
|
}
|
|
}
|
|
return opts
|
|
}()
|
|
|
|
func sortLabel(key string) string {
|
|
switch key {
|
|
case "relevance":
|
|
return "Relevance"
|
|
case "":
|
|
return "Title"
|
|
}
|
|
if s := docSortByKey[key]; s != nil {
|
|
return s.Label
|
|
}
|
|
return strings.ToUpper(key[:1]) + key[1:]
|
|
}
|
|
|
|
var (
|
|
// tmplSet holds one parsed template set PER PAGE (base + partials +
|
|
// page).
|
|
tmplSet map[string]*template.Template
|
|
|
|
funcMap = template.FuncMap{
|
|
"buildInfo": func() map[string]string {
|
|
return map[string]string{"Commit": buildCommit}
|
|
},
|
|
// articleKey is the key of the article page for a publication's
|
|
// link. Every linked publication of a loaded document has one
|
|
// (buildArticles), so the key needs no lookup - and no corpus.
|
|
"articleKey": func(link string) string {
|
|
if link == "" {
|
|
return ""
|
|
}
|
|
return articleKeyFor(link)
|
|
},
|
|
"asset": func(name string) string {
|
|
if h, ok := staticHashes[name]; ok {
|
|
return "/static/" + name + "?v=" + h
|
|
}
|
|
return "/static/" + name
|
|
},
|
|
// wholeArchiveQS is the scope that names every collection, for the
|
|
// links of a FOIA release on a page rendered once for all readers.
|
|
"wholeArchiveQS": func() string { return wholeArchive.suffix() },
|
|
"facetLinks": facetLinks,
|
|
"add": func(a, b int) int { return a + b },
|
|
"eqFold": sameValue,
|
|
"sortOptions": func() []sortOption { return sortOptions },
|
|
"sortLabel": sortLabel,
|
|
"agencyName": agencyName,
|
|
"dict": func(values ...interface{}) (map[string]interface{}, error) {
|
|
if len(values)%2 != 0 {
|
|
return nil, fmt.Errorf("dict: odd number of arguments (%d)", len(values))
|
|
}
|
|
dict := make(map[string]interface{}, len(values)/2)
|
|
for i := 0; i < len(values); i += 2 {
|
|
key, ok := values[i].(string)
|
|
if !ok {
|
|
return nil, fmt.Errorf("dict: argument %d is not a string key (%T)", i, values[i])
|
|
}
|
|
dict[key] = values[i+1]
|
|
}
|
|
return dict, nil
|
|
},
|
|
// pathEscape percent-encodes a value for use as a single URL path
|
|
// segment (so values like "TOP SECRET//SI" don't split the path).
|
|
"pathEscape": func(s string) template.URL {
|
|
return template.URL(url.PathEscape(s))
|
|
},
|
|
"azID": azID,
|
|
"pages": func(current, total int) []int {
|
|
var out []int
|
|
last := 0
|
|
for i := 1; i <= total; i++ {
|
|
if i == 1 || i == total || (i >= current-2 && i <= current+2) {
|
|
if last != 0 && i-last > 1 {
|
|
out = append(out, 0)
|
|
}
|
|
out = append(out, i)
|
|
last = i
|
|
}
|
|
}
|
|
return out
|
|
},
|
|
}
|
|
)
|
|
|
|
// Build identity, stamped at build time so mirrors are comparable at a
|
|
// glance. Shown in the footer next to the data load time:
|
|
//
|
|
// go build -ldflags "-X main.buildCommit=$(git rev-parse --short HEAD)"
|
|
var buildCommit = "dev"
|
|
|
|
// accelPrefix is set by -accel-files: the internal nginx location that maps
|
|
// to the data directory. With it, handleFiles still decides whether a file is
|
|
// served, but answers with an X-Accel-Redirect and nginx sends the bytes
|
|
// (sendfile, ranges, slow readers) without them passing through this process.
|
|
// Only set it behind nginx: any other client gets an empty 200.
|
|
var accelPrefix string
|
|
|
|
// accelExpires is set by -accel-expires: how long the proxy may keep a page
|
|
// this server marks public, sent as X-Accel-Expires (which nginx obeys ahead
|
|
// of Cache-Control and strips from the response). Browsers still get the
|
|
// short max-age. The corpus is read once at startup, so such a page cannot
|
|
// change before the next restart; clear the proxy's cache after one.
|
|
var accelExpires string
|
|
|
|
// apiEnabled is cleared by -no-api: public instances that don't want bulk
|
|
// scraping can turn the JSON API off.
|
|
var apiEnabled = true
|
|
|
|
func main() {
|
|
rebuildRelated := flag.Bool("rebuild-related", false, "Rebuild related docs index and exit")
|
|
flag.BoolVar(&devMode, "dev", false, "Dev mode: serve templates/ and static/ from the working directory and re-parse templates per request, no immutable static cache, verbose errors")
|
|
baseFlag := flag.String("base", os.Getenv("SNOWCHIVE_BASE"),
|
|
"Canonical base URL (e.g. https://archive.example.org) for the absolute URLs in robots.txt and the sitemap. "+
|
|
"Without it those URLs are built from each request's Host header, which is fine only behind a proxy that pins Host. "+
|
|
"Set this on anything internet-facing. Defaults to $SNOWCHIVE_BASE.")
|
|
listen := flag.String("listen", envOr("SNOWCHIVE_LISTEN", "127.0.0.1:8080"),
|
|
"TCP address to listen on. Defaults to $SNOWCHIVE_LISTEN, then 127.0.0.1:8080.")
|
|
root := flag.String("root", envOr("SNOWCHIVE_ROOT", "."),
|
|
"Archive directory holding data/ (the document corpus, one directory per document) and project/ "+
|
|
"(the files about the archive as a whole: glossary.json, assemblies.json, changelog.json and related.json; only -rebuild-related writes there). "+
|
|
"Defaults to $SNOWCHIVE_ROOT, then the working directory.")
|
|
check := flag.Bool("check", false,
|
|
"Load the corpus, print the warnings and the tally, then exit - non-zero when anything was reported. ")
|
|
noAPI := flag.Bool("no-api", envBool("SNOWCHIVE_NO_API"),
|
|
"Disable the read-only JSON API: every /api/ path answers a JSON 403 and the UI stops linking to it. Defaults on when $SNOWCHIVE_NO_API is true (1, t, true).")
|
|
flag.StringVar(&accelPrefix, "accel-files", os.Getenv("SNOWCHIVE_ACCEL_FILES"),
|
|
"Internal nginx location that aliases the data directory (e.g. /_files/). Document files are then answered with an X-Accel-Redirect into it and sent by nginx. "+
|
|
"Only behind nginx. Defaults to $SNOWCHIVE_ACCEL_FILES.")
|
|
accelTTL := flag.Duration("accel-expires", envDuration("SNOWCHIVE_ACCEL_EXPIRES"),
|
|
"How long the proxy may cache a page served to a reader without cookies (e.g. 10m), sent as X-Accel-Expires; browsers keep the short max-age. "+
|
|
"Clear the proxy cache after a restart with new data. Defaults to $SNOWCHIVE_ACCEL_EXPIRES.")
|
|
flag.Parse()
|
|
setRoot(*root)
|
|
if accelPrefix != "" {
|
|
if !strings.HasPrefix(accelPrefix, "/") || !strings.HasSuffix(accelPrefix, "/") {
|
|
log.Fatalf("-accel-files: %q must begin and end with a slash", accelPrefix)
|
|
}
|
|
}
|
|
if *accelTTL >= time.Second {
|
|
accelExpires = strconv.Itoa(int(*accelTTL / time.Second))
|
|
}
|
|
if *baseFlag != "" {
|
|
b, err := parseCanonicalBase(*baseFlag)
|
|
if err != nil {
|
|
log.Fatalf("-base: %v", err)
|
|
}
|
|
canonicalBase = b
|
|
}
|
|
apiEnabled = !*noAPI
|
|
initAssets()
|
|
// Before the load: it pre-renders the big page bodies from these.
|
|
if !devMode {
|
|
if err := parseTemplates(); err != nil {
|
|
log.Fatalf("Fatal: Could not parse templates at startup: %v", err)
|
|
}
|
|
}
|
|
|
|
loadData()
|
|
co := corpus()
|
|
|
|
if *check {
|
|
if loadErrors > 0 || loadWarnings > 0 {
|
|
log.Printf("check failed: %d errors, %d warnings", loadErrors, loadWarnings)
|
|
os.Exit(1)
|
|
}
|
|
log.Printf("check passed: %d documents, nothing to report", len(co.archive.Docs))
|
|
return
|
|
}
|
|
|
|
if *rebuildRelated {
|
|
log.Println("Rebuilding related docs index...")
|
|
|
|
existing := make(map[string][]string)
|
|
if b, err := os.ReadFile(relatedDocsPath()); err == nil {
|
|
if err := json.Unmarshal(b, &existing); err != nil {
|
|
warnf("could not parse existing related.json: %v - starting fresh", err)
|
|
existing = make(map[string][]string)
|
|
} else {
|
|
log.Printf("Loaded existing related.json with %d entries", len(existing))
|
|
}
|
|
}
|
|
|
|
stale := 0
|
|
for id := range existing {
|
|
if co.unknownDoc(id) {
|
|
delete(existing, id)
|
|
stale++
|
|
}
|
|
}
|
|
var missing []Document
|
|
for _, d := range co.archive.Docs {
|
|
if ids, ok := existing[d.ID]; !ok || slices.ContainsFunc(ids, co.unknownDoc) {
|
|
missing = append(missing, d)
|
|
}
|
|
}
|
|
|
|
if len(missing) == 0 && stale == 0 {
|
|
log.Println("No new documents - related.json is up to date.")
|
|
return
|
|
}
|
|
|
|
log.Printf("Computing related docs for %d documents...", len(missing))
|
|
maps.Copy(existing, co.buildRelatedDocs(missing, co.archive.Docs))
|
|
|
|
if err := saveRelatedDocs(existing); err != nil {
|
|
log.Fatalf("Failed to save related docs: %v", err)
|
|
}
|
|
log.Printf("Saved related.json with %d entries (%d computed, %d stale dropped, other entries and manual edits preserved)",
|
|
len(existing), len(missing), stale)
|
|
return
|
|
}
|
|
|
|
handler := withBaseHeaders(newMux())
|
|
|
|
srv := &http.Server{
|
|
Addr: *listen,
|
|
Handler: handler,
|
|
ReadHeaderTimeout: 5 * time.Second,
|
|
ReadTimeout: 15 * time.Second,
|
|
WriteTimeout: writeTimeout, // downloads set their own: fileWriteBudget
|
|
IdleTimeout: 60 * time.Second,
|
|
MaxHeaderBytes: 64 << 10,
|
|
}
|
|
|
|
warnNonLoopback(*listen)
|
|
log.Printf("Archive Server listening on %s (data: %s, project: %s)", srv.Addr, dataDir, projectDir)
|
|
|
|
errCh := make(chan error, 1)
|
|
go func() { errCh <- srv.ListenAndServe() }()
|
|
sig := make(chan os.Signal, 1)
|
|
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
|
|
select {
|
|
case err := <-errCh:
|
|
log.Fatal(err)
|
|
case s := <-sig:
|
|
signal.Reset(os.Interrupt, syscall.SIGTERM)
|
|
log.Printf("%s received - draining connections (again to stop now)", s)
|
|
ctx, cancel := context.WithTimeout(context.Background(), writeTimeout)
|
|
defer cancel()
|
|
if err := srv.Shutdown(ctx); err != nil {
|
|
log.Printf("Shutdown: %v", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (co *Corpus) prerenderPages() {
|
|
co.prerendered = map[string]template.HTML{}
|
|
co.docRowHTML = nil
|
|
if devMode {
|
|
return
|
|
}
|
|
exec := func(key, page, block string, data interface{}) {
|
|
t := tmplSet[page]
|
|
if t == nil {
|
|
return
|
|
}
|
|
var buf bytes.Buffer
|
|
if err := t.ExecuteTemplate(&buf, block, data); err != nil {
|
|
warnf("prerender %s/%s: %v - rendering it per request", page, block, err)
|
|
return
|
|
}
|
|
co.prerendered[key] = template.HTML(buf.String())
|
|
}
|
|
|
|
exec("ids", "ids.html", "id-rows", &idsPage{Rows: co.idRows})
|
|
exec("archive-timeline", "browse.html", "archive-timeline",
|
|
&browsePage{ArchiveYears: co.archiveTimeline})
|
|
exec("glossary", "glossary.html", "glossary-groups",
|
|
&glossaryPage{Groups: co.glossaryGroups})
|
|
exec("articles", "articles.html", "article-groups",
|
|
&articlesPage{Groups: co.articleGroups})
|
|
for key, v := range co.codewordViews {
|
|
exec("codewords:"+key, "codewords.html", "codeword-rows",
|
|
&codewordsPage{Rows: v.Rows})
|
|
}
|
|
for set, tables := range co.exploreTables {
|
|
for i, t := range tables {
|
|
exec(exploreKey(set, i), "explore_table.html", "explore-rows",
|
|
&exploreTablePage{Rows: t.Rows})
|
|
}
|
|
}
|
|
co.prerenderCategoryRows()
|
|
co.prerenderDocRows()
|
|
total := 0
|
|
for _, h := range co.prerendered {
|
|
total += len(h)
|
|
}
|
|
log.Printf("Pre-rendered %d page bodies (%d KB)", len(co.prerendered), total/1024)
|
|
}
|
|
|
|
func (co *Corpus) preHTML(key string) template.HTML { return co.prerendered[key] }
|
|
|
|
func newMux() *http.ServeMux {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("GET /{$}", bind((*Corpus).handleIndex))
|
|
mux.HandleFunc("GET /random", bind((*Corpus).handleRandom))
|
|
mux.HandleFunc("GET /glossary", bind((*Corpus).handleGlossary))
|
|
mux.HandleFunc("GET /preferences", handlePreferences)
|
|
mux.HandleFunc("POST /preferences/save", handlePreferencesSave)
|
|
mux.HandleFunc("GET /pinned", bind((*Corpus).handlePinnedPage))
|
|
mux.HandleFunc("GET /pinned/export", bind((*Corpus).handlePinnedExport))
|
|
mux.HandleFunc("POST /pin/{id}", bind((*Corpus).handlePinToggle))
|
|
mux.HandleFunc("GET /articles", bind((*Corpus).handleArticles))
|
|
mux.HandleFunc("GET /article/{key}", bind((*Corpus).handleArticleView))
|
|
mux.HandleFunc("GET /changelog", bind((*Corpus).handleChangelog))
|
|
mux.HandleFunc("POST /history/clear", handleHistoryClear)
|
|
mux.HandleFunc("POST /pinsearch", bind((*Corpus).handleSearchPinToggle))
|
|
mux.HandleFunc("GET /robots.txt", handleRobots)
|
|
mux.HandleFunc("GET /sitemap.xml", bind((*Corpus).handleSitemap))
|
|
mux.HandleFunc("/", notFound)
|
|
mux.HandleFunc("GET /browse", bind((*Corpus).handleBrowseRoot))
|
|
mux.HandleFunc("GET /browse/{category}", bind((*Corpus).handleCategoryList))
|
|
mux.HandleFunc("GET /explore", bind((*Corpus).handleExploreIndex))
|
|
mux.HandleFunc("GET /explore/{metric}", bind((*Corpus).handleExploreMetric))
|
|
mux.HandleFunc("GET /explore/terms", bind((*Corpus).handleCodewords))
|
|
mux.HandleFunc("GET /explore/ids", bind((*Corpus).handleExploreIDs))
|
|
mux.HandleFunc("GET /explore/ids.txt", bind((*Corpus).handleExploreIDsText))
|
|
mux.HandleFunc("GET /explore/classification", bind((*Corpus).handleClassPatterns))
|
|
mux.HandleFunc("GET /explore/portions", bind((*Corpus).handlePortions))
|
|
mux.HandleFunc("GET /explore/outlets", bind((*Corpus).handleOutlets))
|
|
mux.HandleFunc("GET /explore/assemblies", bind((*Corpus).handleAssemblies))
|
|
mux.HandleFunc("GET /explore/assemblies/{id}", bind((*Corpus).handleAssemblyView))
|
|
mux.HandleFunc("GET /browse/{category}/{value}", bind((*Corpus).handleCategoryFilter))
|
|
mux.HandleFunc("GET /doc/{id}", bind((*Corpus).handleDocView))
|
|
mux.HandleFunc("GET /doc/{id}/pages", bind((*Corpus).handleDocPages))
|
|
mux.HandleFunc("GET /doc/{id}/versions/{slug}", bind((*Corpus).handleVersionView))
|
|
mux.HandleFunc("GET /doc/{id}/versions/{slug}/pages", bind((*Corpus).handleVersionPages))
|
|
mux.HandleFunc("GET /search", bind((*Corpus).handleSearch))
|
|
mux.HandleFunc("GET /about", handleAbout)
|
|
mux.HandleFunc("GET /files/", bind((*Corpus).handleFiles))
|
|
mux.HandleFunc("GET /static/", handleStatic)
|
|
|
|
// JSON API (read-only) - see api.go for endpoint documentation.
|
|
if apiEnabled {
|
|
mux.HandleFunc("GET /api/docs", bind((*Corpus).handleAPIDocs))
|
|
mux.HandleFunc("GET /api/doc/{id}", bind((*Corpus).handleAPIDoc))
|
|
mux.HandleFunc("GET /api/search", bind((*Corpus).handleAPISearch))
|
|
mux.HandleFunc("GET /api/facets", bind((*Corpus).handleAPIFacets))
|
|
mux.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) {
|
|
apiError(w, http.StatusNotFound, "unknown API endpoint")
|
|
})
|
|
} else {
|
|
mux.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) {
|
|
apiError(w, http.StatusForbidden, "the JSON API is disabled on this instance")
|
|
})
|
|
}
|
|
|
|
return mux
|
|
}
|
|
|
|
// withBaseHeaders wraps a handler with the headers every response carries.
|
|
func withBaseHeaders(next http.Handler) http.Handler {
|
|
const csp = "default-src 'self'; img-src 'self'; script-src 'self'; " +
|
|
"style-src 'self' 'unsafe-inline'; object-src 'none'; base-uri 'none'; " +
|
|
"form-action 'self'; frame-ancestors 'self'"
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
h := w.Header()
|
|
h.Set("Content-Security-Policy", csp)
|
|
h.Set("X-Frame-Options", "SAMEORIGIN")
|
|
h.Set("X-Content-Type-Options", "nosniff")
|
|
h.Set("Referrer-Policy", "no-referrer")
|
|
switch r.Method {
|
|
case http.MethodGet, http.MethodHead, http.MethodPost:
|
|
default:
|
|
h.Set("Allow", "GET, HEAD, POST")
|
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
|
return
|
|
}
|
|
next.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
func envOr(key, fallback string) string {
|
|
if v := os.Getenv(key); v != "" {
|
|
return v
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func envBool(key string) bool {
|
|
v := os.Getenv(key)
|
|
if v == "" {
|
|
return false
|
|
}
|
|
b, err := strconv.ParseBool(v)
|
|
if err != nil {
|
|
log.Fatalf("$%s=%q is not a boolean (use 1/0 or true/false)", key, v)
|
|
}
|
|
return b
|
|
}
|
|
|
|
func envDuration(key string) time.Duration {
|
|
v := os.Getenv(key)
|
|
if v == "" {
|
|
return 0
|
|
}
|
|
d, err := time.ParseDuration(v)
|
|
if err != nil {
|
|
log.Fatalf("$%s=%q is not a duration (use e.g. 90s, 10m)", key, v)
|
|
}
|
|
return d
|
|
}
|
|
|
|
// accelPath is the X-Accel-Redirect target for a file: its path relative to
|
|
// the data directory under accelPrefix, percent-encoded because nginx decodes
|
|
// the header as a URI.
|
|
func accelPath(file string) (string, bool) {
|
|
rel, err := filepath.Rel(dataDir, file)
|
|
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
|
return "", false
|
|
}
|
|
u := url.URL{Path: accelPrefix + filepath.ToSlash(rel)}
|
|
return u.EscapedPath(), true
|
|
}
|
|
|
|
// handleFiles serves the files the loader indexed for a document, and no
|
|
// other file its directory may hold (Document.servedFile).
|
|
func (co *Corpus) handleFiles(w http.ResponseWriter, r *http.Request) {
|
|
rest := strings.TrimPrefix(r.URL.Path, "/files/")
|
|
id, sub, ok := strings.Cut(rest, "/")
|
|
if !ok || sub == "" {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
doc := co.docIndex[id]
|
|
if doc == nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
|
|
sub = path.Clean("/" + sub)
|
|
rel := strings.TrimPrefix(sub, "/")
|
|
if strings.Contains(sub, "..") || !doc.servedFile(rel) {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
|
|
cacheIfFile(w, os.DirFS(doc.Dir), rel, "public, max-age=86400")
|
|
if isTranscript(rel) {
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
}
|
|
file := filepath.Join(doc.Dir, filepath.FromSlash(sub))
|
|
if accelPrefix != "" {
|
|
if target, ok := accelPath(file); ok {
|
|
w.Header().Set("X-Accel-Redirect", target)
|
|
return
|
|
}
|
|
}
|
|
_ = http.NewResponseController(w).SetWriteDeadline(time.Now().Add(fileWriteBudget))
|
|
http.ServeFile(w, r, file)
|
|
}
|
|
|
|
const writeTimeout = 30 * time.Second
|
|
|
|
// fileWriteBudget is how long a single download may take in total. A 123 MB
|
|
// PDF over a bad mobile link (~35 KB/s) fits in an hour, so in practice this
|
|
// bound is only reached by a reader that has gone away.
|
|
var fileWriteBudget = time.Hour
|
|
|
|
func cacheIfFile(w http.ResponseWriter, fsys fs.FS, name, policy string) {
|
|
if fi, err := fs.Stat(fsys, name); err == nil && fi.Mode().IsRegular() {
|
|
w.Header().Set("Cache-Control", policy)
|
|
}
|
|
}
|
|
|
|
func handleStatic(w http.ResponseWriter, r *http.Request) {
|
|
if path.Ext(r.URL.Path) == "" {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
rel := strings.TrimPrefix(path.Clean("/"+strings.TrimPrefix(r.URL.Path, "/static/")), "/")
|
|
if h, ok := staticHashes[rel]; ok {
|
|
if r.URL.Query().Get("v") == h {
|
|
w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
|
|
}
|
|
w.Header().Set("ETag", `"`+h+`"`)
|
|
}
|
|
http.StripPrefix("/static/", http.FileServerFS(staticFS)).ServeHTTP(w, r)
|
|
}
|
|
|
|
func navSection(path string) string {
|
|
switch {
|
|
case path == "/":
|
|
return "home"
|
|
case path == "/browse" || strings.HasPrefix(path, "/browse/"):
|
|
return "browse"
|
|
case path == "/search":
|
|
return "search"
|
|
case path == "/articles" || strings.HasPrefix(path, "/article/"):
|
|
return "articles"
|
|
case path == "/glossary":
|
|
return "glossary"
|
|
case path == "/about":
|
|
return "about"
|
|
case path == "/pinned":
|
|
return "pinned"
|
|
case path == "/preferences":
|
|
return "preferences"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func render(w http.ResponseWriter, r *http.Request, data pageData) {
|
|
renderCode(w, r, data, http.StatusOK)
|
|
}
|
|
|
|
func crossSitePOST(r *http.Request) bool {
|
|
switch r.Header.Get("Sec-Fetch-Site") {
|
|
case "same-origin", "none":
|
|
return false
|
|
case "cross-site", "same-site":
|
|
return true
|
|
}
|
|
|
|
if o := r.Header.Get("Origin"); o != "" && o != "null" {
|
|
if u, err := url.Parse(o); err == nil && !originIsHost(u, r.Host) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func originIsHost(origin *url.URL, host string) bool {
|
|
def := map[string]string{"http": "80", "https": "443"}[strings.ToLower(origin.Scheme)]
|
|
op := origin.Port()
|
|
if op == "" {
|
|
op = def
|
|
}
|
|
h, hp, err := net.SplitHostPort(host)
|
|
if err != nil { // no port
|
|
h, hp = strings.Trim(host, "[]"), def
|
|
}
|
|
return strings.EqualFold(origin.Hostname(), h) && op == hp
|
|
}
|
|
|
|
func noStore(w http.ResponseWriter) {
|
|
w.Header().Set("Cache-Control", "private, no-store")
|
|
}
|
|
|
|
func readPOSTForm(w http.ResponseWriter, r *http.Request) bool {
|
|
if crossSitePOST(r) {
|
|
http.Error(w, "cross-site request rejected", http.StatusForbidden)
|
|
return false
|
|
}
|
|
r.Body = http.MaxBytesReader(w, r.Body, 1<<16)
|
|
if err := r.ParseForm(); err != nil {
|
|
http.Error(w, "bad form", http.StatusBadRequest)
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func crossSiteFetch(r *http.Request) bool {
|
|
switch r.Header.Get("Sec-Fetch-Site") {
|
|
case "cross-site", "same-site":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func renderCode(w http.ResponseWriter, r *http.Request, data pageData, code int) {
|
|
name := data.templateName()
|
|
if w.Header().Get("Cache-Control") == "" {
|
|
if (r.Method == http.MethodGet || r.Method == http.MethodHead) &&
|
|
(code == http.StatusOK || code == http.StatusNotFound) &&
|
|
r.Header.Get("Cookie") == "" {
|
|
w.Header().Set("Cache-Control", "public, max-age=10")
|
|
w.Header().Set("Vary", "Cookie")
|
|
if accelExpires != "" {
|
|
w.Header().Set("X-Accel-Expires", accelExpires)
|
|
}
|
|
} else {
|
|
w.Header().Set("Cache-Control", "private, no-cache")
|
|
}
|
|
}
|
|
|
|
b := data.base()
|
|
b.Nav = navSection(r.URL.Path)
|
|
b.APIEnabled = apiEnabled
|
|
// A handler that read the reader's preferences passes them on
|
|
// (pageWith); they are only read here for one that had no use for them.
|
|
if !b.hasPrefs {
|
|
b.Prefs = readPrefs(r)
|
|
}
|
|
b.Theme = b.Prefs.ThemeAttr()
|
|
b.CardPagePreview = b.Prefs.CardPagePreview
|
|
|
|
var t *template.Template
|
|
var err error
|
|
|
|
if devMode {
|
|
t, err = parsePage(name)
|
|
if err != nil {
|
|
log.Printf("Template Parse Error (%s): %v", name, err)
|
|
serverError(w, err)
|
|
return
|
|
}
|
|
} else {
|
|
t = tmplSet[name]
|
|
if t == nil {
|
|
log.Printf("Template not found: %s", name)
|
|
serverError(w, fmt.Errorf("template not found: %s", name))
|
|
return
|
|
}
|
|
}
|
|
|
|
var buf bytes.Buffer
|
|
if err := t.ExecuteTemplate(&buf, name, data); err != nil {
|
|
log.Printf("Template Exec Error (%s): %v", name, err)
|
|
serverError(w, err)
|
|
return
|
|
}
|
|
if code != http.StatusOK {
|
|
w.WriteHeader(code)
|
|
}
|
|
w.Write(bytes.TrimSpace(buf.Bytes()))
|
|
w.Write([]byte("\n"))
|
|
}
|
|
|
|
func notFound(w http.ResponseWriter, r *http.Request) {
|
|
renderCode(w, r, ¬FoundPage{Path: r.URL.Path}, http.StatusNotFound)
|
|
}
|
|
|
|
func parsePage(name string) (*template.Template, error) {
|
|
return template.New(name).Funcs(funcMap).ParseFS(templateFS, "base.html", "partials.html", name)
|
|
}
|
|
|
|
func parseTemplates() error {
|
|
pages, err := fs.Glob(templateFS, "*.html")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tmplSet = make(map[string]*template.Template)
|
|
for _, name := range pages {
|
|
if name == "base.html" || name == "partials.html" {
|
|
continue
|
|
}
|
|
t, err := parsePage(name)
|
|
if err != nil {
|
|
return fmt.Errorf("%s: %w", name, err)
|
|
}
|
|
tmplSet[name] = t
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func serverError(w http.ResponseWriter, err error) {
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
if devMode {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
|
}
|
|
|
|
func warnNonLoopback(addr string) {
|
|
host, _, err := net.SplitHostPort(addr)
|
|
if err != nil {
|
|
return
|
|
}
|
|
if host == "localhost" {
|
|
return
|
|
}
|
|
if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() {
|
|
return
|
|
}
|
|
warnf("-listen %s is not loopback - anything that can route to this host reaches the archive directly, bypassing your proxy's caching, rate limits and logs.", addr)
|
|
}
|