first commit

This commit is contained in:
libroot 2026-11-19 04:40:59 +02:00
commit 8c4dcae475
Signed by: libroot
GPG key ID: 2AC68C029EBE9ACE
89 changed files with 24623 additions and 0 deletions

162
corpus.go Normal file
View file

@ -0,0 +1,162 @@
package main
import (
"html/template"
"net/http"
"sync/atomic"
)
// Corpus is everything a load derives from the data and project directories: the
// documents, every index and table built from them, and the page bodies
// rendered ahead of time. One is built per load and never changed after it
// goes live, so handlers read it without locks.
type Corpus struct {
// api.go
facetsJSON []byte
facetsETag string
// articles.go
articles []Article
articleIndex map[string]*Article // key -> article
articleByLink map[string]*Article // link -> article
articleGroups []articleYearGroup
articlesTitled int
articlesBooks int // of articlesTitled
// assemblies.go
assemblies []*Assembly
assemblyIndex map[string]*Assembly
assembliesByDoc map[string][]*Assembly
// browse.go
// homeStats holds the corpus-derived widgets on the home page - the year
// histograms and top-N facet lists.
//
// By default every figure on the home page describes the REPORTED
// collection; homeStatsSIDtoday (/?sidtoday=1) adds the SIDtoday archive. FOIA
// material can postdate the June 2013 disclosures (a 2014 "year written"
// bar on a Snowden archive reads as impossible at a glance), and the
// SIDtoday archive is +2,100 issues of an internal newsletter that swamps
// the lists and the leak-era shape the charts exist to show.
// Every bar and list entry links to its /browse/{category}/{value} page
// scoped to that collection, so the number shown is the number of
// documents the page lists.
homeStats, homeStatsSIDtoday homeFigures
randomPools map[string][]int
collectionDocs map[string][]*Document
sidtodayDocs []*Document
// everyDoc is every document as pointers, built once.
everyDoc []*Document
archiveTimeline []archiveYear
categoryRowHTML map[string][]template.HTML // table key -> each entry's row, as Entries has them
azRowHTML map[string]template.HTML
categoryRowSize map[string]int
docRowHTML map[string]template.HTML
docRowMarkedHTML map[string]template.HTML // rows with the "reported" badge, for ReportedIssue documents
// categoryTables holds every category's table under its slug.
categoryTables map[string]*categoryTable
// seriesBySize is the series index of the SIDtoday tab.
seriesBySize map[bool][]categoryEntry
// categoryValueCounts is how many values each category's list holds.
categoryValueCounts map[bool]map[string]int
// categoryDocs is the posting list behind every /browse/{category}/{value}
// page: slug -> folded value -> the documents holding it, in corpus order.
categoryDocs map[string]map[string][]int32
// categoryNames maps each category's index keys to the spelling the lists
// and counts show: the first one in corpus order.
categoryNames map[string]map[string]string
// categoryTypeCounts splits each category value's count by document type
// (REPORTED / SIDTODAY / FOIA): the list pages' breakdown columns. They
// count type, not collection.
categoryTypeCounts map[string]map[string]*collCounts
// changelog.go
changelog []ChangeEntry
// codewords.go
codewords []codewordEntry
codewordTable []codewordRow
codewordViews map[string]codewordView
// data.go
archive Archive
serialPrev map[string]string
serialNext map[string]string
seriesIndex map[string]*seriesGroup
// explore.go
exploreTables map[string][]exploreTable
classPatterns struct {
Strips []clsYearRow
Sparse []clsYearRow
Cols []facetEntry // crossCols, each linked to its page
HeatRows []heatRow
}
// explore_ids.go
idRows []idRow
idChecksums string
// glossary.go
glossary []GlossaryEntry
glossaryIndex map[string]*GlossaryEntry
glossaryGroups []glossaryGroup
// main.go
prerendered map[string]template.HTML
// outlets.go
outletTables map[string]outletTable
// portions.go
portionStats portionSummary
// search.go
tfidf TFIDFIndex
// docIndex is populated by loadData() in data.go after archive.Docs is fully
// built. Provides O(1) doc lookup by ID.
docIndex map[string]*Document
// seo.go
sitemapPaths []string
canonicalSitemap []byte
// sitemap.go
sitemapParts [][]byte
}
// live is the corpus requests are served from. It is replaced whole, never
// modified: a load builds a new Corpus and stores it here when it is done.
var live atomic.Pointer[Corpus]
func init() { live.Store(&Corpus{}) }
// corpus returns the live corpus. Handlers do not call it. bind hands each
// request the corpus that was live when it arrived, and everything the
// request touches is reached through that one value.
func corpus() *Corpus { return live.Load() }
// loadData reads the corpus under dataDir into a new Corpus, pre-renders its
// big page bodies (from tmplSet, so parse the templates first).
func loadData() {
co := &Corpus{}
co.load()
co.prerenderPages()
live.Store(co)
}
// bind adapts a handler to the mux. the request is served from the corpus
// that is live when it arrives, and from that one throughout.
func bind(h func(*Corpus, http.ResponseWriter, *http.Request)) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { h(live.Load(), w, r) }
}