package main import ( "html/template" "net/http" "sync/atomic" ) // Corpus is everything a load derives from the data and project directories: the // documents, every index and table built from them, and the page bodies // rendered ahead of time. One is built per load and never changed after it // goes live, so handlers read it without locks. type Corpus struct { // api.go facetsJSON []byte facetsETag string // articles.go articles []Article articleIndex map[string]*Article // key -> article articleByLink map[string]*Article // link -> article articleGroups []articleYearGroup articlesTitled int articlesBooks int // of articlesTitled // assemblies.go assemblies []*Assembly assemblyIndex map[string]*Assembly assembliesByDoc map[string][]*Assembly // browse.go // homeStats holds the corpus-derived widgets on the home page - the year // histograms and top-N facet lists. // // By default every figure on the home page describes the REPORTED // collection; homeStatsSIDtoday (/?sidtoday=1) adds the SIDtoday archive. FOIA // material can postdate the June 2013 disclosures (a 2014 "year written" // bar on a Snowden archive reads as impossible at a glance), and the // SIDtoday archive is +2,100 issues of an internal newsletter that swamps // the lists and the leak-era shape the charts exist to show. // Every bar and list entry links to its /browse/{category}/{value} page // scoped to that collection, so the number shown is the number of // documents the page lists. homeStats, homeStatsSIDtoday homeFigures randomPools map[string][]int collectionDocs map[string][]*Document sidtodayDocs []*Document // everyDoc is every document as pointers, built once. everyDoc []*Document archiveTimeline []archiveYear categoryRowHTML map[string][]template.HTML // table key -> each entry's row, as Entries has them azRowHTML map[string]template.HTML categoryRowSize map[string]int docRowHTML map[string]template.HTML docRowMarkedHTML map[string]template.HTML // rows with the "reported" badge, for ReportedIssue documents // categoryTables holds every category's table under its slug. categoryTables map[string]*categoryTable // seriesBySize is the series index of the SIDtoday tab. seriesBySize map[bool][]categoryEntry // categoryValueCounts is how many values each category's list holds. categoryValueCounts map[bool]map[string]int // categoryDocs is the posting list behind every /browse/{category}/{value} // page: slug -> folded value -> the documents holding it, in corpus order. categoryDocs map[string]map[string][]int32 // categoryNames maps each category's index keys to the spelling the lists // and counts show: the first one in corpus order. categoryNames map[string]map[string]string // categoryTypeCounts splits each category value's count by document type // (REPORTED / SIDTODAY / FOIA): the list pages' breakdown columns. They // count type, not collection. categoryTypeCounts map[string]map[string]*collCounts // changelog.go changelog []ChangeEntry // codewords.go codewords []codewordEntry codewordTable []codewordRow codewordViews map[string]codewordView // data.go archive Archive serialPrev map[string]string serialNext map[string]string seriesIndex map[string]*seriesGroup // explore.go exploreTables map[string][]exploreTable classPatterns struct { Strips []clsYearRow Sparse []clsYearRow Cols []facetEntry // crossCols, each linked to its page HeatRows []heatRow } // explore_ids.go idRows []idRow idChecksums string // glossary.go glossary []GlossaryEntry glossaryIndex map[string]*GlossaryEntry glossaryGroups []glossaryGroup // main.go prerendered map[string]template.HTML // outlets.go outletTables map[string]outletTable // portions.go portionStats portionSummary // search.go tfidf TFIDFIndex // docIndex is populated by loadData() in data.go after archive.Docs is fully // built. Provides O(1) doc lookup by ID. docIndex map[string]*Document // seo.go sitemapPaths []string canonicalSitemap []byte // sitemap.go sitemapParts [][]byte } // live is the corpus requests are served from. It is replaced whole, never // modified: a load builds a new Corpus and stores it here when it is done. var live atomic.Pointer[Corpus] func init() { live.Store(&Corpus{}) } // corpus returns the live corpus. Handlers do not call it. bind hands each // request the corpus that was live when it arrived, and everything the // request touches is reached through that one value. func corpus() *Corpus { return live.Load() } // loadData reads the corpus under dataDir into a new Corpus, pre-renders its // big page bodies (from tmplSet, so parse the templates first). func loadData() { co := &Corpus{} co.load() co.prerenderPages() live.Store(co) } // bind adapts a handler to the mux. the request is served from the corpus // that is live when it arrives, and from that one throughout. func bind(h func(*Corpus, http.ResponseWriter, *http.Request)) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { h(live.Load(), w, r) } }