162 lines
5.1 KiB
Go
162 lines
5.1 KiB
Go
package main
|
|
|
|
import (
|
|
"html/template"
|
|
"net/http"
|
|
"sync/atomic"
|
|
)
|
|
|
|
// Corpus is everything a load derives from the data and project directories: the
|
|
// documents, every index and table built from them, and the page bodies
|
|
// rendered ahead of time. One is built per load and never changed after it
|
|
// goes live, so handlers read it without locks.
|
|
type Corpus struct {
|
|
// api.go
|
|
facetsJSON []byte
|
|
facetsETag string
|
|
|
|
// articles.go
|
|
articles []Article
|
|
articleIndex map[string]*Article // key -> article
|
|
articleByLink map[string]*Article // link -> article
|
|
articleGroups []articleYearGroup
|
|
articlesTitled int
|
|
articlesBooks int // of articlesTitled
|
|
|
|
// assemblies.go
|
|
assemblies []*Assembly
|
|
assemblyIndex map[string]*Assembly
|
|
assembliesByDoc map[string][]*Assembly
|
|
|
|
// browse.go
|
|
// homeStats holds the corpus-derived widgets on the home page - the year
|
|
// histograms and top-N facet lists.
|
|
//
|
|
// By default every figure on the home page describes the REPORTED
|
|
// collection; homeStatsSIDtoday (/?sidtoday=1) adds the SIDtoday archive. FOIA
|
|
// material can postdate the June 2013 disclosures (a 2014 "year written"
|
|
// bar on a Snowden archive reads as impossible at a glance), and the
|
|
// SIDtoday archive is +2,100 issues of an internal newsletter that swamps
|
|
// the lists and the leak-era shape the charts exist to show.
|
|
// Every bar and list entry links to its /browse/{category}/{value} page
|
|
// scoped to that collection, so the number shown is the number of
|
|
// documents the page lists.
|
|
homeStats, homeStatsSIDtoday homeFigures
|
|
|
|
randomPools map[string][]int
|
|
collectionDocs map[string][]*Document
|
|
sidtodayDocs []*Document
|
|
|
|
// everyDoc is every document as pointers, built once.
|
|
everyDoc []*Document
|
|
|
|
archiveTimeline []archiveYear
|
|
categoryRowHTML map[string][]template.HTML // table key -> each entry's row, as Entries has them
|
|
azRowHTML map[string]template.HTML
|
|
categoryRowSize map[string]int
|
|
docRowHTML map[string]template.HTML
|
|
docRowMarkedHTML map[string]template.HTML // rows with the "reported" badge, for ReportedIssue documents
|
|
|
|
// categoryTables holds every category's table under its slug.
|
|
categoryTables map[string]*categoryTable
|
|
|
|
// seriesBySize is the series index of the SIDtoday tab.
|
|
seriesBySize map[bool][]categoryEntry
|
|
|
|
// categoryValueCounts is how many values each category's list holds.
|
|
categoryValueCounts map[bool]map[string]int
|
|
|
|
// categoryDocs is the posting list behind every /browse/{category}/{value}
|
|
// page: slug -> folded value -> the documents holding it, in corpus order.
|
|
categoryDocs map[string]map[string][]int32
|
|
|
|
// categoryNames maps each category's index keys to the spelling the lists
|
|
// and counts show: the first one in corpus order.
|
|
categoryNames map[string]map[string]string
|
|
|
|
// categoryTypeCounts splits each category value's count by document type
|
|
// (REPORTED / SIDTODAY / FOIA): the list pages' breakdown columns. They
|
|
// count type, not collection.
|
|
categoryTypeCounts map[string]map[string]*collCounts
|
|
|
|
// changelog.go
|
|
changelog []ChangeEntry
|
|
|
|
// codewords.go
|
|
codewords []codewordEntry
|
|
codewordTable []codewordRow
|
|
codewordViews map[string]codewordView
|
|
|
|
// data.go
|
|
archive Archive
|
|
serialPrev map[string]string
|
|
serialNext map[string]string
|
|
seriesIndex map[string]*seriesGroup
|
|
|
|
// explore.go
|
|
exploreTables map[string][]exploreTable
|
|
classPatterns struct {
|
|
Strips []clsYearRow
|
|
Sparse []clsYearRow
|
|
Cols []facetEntry // crossCols, each linked to its page
|
|
HeatRows []heatRow
|
|
}
|
|
|
|
// explore_ids.go
|
|
idRows []idRow
|
|
idChecksums string
|
|
|
|
// glossary.go
|
|
glossary []GlossaryEntry
|
|
glossaryIndex map[string]*GlossaryEntry
|
|
glossaryGroups []glossaryGroup
|
|
|
|
// main.go
|
|
prerendered map[string]template.HTML
|
|
|
|
// outlets.go
|
|
outletTables map[string]outletTable
|
|
|
|
// portions.go
|
|
portionStats portionSummary
|
|
|
|
// search.go
|
|
tfidf TFIDFIndex
|
|
|
|
// docIndex is populated by loadData() in data.go after archive.Docs is fully
|
|
// built. Provides O(1) doc lookup by ID.
|
|
docIndex map[string]*Document
|
|
|
|
// seo.go
|
|
sitemapPaths []string
|
|
canonicalSitemap []byte
|
|
|
|
// sitemap.go
|
|
sitemapParts [][]byte
|
|
}
|
|
|
|
// live is the corpus requests are served from. It is replaced whole, never
|
|
// modified: a load builds a new Corpus and stores it here when it is done.
|
|
var live atomic.Pointer[Corpus]
|
|
|
|
func init() { live.Store(&Corpus{}) }
|
|
|
|
// corpus returns the live corpus. Handlers do not call it. bind hands each
|
|
// request the corpus that was live when it arrived, and everything the
|
|
// request touches is reached through that one value.
|
|
func corpus() *Corpus { return live.Load() }
|
|
|
|
// loadData reads the corpus under dataDir into a new Corpus, pre-renders its
|
|
// big page bodies (from tmplSet, so parse the templates first).
|
|
func loadData() {
|
|
co := &Corpus{}
|
|
co.load()
|
|
co.prerenderPages()
|
|
live.Store(co)
|
|
}
|
|
|
|
// bind adapts a handler to the mux. the request is served from the corpus
|
|
// that is live when it arrives, and from that one throughout.
|
|
func bind(h func(*Corpus, http.ResponseWriter, *http.Request)) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) { h(live.Load(), w, r) }
|
|
}
|