first commit
This commit is contained in:
commit
8c4dcae475
89 changed files with 24623 additions and 0 deletions
162
corpus.go
Normal file
162
corpus.go
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"html/template"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// Corpus is everything a load derives from the data and project directories: the
|
||||
// documents, every index and table built from them, and the page bodies
|
||||
// rendered ahead of time. One is built per load and never changed after it
|
||||
// goes live, so handlers read it without locks.
|
||||
type Corpus struct {
|
||||
// api.go
|
||||
facetsJSON []byte
|
||||
facetsETag string
|
||||
|
||||
// articles.go
|
||||
articles []Article
|
||||
articleIndex map[string]*Article // key -> article
|
||||
articleByLink map[string]*Article // link -> article
|
||||
articleGroups []articleYearGroup
|
||||
articlesTitled int
|
||||
articlesBooks int // of articlesTitled
|
||||
|
||||
// assemblies.go
|
||||
assemblies []*Assembly
|
||||
assemblyIndex map[string]*Assembly
|
||||
assembliesByDoc map[string][]*Assembly
|
||||
|
||||
// browse.go
|
||||
// homeStats holds the corpus-derived widgets on the home page - the year
|
||||
// histograms and top-N facet lists.
|
||||
//
|
||||
// By default every figure on the home page describes the REPORTED
|
||||
// collection; homeStatsSIDtoday (/?sidtoday=1) adds the SIDtoday archive. FOIA
|
||||
// material can postdate the June 2013 disclosures (a 2014 "year written"
|
||||
// bar on a Snowden archive reads as impossible at a glance), and the
|
||||
// SIDtoday archive is +2,100 issues of an internal newsletter that swamps
|
||||
// the lists and the leak-era shape the charts exist to show.
|
||||
// Every bar and list entry links to its /browse/{category}/{value} page
|
||||
// scoped to that collection, so the number shown is the number of
|
||||
// documents the page lists.
|
||||
homeStats, homeStatsSIDtoday homeFigures
|
||||
|
||||
randomPools map[string][]int
|
||||
collectionDocs map[string][]*Document
|
||||
sidtodayDocs []*Document
|
||||
|
||||
// everyDoc is every document as pointers, built once.
|
||||
everyDoc []*Document
|
||||
|
||||
archiveTimeline []archiveYear
|
||||
categoryRowHTML map[string][]template.HTML // table key -> each entry's row, as Entries has them
|
||||
azRowHTML map[string]template.HTML
|
||||
categoryRowSize map[string]int
|
||||
docRowHTML map[string]template.HTML
|
||||
docRowMarkedHTML map[string]template.HTML // rows with the "reported" badge, for ReportedIssue documents
|
||||
|
||||
// categoryTables holds every category's table under its slug.
|
||||
categoryTables map[string]*categoryTable
|
||||
|
||||
// seriesBySize is the series index of the SIDtoday tab.
|
||||
seriesBySize map[bool][]categoryEntry
|
||||
|
||||
// categoryValueCounts is how many values each category's list holds.
|
||||
categoryValueCounts map[bool]map[string]int
|
||||
|
||||
// categoryDocs is the posting list behind every /browse/{category}/{value}
|
||||
// page: slug -> folded value -> the documents holding it, in corpus order.
|
||||
categoryDocs map[string]map[string][]int32
|
||||
|
||||
// categoryNames maps each category's index keys to the spelling the lists
|
||||
// and counts show: the first one in corpus order.
|
||||
categoryNames map[string]map[string]string
|
||||
|
||||
// categoryTypeCounts splits each category value's count by document type
|
||||
// (REPORTED / SIDTODAY / FOIA): the list pages' breakdown columns. They
|
||||
// count type, not collection.
|
||||
categoryTypeCounts map[string]map[string]*collCounts
|
||||
|
||||
// changelog.go
|
||||
changelog []ChangeEntry
|
||||
|
||||
// codewords.go
|
||||
codewords []codewordEntry
|
||||
codewordTable []codewordRow
|
||||
codewordViews map[string]codewordView
|
||||
|
||||
// data.go
|
||||
archive Archive
|
||||
serialPrev map[string]string
|
||||
serialNext map[string]string
|
||||
seriesIndex map[string]*seriesGroup
|
||||
|
||||
// explore.go
|
||||
exploreTables map[string][]exploreTable
|
||||
classPatterns struct {
|
||||
Strips []clsYearRow
|
||||
Sparse []clsYearRow
|
||||
Cols []facetEntry // crossCols, each linked to its page
|
||||
HeatRows []heatRow
|
||||
}
|
||||
|
||||
// explore_ids.go
|
||||
idRows []idRow
|
||||
idChecksums string
|
||||
|
||||
// glossary.go
|
||||
glossary []GlossaryEntry
|
||||
glossaryIndex map[string]*GlossaryEntry
|
||||
glossaryGroups []glossaryGroup
|
||||
|
||||
// main.go
|
||||
prerendered map[string]template.HTML
|
||||
|
||||
// outlets.go
|
||||
outletTables map[string]outletTable
|
||||
|
||||
// portions.go
|
||||
portionStats portionSummary
|
||||
|
||||
// search.go
|
||||
tfidf TFIDFIndex
|
||||
|
||||
// docIndex is populated by loadData() in data.go after archive.Docs is fully
|
||||
// built. Provides O(1) doc lookup by ID.
|
||||
docIndex map[string]*Document
|
||||
|
||||
// seo.go
|
||||
sitemapPaths []string
|
||||
canonicalSitemap []byte
|
||||
|
||||
// sitemap.go
|
||||
sitemapParts [][]byte
|
||||
}
|
||||
|
||||
// live is the corpus requests are served from. It is replaced whole, never
|
||||
// modified: a load builds a new Corpus and stores it here when it is done.
|
||||
var live atomic.Pointer[Corpus]
|
||||
|
||||
func init() { live.Store(&Corpus{}) }
|
||||
|
||||
// corpus returns the live corpus. Handlers do not call it. bind hands each
|
||||
// request the corpus that was live when it arrived, and everything the
|
||||
// request touches is reached through that one value.
|
||||
func corpus() *Corpus { return live.Load() }
|
||||
|
||||
// loadData reads the corpus under dataDir into a new Corpus, pre-renders its
|
||||
// big page bodies (from tmplSet, so parse the templates first).
|
||||
func loadData() {
|
||||
co := &Corpus{}
|
||||
co.load()
|
||||
co.prerenderPages()
|
||||
live.Store(co)
|
||||
}
|
||||
|
||||
// bind adapts a handler to the mux. the request is served from the corpus
|
||||
// that is live when it arrives, and from that one throughout.
|
||||
func bind(h func(*Corpus, http.ResponseWriter, *http.Request)) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) { h(live.Load(), w, r) }
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue