package main
import (
"bytes"
"cmp"
"fmt"
"html/template"
"math/rand"
"net/http"
"net/url"
"regexp"
"slices"
"sort"
"strconv"
"strings"
"unicode"
"unicode/utf8"
)
type facetEntry struct {
Name string
Count int
URL string
}
// facetLinks renders a metadata field's values as links to their category
// pages, wherever a card, a table row, a search result or a document page
// lists them. It takes the template's dict:
//
// Vals the values ("-" stands for none)
// Cat the category's slug
// Sep between the links: "dot", "space", or a comma when not given
// AsList one
per link instead
// Class the links' class
// Active the value of the page the links are on, when it is this
// category's: its own link is marked active
// QS the collection scope the links carry ("?coll=..."), when the page
// they open would otherwise leave out the document they come from
//
// The output is what html/template made of the same markup: every value
// goes through htmlEscape.
func facetLinks(p map[string]interface{}) (template.HTML, error) {
vals, ok := p["Vals"].([]string)
if !ok && p["Vals"] != nil {
return "", fmt.Errorf("facetLinks: Vals is %T, not a list of values", p["Vals"])
}
if len(vals) == 0 {
return "-", nil
}
cat, _ := p["Cat"].(string)
sep, _ := p["Sep"].(string)
class, _ := p["Class"].(string)
active, _ := p["Active"].(string)
qs, _ := p["QS"].(string)
asList, _ := p["AsList"].(bool)
var b strings.Builder
if asList {
b.WriteString(`")
}
return template.HTML(b.String()), nil
}
func htmlEscape(s string) string {
if !strings.ContainsAny(s, "\x00\"&'+<>") && isASCII(s) {
return s
}
var b strings.Builder
b.Grow(len(s) + 16)
for i := 0; i < len(s); {
r, w := utf8.DecodeRuneInString(s[i:])
switch {
case r == 0:
b.WriteString("\uFFFD")
case r == '"':
b.WriteString(""")
case r == '&':
b.WriteString("&")
case r == '\'':
b.WriteString("'")
case r == '+':
b.WriteString("+")
case r == '<':
b.WriteString("<")
case r == '>':
b.WriteString(">")
case r >= 0xfdd0 && r <= 0xfdef, r >= 0xfff0 && r <= 0xffff:
fmt.Fprintf(&b, "%x;", r)
default:
b.WriteString(s[i : i+w])
}
i += w
}
return b.String()
}
// rankCounts returns the keys of a count map in rank order (most counted
// first, ties alphabetically ignoring accents and case, then by the raw
// spelling)
func rankCounts(m map[string]int, n int) []string {
type keyed struct {
name, fold string
count int
}
all := make([]keyed, 0, len(m))
for k, v := range m {
all = append(all, keyed{k, normFoldString(k), v})
}
sort.Slice(all, func(i, j int) bool {
a, b := &all[i], &all[j]
if a.count != b.count {
return a.count > b.count
}
if a.fold != b.fold {
return a.fold < b.fold
}
return a.name < b.name
})
if n >= 0 && len(all) > n {
all = all[:n]
}
out := make([]string, len(all))
for i := range all {
out[i] = all[i].name
}
return out
}
func topFacets(m map[string]int, n int) []facetEntry {
names := rankCounts(m, n)
out := make([]facetEntry, len(names))
for i, k := range names {
out[i] = facetEntry{Name: k, Count: m[k]}
}
return out
}
func barPct(count, denom, floor int) int {
if denom <= 0 || count <= 0 {
return 0
}
pct := count * 100 / denom
if pct < floor {
pct = floor
}
return pct
}
func sortedYears(m map[string]int) (years []string, maxCount int) {
years = make([]string, 0, len(m))
for y, c := range m {
years = append(years, y)
if c > maxCount {
maxCount = c
}
}
sort.Strings(years)
return years, maxCount
}
func yearBars(m map[string]int, urlFor func(y string) string) []timelineBar {
years, maxCount := sortedYears(m)
if maxCount == 0 {
return nil
}
bars := make([]timelineBar, 0, len(years))
for _, y := range years {
bars = append(bars, timelineBar{
Year: y,
Count: m[y],
Pct: barPct(m[y], maxCount, 1),
URL: urlFor(y),
})
}
return bars
}
type homeFigures struct {
TopAgencies, TopPrograms, TopCountries, TopPeople,
TopJournalists, TopPublishers, TopTags []facetEntry
DocHist, PubHist []timelineBar
}
func (co *Corpus) buildHomeStats() {
co.homeStats = co.homeFigures(false)
co.homeStatsSIDtoday = co.homeFigures(true)
co.buildRandomPools()
}
func (co *Corpus) homeFigures(sidtoday bool) homeFigures {
scope := scopeOf(map[string]bool{collectionNews: true, collectionArchive: sidtoday})
top := func(slug string, n int) []facetEntry {
entries := topFacets(co.homeFacet(slug, sidtoday), n)
for i := range entries {
entries[i].URL = scope.link(categoryValueURL(slug, entries[i].Name))
}
return entries
}
f := homeFigures{
TopAgencies: top("agency", 10),
TopPrograms: top("program", 20),
TopCountries: top("country", 15),
TopPeople: top("people", 15),
TopJournalists: top("journalist", 12),
TopPublishers: top("publisher", 10),
TopTags: top("tag", 25),
}
f.DocHist, f.PubHist = co.homeHistograms(scope)
return f
}
func (co *Corpus) homeFacet(slug string, sidtoday bool) map[string]int {
out := make(map[string]int)
for value, c := range co.archive.FacetsByColl[slug] {
n := c.News
if sidtoday {
n += c.Archive
}
if n > 0 {
out[value] = n
}
}
return out
}
func (co *Corpus) homeHistograms(scope collScope) (docHist, pubHist []timelineBar) {
docYears := map[string]int{}
pubYears := map[string]int{}
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
if !scope.shows(d) {
continue
}
if y := docYear(d); y != "" {
docYears[y]++
}
if y := d.FirstPubYear(); y != "" {
pubYears[y]++
}
}
link := func(slug string) func(string) string {
return func(y string) string { return scope.link(categoryValueURL(slug, y)) }
}
return yearBars(docYears, link("docdate")), yearBars(pubYears, link("published"))
}
func (co *Corpus) addFacetNav(b *Base, prefs Prefs) {
b.ShowCatBoxes = true
b.CatCounts = co.categoryValueCounts[prefs.IncludeFOIA]
}
func (co *Corpus) handleIndex(w http.ResponseWriter, r *http.Request) {
prefs := readPrefs(r)
var recentDocs []*Document
if prefs.TrackRecent {
for _, id := range readIDListCookie(r, recentDocsCookie, recentDocsMax) {
if d := co.docIndex[id]; d != nil {
recentDocs = append(recentDocs, d)
}
}
}
if len(recentDocs) > 0 {
noStore(w)
}
withSIDtoday := r.URL.Query().Get("sidtoday") == "1"
stats := co.homeStats
if withSIDtoday {
stats = co.homeStatsSIDtoday
}
render(w, r, &indexPage{
Base: pageWith(prefs),
homeFigures: stats,
TrackRecent: prefs.TrackRecent,
RecentDocs: recentDocs,
SnowdenTotal: co.archive.CollectionCounts[collectionNews] + co.archive.CollectionCounts[collectionArchive],
ArchiveTotal: co.archive.CollectionCounts[collectionArchive],
FOIATotal: co.archive.CollectionCounts[collectionFOIA],
IncludeFOIA: prefs.IncludeFOIA,
ReportedTotal: co.archive.CollectionCounts[collectionNews],
WithSIDtoday: withSIDtoday,
})
}
// randomWeights says how often /random lands in each collection
var randomWeights = []struct {
coll string
weight int
}{
{collectionNews, 65},
{collectionArchive, 25},
{collectionFOIA, 10},
}
func (co *Corpus) buildRandomPools() {
co.randomPools = map[string][]int{}
for i, d := range co.archive.Docs {
co.randomPools[d.Collection] = append(co.randomPools[d.Collection], i)
}
}
func (co *Corpus) buildCollectionDocs() {
co.everyDoc = make([]*Document, len(co.archive.Docs))
co.collectionDocs = make(map[string][]*Document, len(validCollections))
co.sidtodayDocs = nil
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
co.everyDoc[i] = d
co.collectionDocs[d.Collection] = append(co.collectionDocs[d.Collection], d)
if d.Type == "SIDTODAY" {
co.sidtodayDocs = append(co.sidtodayDocs, d)
}
}
}
func (co *Corpus) randomDoc(includeFOIA bool) *Document {
pool := func(coll string) []int {
if coll == collectionFOIA && !includeFOIA {
return nil
}
return co.randomPools[coll]
}
total := 0
for _, rw := range randomWeights {
if len(pool(rw.coll)) > 0 {
total += rw.weight
}
}
if total == 0 {
return nil
}
n := rand.Intn(total)
for _, rw := range randomWeights {
p := pool(rw.coll)
if len(p) == 0 {
continue
}
if n < rw.weight {
return &co.archive.Docs[p[rand.Intn(len(p))]]
}
n -= rw.weight
}
return nil
}
func (co *Corpus) handleRandom(w http.ResponseWriter, r *http.Request) {
d := co.randomDoc(readPrefs(r).IncludeFOIA)
if d == nil {
notFound(w, r)
return
}
http.Redirect(w, r, docPath(d.ID), http.StatusFound)
}
func handleAbout(w http.ResponseWriter, r *http.Request) {
render(w, r, &aboutPage{})
}
func paginate(total, page, perPage int) (start, end, clampedPage, totalPages int) {
if perPage <= 0 {
return 0, total, 1, 1
}
totalPages = (total + perPage - 1) / perPage
if page < 1 {
page = 1
}
maxPage := totalPages
if maxPage < 1 {
maxPage = 1
}
if page > maxPage {
page = maxPage
}
start = (page - 1) * perPage
if start < 0 || start > total {
start = total
}
end = start + perPage
if end < start || end > total {
end = total
}
return start, end, page, totalPages
}
// buildListView assembles everything the shared doc-cards / doc-table
// layouts need: view, sort, pagination, the link bases their partials
// build on, and the card display prefs.
//
// The page embeds what is returned and adds its own fields, including the
// cards' ActiveCategory and ActiveValue when it has a filter value for them
// to highlight.
func (co *Corpus) buildListView(q url.Values, prefs Prefs, docs []*Document, basePath string, params url.Values, defaultSort string, explicitView bool) listView {
return co.buildListViewBoosted(q, prefs, docs, basePath, params, defaultSort, explicitView, "")
}
func (co *Corpus) buildListViewBoosted(q url.Values, prefs Prefs, docs []*Document, basePath string, params url.Values, defaultSort string, explicitView bool, boost string) listView {
view := q.Get("view")
sortBy := sortToken(q.Get("sort"))
order := sortToken(q.Get("order"))
if docSortByKey[sortBy] == nil {
sortBy = ""
}
effSort := cmp.Or(sortBy, defaultSort)
sortDocs(docs, effSort, order)
if sortBy == "" && boost != "" {
boostTitleMatches(docs, boost)
}
page, _ := strconv.Atoi(q.Get("p"))
finalDocs := docs
totalPages := 1
if view == "table" {
page = 1
} else {
var start, end int
start, end, page, totalPages = paginate(len(docs), page, prefs.cardsPerPage())
finalDocs = docs[start:end]
}
baseQ := cloneValues(params)
sortQ := cloneValues(params)
if view != "" && (explicitView || view == "table") {
baseQ.Set("view", view)
}
lv := listView{
Docs: finalDocs,
Total: len(docs),
Page: page,
TotalPages: totalPages,
View: view,
Sort: sortBy,
Order: order,
DefaultSort: defaultSort,
BaseURL: queryBase(basePath, baseQ),
SortURL: queryBase(basePath, sortQ),
TableHeads: sortHeads(queryBase(basePath, baseQ), docTableCols, effSort, order == "desc"),
Cards: newCardView(prefs, finalDocs),
}
if view == "table" {
lv.RowsHTML = co.buildDocRows(finalDocs, false)
}
return lv
}
// selectedTypes reads the ?type= params into a set, and reports whether the
// request actually filters (no params means everything, not nothing).
func selectedTypes(v url.Values) (map[string]bool, bool) {
sel := make(map[string]bool, len(v["type"]))
for _, t := range v["type"] {
if t = strings.ToUpper(strings.TrimSpace(t)); t != "" {
sel[t] = true
}
}
return sel, len(sel) > 0
}
func typeSelected(d *Document, sel map[string]bool) bool {
return len(sel) == 0 || sel[d.Type]
}
func typeCounts(docs []*Document) map[string]int {
counts := make(map[string]int)
for i := range docs {
counts[docs[i].Type]++
}
return counts
}
func filterByType(docs []*Document, sel map[string]bool) []*Document {
if len(sel) == 0 {
return docs
}
kept := docs[:0]
for _, d := range docs {
if typeSelected(d, sel) {
kept = append(kept, d)
}
}
return kept
}
func allTypesSelected(counts map[string]int) map[string]bool {
sel := make(map[string]bool, len(counts))
for t := range counts {
sel[t] = true
}
return sel
}
var validCollections = map[string]bool{
collectionNews: true,
collectionArchive: true,
collectionFOIA: true,
}
var collectionOrder = []string{collectionNews, collectionArchive, collectionFOIA}
func collectionParams(sel map[string]bool) []string {
var out []string
for _, c := range collectionOrder {
if sel[c] {
out = append(out, c)
}
}
return out
}
// defaultCollections is what a page shows when its URL names no
// collection: FOIA releases are opt-in (Prefs.IncludeFOIA).
var defaultCollections = map[string]bool{collectionNews: true, collectionArchive: true}
// collScope is a page's collection scope: the documents it shows and what
// its own links carry. Every link a page builds from its documents goes
// through link, so the number on a link is the number its click returns.
type collScope struct {
filter map[string]bool // the documents shown; nil: every collection
shown map[string]bool // the same as a set, never nil (the form's boxes)
defaulted bool // the URL named no collection: the preference decided
colls []string // the coll= the page's links carry; nil: none
extra string // more scope for the links (a publisher page's year)
}
// scopeOf is an explicit scope: sel, with links that name it, so a count
// holds whatever the next reader's FOIA preference.
func scopeOf(sel map[string]bool) collScope {
s := collScope{shown: sel, colls: collectionParams(sel)}
if len(s.colls) < len(validCollections) {
s.filter = sel
}
return s
}
// wholeArchive scopes links to all three collections: a count taken over
// the whole archive must not change with the reader's FOIA preference.
var wholeArchive = scopeOf(validCollections)
// pageScope reads a page's scope: the coll= its URL names or, when it
// names none, the reader's preference, FOIA releases only when opted in.
func pageScope(v url.Values, p Prefs, pinDefault bool) collScope {
if _, given := v["coll"]; given {
if f := collectionFilter(v); f != nil {
return scopeOf(f)
}
return wholeArchive
}
s := scopeOf(defaultCollections)
if p.IncludeFOIA {
s = wholeArchive
}
s.defaulted = true
if !pinDefault {
s.colls = nil
}
return s
}
func (s collScope) shows(d *Document) bool { return inCollections(d, s.filter) }
func (s collScope) hasValue(co *Corpus, slug, value string) bool {
c := co.archive.FacetsByColl[slug][co.categoryNames[slug][valueKey(value)]]
if c == nil {
return false
}
in := func(coll string, n int) bool { return n > 0 && (s.filter == nil || s.filter[coll]) }
return in(collectionNews, c.News) || in(collectionArchive, c.Archive) || in(collectionFOIA, c.FOIA)
}
func (s collScope) docs(docs []*Document) []*Document { return filterByCollection(docs, s.filter) }
func (s collScope) explicit() bool { return s.filter != nil && !s.defaulted }
func (s collScope) including(coll string) collScope {
if s.filter == nil || s.filter[coll] {
return s
}
return wholeArchive
}
func (s collScope) and(qs string) collScope {
if s.extra != "" {
qs = s.extra + "&" + qs
}
s.extra = qs
return s
}
func (s collScope) qs() string {
q := ""
if len(s.colls) > 0 {
q = url.Values{"coll": s.colls}.Encode()
}
switch {
case q == "":
return s.extra
case s.extra != "":
return q + "&" + s.extra
}
return q
}
func (s collScope) link(u string) string { return withQuery(u, s.qs()) }
func (s collScope) suffix() string {
if q := s.qs(); q != "" {
return "?" + q
}
return ""
}
func withQuery(link, qs string) string {
switch {
case qs == "" || link == "":
return link
case strings.Contains(link, "?"):
return link + "&" + qs
}
return link + "?" + qs
}
func cloneValues(v url.Values) url.Values {
out := make(url.Values, len(v))
for k, vs := range v {
out[k] = append([]string(nil), vs...)
}
return out
}
func (co *Corpus) handleBrowseRoot(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
collection := q.Get("collection")
if !validCollections[collection] {
collection = collectionNews
}
if collection == collectionArchive {
co.renderArchiveBrowse(w, r, q)
return
}
prefs := readPrefs(r)
selTypes, typeFiltered := selectedTypes(q)
typeFiltered = typeFiltered && collection == collectionNews
selectedTypeSet := map[string]bool{}
if collection == collectionNews {
selectedTypeSet = selTypes
if !typeFiltered {
selectedTypeSet = allTypesSelected(co.archive.NewsTypes)
}
}
docs := append([]*Document(nil), co.collectionDocs[collection]...)
if collection == collectionNews {
docs = filterByType(docs, selectedTypeSet)
}
params := url.Values{}
if collection != collectionNews {
params.Set("collection", collection)
}
if typeFiltered {
params["type"] = q["type"]
}
data := &browsePage{
Base: pageWith(prefs),
listView: co.buildListView(q, prefs, docs, "/browse", params, "title", false),
Collection: collection,
CollectionCounts: co.archive.CollectionCounts,
SIDTodayCount: co.archive.SIDTodayCount,
SelectedTypes: selectedTypeSet,
AllTypes: co.archive.NewsTypes,
Params: q,
}
co.addFacetNav(&data.Base, prefs)
render(w, r, data)
}
// --- SIDtoday archive: chronological view ---
type archiveMonth struct {
Name string
Docs []*Document
}
type archiveYear struct {
Year string
Count int
Months []archiveMonth
}
var monthNames = [...]string{"", "January", "February", "March", "April",
"May", "June", "July", "August", "September", "October", "November",
"December"}
func (co *Corpus) buildArchiveTimeline() {
co.archiveTimeline = nil
docs := append([]*Document(nil), co.sidtodayDocs...)
sort.SliceStable(docs, func(i, j int) bool {
a, b := docs[i].DocMeta.DateCreated, docs[j].DocMeta.DateCreated
if (a == "") != (b == "") {
return b == ""
}
if a != b {
return a < b
}
ta, tb := strings.ToLower(docs[i].Title), strings.ToLower(docs[j].Title)
if ta == tb {
return false
}
return ta < tb
})
for _, d := range docs {
y, m := "Undated", ""
date := d.DocMeta.DateCreated
if len(date) >= 4 {
y = date[:4]
}
if len(date) >= 7 {
if n, err := strconv.Atoi(date[5:7]); err == nil && n >= 1 && n <= 12 {
m = monthNames[n]
}
}
if len(co.archiveTimeline) == 0 || co.archiveTimeline[len(co.archiveTimeline)-1].Year != y {
co.archiveTimeline = append(co.archiveTimeline, archiveYear{Year: y})
}
yr := &co.archiveTimeline[len(co.archiveTimeline)-1]
yr.Count++
if len(yr.Months) == 0 || yr.Months[len(yr.Months)-1].Name != m {
yr.Months = append(yr.Months, archiveMonth{Name: m})
}
mo := &yr.Months[len(yr.Months)-1]
mo.Docs = append(mo.Docs, d)
}
}
func (co *Corpus) renderArchiveBrowse(w http.ResponseWriter, r *http.Request, q url.Values) {
prefs := readPrefs(r)
view := q.Get("view")
data := &browsePage{
Base: pageWith(prefs),
Collection: collectionArchive,
CollectionCounts: co.archive.CollectionCounts,
SIDTodayCount: co.archive.SIDTodayCount,
SeriesList: co.seriesBySize[prefs.IncludeFOIA],
}
co.addFacetNav(&data.Base, prefs)
if view == "card" || view == "table" {
docs := append([]*Document(nil), co.sidtodayDocs...)
params := url.Values{"collection": {collectionArchive}}
data.listView = co.buildListView(q, prefs, docs, "/browse", params, "date", true)
data.MarkReported, data.Cards.MarkReported = true, true
if data.RowsHTML != "" {
data.RowsHTML = co.buildDocRows(data.Docs, true)
}
render(w, r, data)
return
}
data.Total = co.archive.SIDTodayCount
data.View = view
data.ArchiveYears = co.archiveTimeline
data.TimelineHTML = co.preHTML("archive-timeline")
render(w, r, data)
}
func docPath(id string) string { return "/doc/" + url.PathEscape(id) }
func filePath(id, name string) string {
return "/files/" + url.PathEscape(id) + "/" + url.PathEscape(name)
}
func versionPath(id, slug string) string {
return docPath(id) + "/versions/" + url.PathEscape(slug)
}
func versionFilePath(id, slug, name string) string {
return "/files/" + url.PathEscape(id) + "/versions/" + url.PathEscape(slug) + "/" + url.PathEscape(name)
}
func imageBase(id, slug string) string {
if slug == "" {
return "/files/" + url.PathEscape(id) + "/images/"
}
return "/files/" + url.PathEscape(id) + "/versions/" + url.PathEscape(slug) + "/images/"
}
func (co *Corpus) publishedTogether(doc *Document) []*Document {
var out []*Document
seen := map[string]bool{doc.ID: true}
for _, p := range doc.Publication {
if p.Link == "" {
continue
}
a := co.articleByLink[p.Link]
if a == nil {
continue
}
for _, id := range a.DocIDs {
if seen[id] {
continue
}
seen[id] = true
if d := co.docIndex[id]; d != nil {
out = append(out, d)
}
}
}
return out
}
// ogImageAbs is the document's first page image, or its thumbnail when it
// has no page images.
func ogImageAbs(r *http.Request, doc *Document) string {
switch {
case len(doc.Images) > 0:
return requestBase(r) + imageBase(doc.ID, "") + url.PathEscape(doc.Images[0])
case doc.HasThumb:
return requestBase(r) + filePath(doc.ID, "thumb.jpg")
}
return ""
}
// queryBase is path plus q, ready for one more parameter to be appended.
func queryBase(path string, q url.Values) string {
if enc := q.Encode(); enc != "" {
return path + "?" + enc + "&"
}
return path + "?"
}
func (co *Corpus) handleDocPages(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
doc := co.docIndex[id]
if doc == nil {
notFound(w, r)
return
}
prefs := readPrefs(r)
render(w, r, &pagesPage{
Base: pageWith(prefs),
viewer: viewer{Images: doc.Images, Sizes: doc.ImageSizes, ImageBase: imageBase(doc.ID, "")},
Doc: doc,
TextOnly: prefs.TextOnly,
})
}
func (co *Corpus) handleDocView(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
doc := co.docIndex[id]
if doc == nil {
notFound(w, r)
return
}
together := co.publishedTogether(doc)
shownTogether := together
var moreArticles []*Article
if len(together) > pubTogetherCap {
shownTogether = together[:pubTogetherCap]
moreArticles = co.docArticles(doc)
}
prefs := readPrefs(r)
scope := pageScope(nil, prefs, false).including(doc.Collection)
var relatedDocs []*Document
for _, relID := range co.archive.RelatedDocs[id] {
if related := co.docIndex[relID]; related != nil && scope.shows(related) {
relatedDocs = append(relatedDocs, related)
}
}
if prefs.TrackRecent && !crossSiteFetch(r) {
pushRecentDoc(w, r, id)
}
var serialPrevDoc, serialNextDoc *Document
if doc.Type == "SIDTODAY" {
if id, ok := co.serialPrev[doc.ID]; ok {
serialPrevDoc = co.docIndex[id]
}
if id, ok := co.serialNext[doc.ID]; ok {
serialNextDoc = co.docIndex[id]
}
}
embedBase := requestBase(r)
absURL := embedBase + docPath(doc.ID)
pdfURL := embedBase + filePath(doc.ID, doc.PDFFile)
render(w, r, &docPage{
Base: pageWith(prefs),
viewer: viewer{
Images: doc.Images,
Sizes: doc.ImageSizes,
ImageBase: imageBase(doc.ID, ""),
Content: doc.HTMLContent,
},
Doc: doc,
BrowseQS: scope.suffix(),
OGImageURL: ogImageAbs(r, doc),
SeriesGroup: co.seriesIndex[valueKey(doc.DocMeta.Series.Name)],
Assemblies: co.assembliesByDoc[doc.ID],
DocAbsURL: absURL,
Embeds: buildDocEmbeds(doc, absURL, pdfURL),
SerialPrev: serialPrevDoc,
SerialNext: serialNextDoc,
PublishedTogether: together,
PublishedShown: len(shownTogether),
PublishedMore: moreArticles,
RelatedDocs: relatedDocs,
PublishedCards: newCardView(prefs, shownTogether),
RelatedCards: newCardView(prefs, relatedDocs),
Pinned: isPinned(r, id),
})
}
func sortToken(s string) string {
for i := 0; i < len(s); i++ {
if s[i] < 'a' || s[i] > 'z' {
return ""
}
}
return s
}
// firstLetter returns the uppercase A-Z bucket for a facet value, folding
// accents first ("Çağlar" -> "C"), with "#" for anything non-alphabetic.
func firstLetter(s string) string {
s = strings.ToUpper(strings.TrimSpace(foldAccents(s)))
if s != "" && s[0] >= 'A' && s[0] <= 'Z' {
return string(s[0])
}
return "#"
}
// TODO split this huge ass file to separate files, browse_cateogry etc?
// category describes one browsable dimension: its URL slug, the two page
// headings, and the search field code its values map to
type category struct {
Slug string
Title string // list page e.g. "Browse by Programs"
Label string // value page e.g. "Program: XKEYSCORE"
Field string // search field code (fqf), "" for date categories
DateParam string // "date_doc" / "date_pub" for the date categories, else ""
Blurb string
}
func (c *category) isDate() bool { return c.DateParam != "" }
// facet = category's corpus counts (value -> documents)
func (c *category) Facet(co *Corpus) map[string]int { return co.archive.Facets[c.Slug] }
var categories = []category{
{"program", "Programs", "Program", "PR", "",
"Codenames and programs recorded for each document."},
{"journalist", "Journalists", "Journalist", "RE", "",
"Journalists bylined on the stories that published these documents. Contributors credited without a byline are searchable but not listed here."},
{"agency", "Agencies", "Agency", "CR", "",
"The agency that WROTE each document."},
{"publisher", "Publishers", "Publisher", "PU", "",
"Who published each document. Mostly a news outlet, but sometimes a book. A partner outlet that reported the story without publishing the documents keeps its byline and its article, but is not a publisher here."},
{"country", "Countries", "Country", "CO", "",
"Countries mentioned in each document. A country may be mentioned directly (e.g., France) or inferred from a related reference, such as a city (Paris -> France) or a domain name (.fr -> France)."},
{"people", "People", "Person", "PE", "",
"People who appear in these documents, named in the text or recognised in an image. An appearance implies nothing about the person beyond that appearance."},
{"classification", "Classification Levels", "Classification", "CL", "",
"Each document's classification level, as component values. If a document contains multiple classifications, the highest is used here."},
{"dissemination", "Dissemination Controls", "Dissemination control", "DI", "",
"Handling controls assessed for each document, as component values. If a document contains multiple disseminations, the most restrictive is used here."},
{"banner", "Classification Banners", "Banner", "BA", "",
"The marking line each document prints."},
{"genre", "Genres", "Genre", "GE", "",
"What kind of document it is: slides, report, wiki, manual, guide, newsletter."},
{"series", "Document Series", "Series", "SE", "",
"Recurring columns and multi-part runs in SIDtoday documents. A series lists its parts in the order the documents print. A missing part shows as a gap."},
{"published", "Publication Years", "Publication year", "", "date_pub",
"When a publisher first published each document. If a document is published multiple times, only the oldest one is used here."},
{"docdate", "Document Dates", "Document year", "", "date_doc",
"When each document was written, from its own dateline. Undated documents don't appear here."},
{"tag", "Tags", "Tag", "TA", "",
"Free-text subjects the curators added. Unlike the other axes these are not necessarily from the documents' own vocabulary, so they are the loosest thing to browse by (and the most forgiving)."},
}
// pubTogetherCap = on doc view, how many published together docs are shown, max
const pubTogetherCap = 10
func (co *Corpus) docArticles(doc *Document) []*Article {
var out []*Article
for _, p := range doc.Publication {
a := co.articleByLink[p.Link]
if a != nil && len(a.DocIDs) > 1 && !slices.Contains(out, a) {
out = append(out, a)
}
}
return out
}
func categoryValueURL(slug, value string) string {
return "/browse/" + slug + "/" + url.PathEscape(value)
}
func inCollections(d *Document, sel map[string]bool) bool {
return sel == nil || sel[d.Collection]
}
func filterByCollection(docs []*Document, sel map[string]bool) []*Document {
if sel == nil {
return docs
}
out := make([]*Document, 0, len(docs))
for _, d := range docs {
if inCollections(d, sel) {
out = append(out, d)
}
}
return out
}
func categoryBySlug(slug string) *category {
for i := range categories {
if categories[i].Slug == slug {
return &categories[i]
}
}
return nil
}
const azMinEntries = 50
func (co *Corpus) prerenderCategoryRows() {
t := tmplSet["category.html"]
if t == nil {
return
}
exec := func(block string, data interface{}) (template.HTML, bool) {
var buf bytes.Buffer
if err := t.ExecuteTemplate(&buf, block, data); err != nil {
warnf("prerender category %s: %v - rendering rows per request", block, err)
return "", false
}
return template.HTML(buf.String()), true
}
co.categoryRowHTML = make(map[string][]template.HTML, len(co.categoryTables))
co.categoryRowSize = make(map[string]int, len(co.categoryTables))
co.azRowHTML = map[string]template.HTML{}
for ci := range categories {
for _, key := range []string{categories[ci].Slug, noFOIAKey(categories[ci].Slug)} {
entries := co.categoryTables[key].Entries
rows := make([]template.HTML, len(entries))
for i, e := range entries {
h, ok := exec("category-row", e)
if !ok {
return
}
rows[i] = h
co.categoryRowSize[key] += len(h)
if _, seen := co.azRowHTML[e.FirstLetter]; !seen {
if h, ok := exec("category-az-row", e.FirstLetter); ok {
co.azRowHTML[e.FirstLetter] = h
}
}
}
co.categoryRowSize[key] += 26 * 64
co.categoryRowHTML[key] = rows
}
}
}
func (co *Corpus) prerenderDocRows() {
t := tmplSet["browse.html"]
if t == nil {
return
}
rows := make(map[string]template.HTML, len(co.archive.Docs))
marked := map[string]template.HTML{}
var buf bytes.Buffer
render := func(row docTableRow) (template.HTML, bool) {
buf.Reset()
if err := t.ExecuteTemplate(&buf, "doc-table-row", row); err != nil {
warnf("prerender doc-table-row: %v - rendering table rows per request", err)
return "", false
}
return template.HTML(buf.String()), true
}
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
row, ok := render(docTableRow{Document: d})
if !ok {
return
}
rows[d.ID] = row
if d.ReportedIssue() {
if marked[d.ID], ok = render(docTableRow{d, true}); !ok {
return
}
}
}
co.docRowHTML, co.docRowMarkedHTML = rows, marked
}
// buildDocRows joins the documents' pre-rendered table rows; markReported
// takes the rows that badge an individually reported SIDtoday issue.
func (co *Corpus) buildDocRows(docs []*Document, markReported bool) template.HTML {
rows := make([]template.HTML, len(docs))
n := 0
for i, d := range docs {
row, ok := co.docRowMarkedHTML[d.ID]
if !markReported || !ok {
row, ok = co.docRowHTML[d.ID]
}
if !ok {
return ""
}
rows[i] = row
n += len(row)
}
var b strings.Builder
b.Grow(n)
for _, row := range rows {
b.WriteString(string(row))
}
return template.HTML(b.String())
}
func azID(letter string) string {
if letter == "#" {
return "other"
}
return letter
}
func (co *Corpus) buildCategoryRows(key string, t *categoryTable, order []int32, showAZ bool) template.HTML {
rows := co.categoryRowHTML[key]
if len(rows) != len(t.Entries) {
return ""
}
var b strings.Builder
b.Grow(co.categoryRowSize[key])
prevL := ""
for _, n := range order {
if l := t.Entries[n].FirstLetter; showAZ && l != prevL {
b.WriteString(string(co.azRowHTML[l]))
prevL = l
}
b.WriteString(string(rows[n]))
}
return template.HTML(b.String())
}
func (co *Corpus) handleCategoryList(w http.ResponseWriter, r *http.Request) {
cat := r.PathValue("category")
q := r.URL.Query()
sortBy := sortToken(q.Get("sort"))
order := sortToken(q.Get("order"))
c := categoryBySlug(cat)
if c == nil {
notFound(w, r)
return
}
prefs := readPrefs(r)
collCols := cat != "series"
listSort := "name"
switch {
case sortBy == "count":
listSort = sortBy
case !collCols:
case sortBy == "reported", sortBy == "sidtoday":
listSort = sortBy
case sortBy == "foia":
if prefs.IncludeFOIA {
listSort = sortBy
}
}
desc := order == "desc"
if listSort != "name" {
desc = order != "asc"
}
key := readerListKey(cat, prefs)
table := co.categoryTables[key]
rowOrder := table.order(listSort, desc)
showAZ := listSort == "name" && len(rowOrder) >= azMinEntries
var letters []string
if showAZ {
letters = table.letters[desc]
}
collSort := listSort == "reported" || listSort == "sidtoday" || listSort == "foia"
nameCol := "NAME"
if c.isDate() {
nameCol = "YEAR"
}
cols := []sortCol{
{Key: "name", Label: nameCol},
{Key: "count", Label: "TOTAL", Class: "num-col", DescFirst: true},
}
if collCols {
cols = append(cols,
sortCol{Key: "reported", Label: "REPORTED", Class: "num-col coll-col", DescFirst: true},
sortCol{Key: "sidtoday", Label: "SIDTODAY", Class: "num-col coll-col", DescFirst: true})
}
if collCols && prefs.IncludeFOIA {
cols = append(cols, sortCol{Key: "foia", Label: "FOIA", Class: "num-col coll-col", DescFirst: true})
}
data := &categoryPage{
Base: pageWith(prefs),
Title: c.Title,
Blurb: c.Blurb,
Heads: sortHeads("?", cols, listSort, desc),
RowsHTML: co.buildCategoryRows(key, table, rowOrder, showAZ),
CollCols: collCols,
ShowCollCols: collCols && (prefs.ShowCollCols || collSort),
ShowAZ: showAZ,
Letters: letters,
}
data.ActiveCategory = cat
if data.RowsHTML == "" {
data.Entries = table.ordered(rowOrder)
}
co.addFacetNav(&data.Base, prefs)
switch cat {
case "classification", "dissemination":
data.SeeAlso = []seeAlsoLink{{"/browse/banner", "full classification banners (verbatim marking lines)"}}
case "banner":
data.SeeAlso = []seeAlsoLink{
{"/browse/classification", "classification components"},
{"/browse/dissemination", "dissemination controls"},
}
}
render(w, r, data)
}
// categoryEntry = one row of a /browse/{category} list page
type categoryEntry struct {
Name string
URL template.URL // /browse/{category}/{name}, path-escaped
Note string // the agency expansion on the agency list, else ""
Count int // total, all collections
News int // type REPORTED (categoryTypeCounts), any collection
Archive int // type SIDTODAY, incl. individually reported issues
FOIA int // type FOIA
NonFOIA int // in the Reported and SIDtoday collections: the total when FOIA releases are left out
NoFOIA bool // a row of the FOIA-less table: no FOIA column
FirstLetter string
fold string // accent-folded lowercase name: the name sort key
}
// categoryTable = one category's list page
type categoryTable struct {
Entries []categoryEntry // by name, ascending
orders map[string][]int32 // categoryListKey -> indices into Entries, in that order
letters map[bool][]string // the two name orderings' A-Z letters as they come, by desc
}
func (t *categoryTable) order(sortBy string, desc bool) []int32 {
return t.orders[categoryListKey(sortBy, desc)]
}
func (t *categoryTable) ordered(order []int32) []categoryEntry {
out := make([]categoryEntry, len(order))
for i, n := range order {
out[i] = t.Entries[n]
}
return out
}
// noFOIAKey is where a category's FOIA-less table is kept in categoryLists
// and categoryRowHTML.
func noFOIAKey(slug string) string { return slug + "|nofoia" }
// readerListKey is the table a reader's category list comes from: the
// FOIA-less one unless they include FOIA releases.
func readerListKey(slug string, p Prefs) string {
if p.IncludeFOIA {
return slug
}
return noFOIAKey(slug)
}
// newCategoryTable sorts entries by name and works out the orderings the
// sorts ask for, ascending and descending.
func newCategoryTable(entries []categoryEntry, sorts ...string) *categoryTable {
sortCategoryEntries(entries, "name", false)
t := &categoryTable{Entries: entries, orders: map[string][]int32{}, letters: map[bool][]string{}}
for _, sortBy := range sorts {
for _, desc := range []bool{false, true} {
order := make([]int32, len(entries))
for i := range order {
order[i] = int32(i)
}
sort.Slice(order, func(i, j int) bool {
return categoryLess(&entries[order[i]], &entries[order[j]], sortBy, desc)
})
t.orders[categoryListKey(sortBy, desc)] = order
if sortBy != "name" {
continue
}
seen := map[string]bool{}
for _, n := range order {
if l := entries[n].FirstLetter; !seen[l] {
seen[l] = true
t.letters[desc] = append(t.letters[desc], l)
}
}
}
}
return t
}
func categoryListKey(sortBy string, desc bool) string {
if desc {
return sortBy + "/desc"
}
return sortBy + "/asc"
}
func (co *Corpus) buildCategoryLists() {
co.categoryTables = make(map[string]*categoryTable, 2*len(categories))
co.categoryValueCounts = map[bool]map[string]int{false: {}, true: {}}
for ci := range categories {
c := &categories[ci]
byType := co.categoryTypeCounts[c.Slug]
byColl := co.archive.FacetsByColl[c.Slug]
facet := c.Facet(co)
base := make([]categoryEntry, 0, len(facet))
for name, count := range facet {
e := categoryEntry{
Name: name,
URL: template.URL(categoryValueURL(c.Slug, name)),
Count: count,
FirstLetter: firstLetter(name),
fold: normFoldString(name),
}
if c.Slug == "agency" {
e.Note = agencyName(name)
}
if tc := byType[name]; tc != nil {
e.News, e.Archive, e.FOIA = tc.News, tc.Archive, tc.FOIA
}
if cc := byColl[name]; cc != nil {
e.NonFOIA = cc.News + cc.Archive
}
base = append(base, e)
}
noFOIA := make([]categoryEntry, 0, len(base))
for i := range base {
e := base[i]
if e.NonFOIA > 0 {
e.Count = e.NonFOIA
e.NoFOIA = true
noFOIA = append(noFOIA, e)
}
}
co.categoryTables[c.Slug] = newCategoryTable(base, "name", "count", "reported", "sidtoday", "foia")
co.categoryTables[noFOIAKey(c.Slug)] = newCategoryTable(noFOIA, "name", "count", "reported", "sidtoday")
co.categoryValueCounts[true][c.Slug] = len(base)
co.categoryValueCounts[false][c.Slug] = len(noFOIA)
}
co.seriesBySize = map[bool][]categoryEntry{}
for _, foia := range []bool{false, true} {
t := co.categoryTables[readerListKey("series", Prefs{IncludeFOIA: foia})]
co.seriesBySize[foia] = t.ordered(t.order("count", true))
}
}
func sortCategoryEntries(entries []categoryEntry, sortBy string, desc bool) {
sort.Slice(entries, func(i, j int) bool {
return categoryLess(&entries[i], &entries[j], sortBy, desc)
})
}
// categoryLess orders two rows of a category table: by the sorted column in
// the direction asked, then the larger total first, then by name.
func categoryLess(a, b *categoryEntry, sortBy string, desc bool) bool {
col := func(e *categoryEntry) int {
switch sortBy {
case "count":
return e.Count
case "reported":
return e.News
case "sidtoday":
return e.Archive
case "foia":
return e.FOIA
}
return 0
}
byName := sortBy == "name"
if !byName {
if x, y := col(a), col(b); x != y {
if desc {
return x > y
}
return x < y
}
if a.Count != b.Count {
return a.Count > b.Count
}
}
if a.fold != b.fold {
if byName && desc {
return a.fold > b.fold
}
return a.fold < b.fold
}
return a.Name < b.Name
}
type seeAlsoLink struct {
URL string
Text string
}
type timelineBar struct {
Year string
Count int
Pct int
URL string
}
func listSortKey(vals []string) string {
best := ""
for _, v := range vals {
lv := strings.ToLower(strings.TrimSpace(v))
if lv != "" && (best == "" || lv < best) {
best = lv
}
}
return best
}
var classAlias = map[string]string{
"ts": "top secret", "s": "secret", "c": "confidential", "u": "unclassified",
"strap1": "strap 1", "strap2": "strap 2", "talent keyhole": "tk",
}
func foldClass(v string) string {
lv := strings.ToLower(strings.TrimSpace(v))
if a, ok := classAlias[lv]; ok {
return a
}
return lv
}
// classSortKey ranks a marking set per the curators' order: level first
// (TOP SECRET > SECRET > CONFIDENTIAL > UNCLASSIFIED), a level WITH SI
// above the bare level, and the restrictive compartments raising rank
// cumulatively - STRAP 2 (weight 4) > STRAP 1 (2), TK and MR (1 each) -
// so TOP SECRET SI TK outranks TOP SECRET SI.
func classSortKey(vals []string) string {
if len(vals) == 0 {
return ""
}
level, si, raise := 0, 0, 0
for _, v := range vals {
switch foldClass(v) {
case "top secret":
if level < 4 {
level = 4
}
case "secret":
if level < 3 {
level = 3
}
case "confidential":
if level < 2 {
level = 2
}
case "unclassified":
if level < 1 {
level = 1
}
case "si", "comint":
si = 1
case "strap 2":
raise += 4
case "strap 1":
raise += 2
case "tk", "mr":
raise++
}
}
if level == 0 && si == 0 && raise == 0 {
return ""
}
r := strconv.Itoa(raise)
if len(r) < 2 {
r = "0" + r
}
return strconv.Itoa(level) + strconv.Itoa(si) + r
}
// TODO this file really needs the split.
// searchClause is one row of the search form: the field code, the value as
// typed into it and how the row joins the ones before it.
type searchClause struct{ Conj, Field, Value string }
// searchURL builds an executed /search link
func searchURL(clauses []searchClause, ranges ...[2]string) string {
q := url.Values{}
q.Set("do_search", "1")
q.Set("action", "search")
for _, c := range clauses {
conj := c.Conj
if conj == "" {
conj = "and"
}
q.Add("fqc", conj)
q.Add("fqv", c.Value)
q.Add("fqf", c.Field)
}
for _, r := range ranges {
q.Set(r[0]+"_start", r[1])
q.Set(r[0]+"_end", r[1])
}
return "/search?" + q.Encode()
}
func dateRangeSearchURL(fqf, fqv string, ranges ...[2]string) string {
return searchURL([]searchClause{{Field: fqf, Value: fqv}}, ranges...)
}
func appendClause(base, field, value string) string {
return base + "fqf=" + url.QueryEscape(field) +
"&fqv=" + url.QueryEscape(value) + "&fqc=and"
}
// entityTimelineBy is the per-year histogram behind both value-page
// timelines. docs are the page's documents, already narrowed to its
// collection scope, so every bar link carries that scope too.
// yearsOf returns a document's distinct years on the bar axis.
func entityTimelineBy(docs []*Document, c *category, val string, scope collScope, dateParam string, yearsOf func(*Document) []string) []timelineBar {
counts := map[string]int{}
for _, d := range docs {
for _, y := range yearsOf(d) {
counts[y]++
}
}
if len(counts) < 2 {
return nil
}
pageParam := ""
if c.isDate() && validDatePrefix(val) {
pageParam = c.DateParam
}
return yearBars(counts, func(y string) string {
switch {
case c.Field != "":
return scope.link(dateRangeSearchURL(c.Field, exactClause(val), [2]string{dateParam, y}))
case pageParam == "date_doc" && dateParam == "date_pub":
return scope.link(dateRangeSearchURL("TE", "", [2]string{pageParam, val}, [2]string{dateParam, y}))
}
return ""
})
}
func entityTimeline(docs []*Document, c *category, val string, scope collScope) []timelineBar {
return entityTimelineBy(docs, c, val, scope, "date_doc", func(d *Document) []string {
if y := docYear(d); y != "" {
return []string{y}
}
return nil
})
}
func entityPubTimeline(docs []*Document, c *category, val string, scope collScope) []timelineBar {
switch c.Slug {
case "journalist", "publisher":
return entityTimelineBy(docs, c, val, scope, "date_pub", func(d *Document) []string {
return pubYearsMatching(d, c.Field, val)
})
}
return entityTimelineBy(docs, c, val, scope, "date_pub", pubYearsOf)
}
type monthNav struct {
Months []monthNavEntry
Year string
YearURL string
YearActive bool
Monthless int
}
func (co *Corpus) buildMonthNav(c *category, val string, allMatching []*Document, scope collScope) *monthNav {
cat := c.Slug
if !c.isDate() || !validDatePrefix(val) {
return nil
}
year := val[:4]
yearDocs := allMatching
if len(val) == 7 {
yearDocs = scope.docs(co.categoryMatches(cat, year))
}
counts := map[string]int{}
for _, d := range yearDocs {
ds := d.DocMeta.DateCreated
if cat == "published" {
ds = oldestPubDate(d)
}
if len(ds) >= 7 {
counts[ds[5:7]]++
}
}
var months []monthNavEntry
shown := 0
for m := 1; m <= 12; m++ {
mm := strconv.Itoa(m)
if m < 10 {
mm = "0" + mm
}
c := counts[mm]
if c == 0 {
continue
}
shown += c
months = append(months, monthNavEntry{
Label: monthNames[m][:3],
URL: scope.link(categoryValueURL(cat, year+"-"+mm)),
Count: c,
Active: len(val) == 7 && val[5:7] == mm,
})
}
if len(months) == 0 {
return nil
}
return &monthNav{
Months: months,
Year: year,
YearURL: scope.link(categoryValueURL(cat, year)),
YearActive: len(val) == 4,
Monthless: len(yearDocs) - shown,
}
}
// bdGroup = one of a value page's breakdown charts (its banners, its
// classification levels), each value a bar.
type bdGroup struct {
Label string
Values []bdVal
}
type bdVal struct {
Value string
Count int
URL string
Pct int
}
func (co *Corpus) handleCategoryFilter(w http.ResponseWriter, r *http.Request) {
cat := r.PathValue("category")
val := r.PathValue("value")
c := categoryBySlug(cat)
if c == nil {
notFound(w, r)
return
}
if name, ok := co.categoryNames[cat][valueKey(val)]; ok && name != val {
u := categoryValueURL(cat, name)
if r.URL.RawQuery != "" {
u += "?" + r.URL.RawQuery
}
http.Redirect(w, r, u, http.StatusMovedPermanently)
return
}
prefs := readPrefs(r)
q := r.URL.Query()
selectedTypeSet, typeFiltered := selectedTypes(q)
scope := pageScope(q, prefs, false)
everything := co.categoryMatches(cat, val)
allMatching := scope.docs(everything)
pubYear := ""
if y := q.Get("year"); cat == "publisher" && len(y) == 4 && allDigits(y) {
pubYear = y
allMatching = publishedBy(allMatching, val, y)
}
viewTypeCounts := typeCounts(allMatching)
if !typeFiltered {
selectedTypeSet = allTypesSelected(co.archive.Types)
}
results := append([]*Document(nil), allMatching...)
results = filterByType(results, selectedTypeSet)
basePath := categoryValueURL(cat, val)
params := url.Values{}
if typeFiltered {
params["type"] = q["type"]
}
if scope.colls != nil {
params["coll"] = scope.colls
}
if pubYear != "" {
params.Set("year", pubYear)
}
defSort, boost := "title", val
if cat == "series" {
defSort = "series"
}
if cat == "series" || c.isDate() {
boost = ""
}
data := &listPage{
Base: pageWith(prefs),
listView: co.buildListViewBoosted(q, prefs, results, basePath, params, defSort, false, boost),
}
searchLink := ""
if c.Field != "" {
searchLink = searchURL([]searchClause{
{Field: "TE"},
{Field: c.Field, Value: exactClause(val)},
})
} else if c.isDate() && validDatePrefix(val) {
searchLink = dateRangeSearchURL("TE", "", [2]string{c.DateParam, val})
}
docScope := scope
if pubYear != "" {
docScope = scope.and(url.Values{"date_pub_start": {pubYear}, "date_pub_end": {pubYear}}.Encode())
}
searchLink = docScope.link(searchLink)
if mn := co.buildMonthNav(c, val, allMatching, scope); mn != nil {
data.MonthNav = mn.Months
data.MonthNavYear = mn.Year
data.MonthNavYearURL = mn.YearURL
data.MonthNavYearActive = mn.YearActive
if mn.Monthless > 0 {
data.MonthlessCount = mn.Monthless
}
}
var gloss *GlossaryEntry
if cat == "program" {
gloss = co.glossaryLookup(val)
}
data.GlossaryEntry = gloss
if pubYear != "" {
data.YearScope = pubYear
data.YearScopeAllURL = scope.link(basePath)
}
refs := categoryRefinements(allMatching, c, val, docScope, pubYear)
data.Refinements = refs
var breakdowns []bdGroup
for _, g := range refs {
if g.Label != "Banner" && g.Label != "Classification" {
continue
}
top := 0
for _, v := range g.Values {
if v.Count > top {
top = v.Count
}
}
bg := bdGroup{Label: g.Label}
for _, v := range g.Values {
bg.Values = append(bg.Values, bdVal{v.Name, v.Count, v.URL, barPct(v.Count, top, 2)})
}
breakdowns = append(breakdowns, bg)
}
data.Breakdowns = breakdowns
data.Timeline = entityTimeline(allMatching, c, val, docScope)
if pubYear == "" {
// a year-scoped page has one publication year: no timeline
data.PubTimeline = entityPubTimeline(allMatching, c, val, scope)
}
co.addFacetNav(&data.Base, prefs)
data.Title = val
data.ActiveCategory = cat
data.Cards.ActiveCategory, data.Cards.ActiveValue = cat, val
data.SearchLink = searchLink
data.SelectedTypes = selectedTypeSet
data.AllTypes = viewTypeCounts
data.Params = q
data.CatLabel = c.Label
data.Blurb = c.Blurb
render(w, r, data)
}
// categoryRefinements builds the "combine with another filter" facet groups
// for a category value page. For the docs matching this page's value, tally
// the other browsable dimensions and link each value to a TWO-clause search
// (this page's value AND that value). On a publisher page scoped to pubYear
// the links carry that publication year, and the tallies read the
// journalists the way that search will (refineValues).
func categoryRefinements(docs []*Document, c *category, val string, scope collScope, pubYear string) []searchFacet {
if c.Field == "" || len(docs) < 2 {
return nil
}
base := scope.link(searchURL([]searchClause{{Field: c.Field, Value: exactClause(val)}})) + "&"
groups := []facetGroup{
{"Banner", "BA"}, {"Classification", "CL"}, {"Dissemination", "DI"}, {"Genre", "GE"},
{"Programs", "PR"}, {"Agencies", "CR"}, {"Countries", "CO"}, {"Journalists", "RE"},
}
kept := groups[:0]
for _, g := range groups {
if g.Code != c.Field {
kept = append(kept, g)
}
}
return facetGroups(docs, kept, base, refineValues(pubYear, pubYear))
}
func (co *Corpus) buildCategoryDocs() {
co.categoryDocs = make(map[string]map[string][]int32, len(categories))
co.categoryNames = make(map[string]map[string]string, len(categories))
co.archive.Facets = make(map[string]map[string]int, len(categories))
co.archive.FacetsByColl = make(map[string]map[string]*collCounts, len(categories))
co.categoryTypeCounts = make(map[string]map[string]*collCounts, len(categories))
for ci := range categories {
c := &categories[ci]
byValue := make(map[string][]int32)
name := make(map[string]string) // index key -> display value
add := func(key, value string, i int) {
if _, ok := name[key]; !ok {
name[key] = value
}
byValue[key] = append(byValue[key], int32(i))
}
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
if c.isDate() {
date := d.DocMeta.DateCreated
if c.DateParam == "date_pub" {
date = oldestPubDate(d)
}
if len(date) >= 4 {
add(date[:4], date[:4], i)
}
if len(date) >= 7 {
add(date[:7], date[:7], i)
}
continue
}
for _, v := range facetValues(d, c.Field) {
add(valueKey(v), v, i)
}
}
counts := make(map[string]int)
byColl := make(map[string]*collCounts)
byType := make(map[string]*collCounts)
for key, docs := range byValue {
if c.isDate() && len(key) != 4 {
continue
}
cc, tc := &collCounts{}, &collCounts{}
for _, n := range docs {
cc.add(co.archive.Docs[n].Collection)
tc.addType(co.archive.Docs[n].Type)
}
counts[name[key]] = len(docs)
byColl[name[key]] = cc
byType[name[key]] = tc
}
co.categoryDocs[c.Slug] = byValue
co.categoryNames[c.Slug] = name
co.archive.Facets[c.Slug] = counts
co.archive.FacetsByColl[c.Slug] = byColl
co.categoryTypeCounts[c.Slug] = byType
}
}
// categoryMatches returns the documents on a /browse/{category}/{value}
// page, in corpus order, as a fresh slice of pointers the caller owns.
func (co *Corpus) categoryMatches(cat, val string) []*Document {
c := categoryBySlug(cat)
if c == nil {
return nil
}
if c.isDate() && !validDatePrefix(val) {
return nil
}
idx := co.categoryDocs[cat][valueKey(val)]
out := make([]*Document, len(idx))
for i, n := range idx {
out[i] = &co.archive.Docs[n]
}
return out
}
func validDatePrefix(val string) bool {
switch len(val) {
case 4:
return allDigits(val)
case 7:
return allDigits(val[:4]) && val[4] == '-' && allDigits(val[5:])
}
return false
}
func validDate(val string) bool {
if len(val) == 10 {
return validDatePrefix(val[:7]) && val[7] == '-' && allDigits(val[8:])
}
return validDatePrefix(val)
}
func allDigits(s string) bool {
if s == "" {
return false
}
for i := 0; i < len(s); i++ {
if s[i] < '0' || s[i] > '9' {
return false
}
}
return true
}
func publishedBy(docs []*Document, publisher, year string) []*Document {
var out []*Document
for _, d := range docs {
for _, p := range d.Publication {
if !p.ReportingOnly && sameValue(p.Publisher, publisher) &&
strings.HasPrefix(p.DatePublished, year) {
out = append(out, d)
break
}
}
}
return out
}
func publisherYearURL(publisher, year string) string {
return categoryValueURL("publisher", publisher) + "?year=" + url.QueryEscape(year)
}
func oldestPubDate(d *Document) string {
oldest := ""
for _, p := range d.Publication {
if len(p.DatePublished) >= 4 && (oldest == "" || p.DatePublished < oldest) {
oldest = p.DatePublished
}
}
return oldest
}
func (d *Document) FirstPubYear() string {
if date := oldestPubDate(d); len(date) >= 4 {
return date[:4]
}
return ""
}
func docYear(d *Document) string {
if len(d.DocMeta.DateCreated) >= 4 {
return d.DocMeta.DateCreated[:4]
}
return ""
}
// pubYearsOf lists the years the document was published in, any entry's,
// in the entries' order.
func pubYearsOf(d *Document) []string {
if d.fields != nil {
return d.fields.pubYears
}
var out []string
for _, p := range d.Publication {
if len(p.DatePublished) < 4 {
continue
}
if y := p.DatePublished[:4]; !slices.Contains(out, y) {
out = append(out, y)
}
}
return out
}
func pubYearsMatching(d *Document, field, val string) []string {
var out []string
for i := range d.Publication {
date := d.Publication[i].DatePublished
if len(date) < 4 || slices.Contains(out, date[:4]) {
continue
}
for _, v := range entryValues(d, i, field) {
if sameValue(v, val) {
out = append(out, date[:4])
break
}
}
}
return out
}
type monthNavEntry struct {
Label string
URL string
Count int
Active bool
}
// docSort is one way to order a list of documents. The sort bar, the
// doc-table's column headers, the list pages, search and /api/docs all read
// this table, so a sort means the same thing everywhere.
type docSort struct {
Key string
Label string
Value func(d *Document) string
EmptiesLast bool
DescFirst bool
InBar bool // offered in the sort bar
}
// titleSortKey sorts a title by its first letter or digit
func titleSortKey(d *Document) string {
return strings.TrimLeftFunc(strings.ToLower(d.Title), func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
})
}
var docSorts = []docSort{
{"title", "Title", titleSortKey, false, false, true},
{"agency", "Agency", func(d *Document) string { return strings.ToLower(d.DocMeta.Creator) }, false, false, true},
{"date", "Doc Date", func(d *Document) string { return d.DocMeta.DateCreated }, true, false, true},
{"pubdate", "Pub Date", oldestPubDate, true, false, true},
{"classification", "Classification", func(d *Document) string { return classSortKey(d.DocMeta.Classification) }, true, true, false},
{"dissemination", "Dissemination", func(d *Document) string { return listSortKey(d.DocMeta.Dissemination) }, true, false, false},
{"program", "Programs", func(d *Document) string { return listSortKey(d.DocMeta.Programs) }, true, false, false},
{"country", "Countries", func(d *Document) string { return listSortKey(d.DocMeta.Countries) }, true, false, false},
{"people", "People", func(d *Document) string { return listSortKey(d.DocMeta.People) }, true, false, false},
{"journalist", "Journalists", func(d *Document) string { return listSortKey(d.JournalistsDeduped()) }, true, false, false},
{"type", "Type", func(d *Document) string { return strings.ToLower(d.Type) }, false, false, false},
{"series", "Series order", func(d *Document) string {
if p := d.DocMeta.Series.Part; p > 0 {
return fmt.Sprintf("%06d", p)
}
return ""
}, true, false, false},
}
var docSortByKey = func() map[string]*docSort {
m := make(map[string]*docSort, len(docSorts))
for i := range docSorts {
m[docSorts[i].Key] = &docSorts[i]
}
return m
}()
var docTableCols = func() []sortCol {
col := func(key string) sortCol {
s := docSortByKey[key]
return sortCol{Key: s.Key, Label: s.Label, DescFirst: s.DescFirst}
}
return []sortCol{
col("title"), col("agency"), col("date"), col("pubdate"),
col("classification"), col("dissemination"), col("program"),
col("country"), col("people"), col("journalist"),
{Label: "Published"},
col("type"),
}
}()
type sortCol struct {
Key, Label, Class string
DescFirst, Fixed bool
}
type sortHead struct {
sortCol
URL string
Active, Desc bool
}
func sortHeads(base string, cols []sortCol, cur string, desc bool) []sortHead {
heads := make([]sortHead, len(cols))
for i, c := range cols {
h := sortHead{sortCol: c}
if c.Key != "" {
next := c.DescFirst
if h.Active = c.Key == cur; h.Active {
h.Desc = desc
if !c.Fixed {
next = !desc
}
}
h.URL = base + "sort=" + c.Key
if !c.Fixed {
h.URL += "&order=" + orderParam(next)
}
}
heads[i] = h
}
return heads
}
func orderParam(desc bool) string {
if desc {
return "desc"
}
return "asc"
}
// sortCmp orders two sort keys: in the direction asked, an empty key after
// the others either way where the sort puts its empties last.
func sortCmp(a, b string, desc, emptiesLast bool) int {
if emptiesLast && (a == "") != (b == "") {
if b == "" {
return -1
}
return 1
}
if desc {
return cmp.Compare(b, a)
}
return cmp.Compare(a, b)
}
// boostTitleMatches moves the documents whose title has word as a whole word,
// in any case, to the front, keeping each group's order e.g. on
// /browse/tag/Tor the documents about Tor come before the ones that only carry
// the tag.
func boostTitleMatches(docs []*Document, word string) {
re, err := regexp.Compile(`(?i)\b` + regexp.QuoteMeta(word) + `\b`)
if err != nil {
return
}
var hit, rest []*Document
for _, d := range docs {
if re.MatchString(d.Title) {
hit = append(hit, d)
} else {
rest = append(rest, d)
}
}
copy(docs, append(hit, rest...))
}
func sortDocs(docs []*Document, sortBy, order string) {
desc := order == "desc"
if docSortByKey[sortBy] == nil {
sortBy = "title"
}
copy(docs, sortByField(docs, func(d *Document) *Document { return d }, sortBy, desc))
}
func sortScored(scored []ScoredDoc, sortBy, order string) []ScoredDoc {
if docSortByKey[sortBy] == nil {
return scored
}
return sortByField(scored, func(s ScoredDoc) *Document { return s.Doc }, sortBy, order == "desc")
}
func sortByField[T any](items []T, doc func(T) *Document, sortBy string, desc bool) []T {
ds := docSortByKey[sortBy]
keys := make([]string, len(items))
for i := range items {
keys[i] = ds.Value(doc(items[i]))
}
idx := make([]int, len(items))
for i := range idx {
idx[i] = i
}
slices.SortStableFunc(idx, func(a, b int) int {
return sortCmp(keys[a], keys[b], desc, ds.EmptiesLast)
})
out := make([]T, len(items))
for i, o := range idx {
out[i] = items[o]
}
return out
}
func (co *Corpus) docVersion(r *http.Request) (*Document, *DocVersion) {
doc := co.docIndex[r.PathValue("id")]
if doc == nil {
return nil, nil
}
return doc, doc.VersionFor(r.PathValue("slug"))
}
func (co *Corpus) handleVersionView(w http.ResponseWriter, r *http.Request) {
doc, v := co.docVersion(r)
if v == nil {
notFound(w, r)
return
}
pubs := doc.Publication[:0:0]
for _, i := range v.PubIndex {
pubs = append(pubs, doc.Publication[i])
}
prefs := readPrefs(r)
render(w, r, &versionPage{
Base: pageWith(prefs),
viewer: viewer{
Images: v.Images,
Sizes: v.ImageSizes,
ImageBase: imageBase(doc.ID, v.Slug),
Content: v.HTMLContent,
Version: v,
},
Doc: doc,
Publications: pubs,
PDFURL: versionFilePath(doc.ID, v.Slug, v.PDFFile),
})
}
func (co *Corpus) handleVersionPages(w http.ResponseWriter, r *http.Request) {
doc, v := co.docVersion(r)
if v == nil {
notFound(w, r)
return
}
prefs := readPrefs(r)
render(w, r, &pagesPage{
Base: pageWith(prefs),
viewer: viewer{Images: v.Images, Sizes: v.ImageSizes, ImageBase: imageBase(doc.ID, v.Slug), Version: v},
Doc: doc,
TextOnly: prefs.TextOnly,
})
}