package main import ( "bytes" "cmp" "fmt" "html/template" "math/rand" "net/http" "net/url" "regexp" "slices" "sort" "strconv" "strings" "unicode" "unicode/utf8" ) type facetEntry struct { Name string Count int URL string } // facetLinks renders a metadata field's values as links to their category // pages, wherever a card, a table row, a search result or a document page // lists them. It takes the template's dict: // // Vals the values ("-" stands for none) // Cat the category's slug // Sep between the links: "dot", "space", or a comma when not given // AsList one
  • per link instead // Class the links' class // Active the value of the page the links are on, when it is this // category's: its own link is marked active // QS the collection scope the links carry ("?coll=..."), when the page // they open would otherwise leave out the document they come from // // The output is what html/template made of the same markup: every value // goes through htmlEscape. func facetLinks(p map[string]interface{}) (template.HTML, error) { vals, ok := p["Vals"].([]string) if !ok && p["Vals"] != nil { return "", fmt.Errorf("facetLinks: Vals is %T, not a list of values", p["Vals"]) } if len(vals) == 0 { return "-", nil } cat, _ := p["Cat"].(string) sep, _ := p["Sep"].(string) class, _ := p["Class"].(string) active, _ := p["Active"].(string) qs, _ := p["QS"].(string) asList, _ := p["AsList"].(bool) var b strings.Builder if asList { b.WriteString(`") } return template.HTML(b.String()), nil } func htmlEscape(s string) string { if !strings.ContainsAny(s, "\x00\"&'+<>") && isASCII(s) { return s } var b strings.Builder b.Grow(len(s) + 16) for i := 0; i < len(s); { r, w := utf8.DecodeRuneInString(s[i:]) switch { case r == 0: b.WriteString("\uFFFD") case r == '"': b.WriteString(""") case r == '&': b.WriteString("&") case r == '\'': b.WriteString("'") case r == '+': b.WriteString("+") case r == '<': b.WriteString("<") case r == '>': b.WriteString(">") case r >= 0xfdd0 && r <= 0xfdef, r >= 0xfff0 && r <= 0xffff: fmt.Fprintf(&b, "&#x%x;", r) default: b.WriteString(s[i : i+w]) } i += w } return b.String() } // rankCounts returns the keys of a count map in rank order (most counted // first, ties alphabetically ignoring accents and case, then by the raw // spelling) func rankCounts(m map[string]int, n int) []string { type keyed struct { name, fold string count int } all := make([]keyed, 0, len(m)) for k, v := range m { all = append(all, keyed{k, normFoldString(k), v}) } sort.Slice(all, func(i, j int) bool { a, b := &all[i], &all[j] if a.count != b.count { return a.count > b.count } if a.fold != b.fold { return a.fold < b.fold } return a.name < b.name }) if n >= 0 && len(all) > n { all = all[:n] } out := make([]string, len(all)) for i := range all { out[i] = all[i].name } return out } func topFacets(m map[string]int, n int) []facetEntry { names := rankCounts(m, n) out := make([]facetEntry, len(names)) for i, k := range names { out[i] = facetEntry{Name: k, Count: m[k]} } return out } func barPct(count, denom, floor int) int { if denom <= 0 || count <= 0 { return 0 } pct := count * 100 / denom if pct < floor { pct = floor } return pct } func sortedYears(m map[string]int) (years []string, maxCount int) { years = make([]string, 0, len(m)) for y, c := range m { years = append(years, y) if c > maxCount { maxCount = c } } sort.Strings(years) return years, maxCount } func yearBars(m map[string]int, urlFor func(y string) string) []timelineBar { years, maxCount := sortedYears(m) if maxCount == 0 { return nil } bars := make([]timelineBar, 0, len(years)) for _, y := range years { bars = append(bars, timelineBar{ Year: y, Count: m[y], Pct: barPct(m[y], maxCount, 1), URL: urlFor(y), }) } return bars } type homeFigures struct { TopAgencies, TopPrograms, TopCountries, TopPeople, TopJournalists, TopPublishers, TopTags []facetEntry DocHist, PubHist []timelineBar } func (co *Corpus) buildHomeStats() { co.homeStats = co.homeFigures(false) co.homeStatsSIDtoday = co.homeFigures(true) co.buildRandomPools() } func (co *Corpus) homeFigures(sidtoday bool) homeFigures { scope := scopeOf(map[string]bool{collectionNews: true, collectionArchive: sidtoday}) top := func(slug string, n int) []facetEntry { entries := topFacets(co.homeFacet(slug, sidtoday), n) for i := range entries { entries[i].URL = scope.link(categoryValueURL(slug, entries[i].Name)) } return entries } f := homeFigures{ TopAgencies: top("agency", 10), TopPrograms: top("program", 20), TopCountries: top("country", 15), TopPeople: top("people", 15), TopJournalists: top("journalist", 12), TopPublishers: top("publisher", 10), TopTags: top("tag", 25), } f.DocHist, f.PubHist = co.homeHistograms(scope) return f } func (co *Corpus) homeFacet(slug string, sidtoday bool) map[string]int { out := make(map[string]int) for value, c := range co.archive.FacetsByColl[slug] { n := c.News if sidtoday { n += c.Archive } if n > 0 { out[value] = n } } return out } func (co *Corpus) homeHistograms(scope collScope) (docHist, pubHist []timelineBar) { docYears := map[string]int{} pubYears := map[string]int{} for i := range co.archive.Docs { d := &co.archive.Docs[i] if !scope.shows(d) { continue } if y := docYear(d); y != "" { docYears[y]++ } if y := d.FirstPubYear(); y != "" { pubYears[y]++ } } link := func(slug string) func(string) string { return func(y string) string { return scope.link(categoryValueURL(slug, y)) } } return yearBars(docYears, link("docdate")), yearBars(pubYears, link("published")) } func (co *Corpus) addFacetNav(b *Base, prefs Prefs) { b.ShowCatBoxes = true b.CatCounts = co.categoryValueCounts[prefs.IncludeFOIA] } func (co *Corpus) handleIndex(w http.ResponseWriter, r *http.Request) { prefs := readPrefs(r) var recentDocs []*Document if prefs.TrackRecent { for _, id := range readIDListCookie(r, recentDocsCookie, recentDocsMax) { if d := co.docIndex[id]; d != nil { recentDocs = append(recentDocs, d) } } } if len(recentDocs) > 0 { noStore(w) } withSIDtoday := r.URL.Query().Get("sidtoday") == "1" stats := co.homeStats if withSIDtoday { stats = co.homeStatsSIDtoday } render(w, r, &indexPage{ Base: pageWith(prefs), homeFigures: stats, TrackRecent: prefs.TrackRecent, RecentDocs: recentDocs, SnowdenTotal: co.archive.CollectionCounts[collectionNews] + co.archive.CollectionCounts[collectionArchive], ArchiveTotal: co.archive.CollectionCounts[collectionArchive], FOIATotal: co.archive.CollectionCounts[collectionFOIA], IncludeFOIA: prefs.IncludeFOIA, ReportedTotal: co.archive.CollectionCounts[collectionNews], WithSIDtoday: withSIDtoday, }) } // randomWeights says how often /random lands in each collection var randomWeights = []struct { coll string weight int }{ {collectionNews, 65}, {collectionArchive, 25}, {collectionFOIA, 10}, } func (co *Corpus) buildRandomPools() { co.randomPools = map[string][]int{} for i, d := range co.archive.Docs { co.randomPools[d.Collection] = append(co.randomPools[d.Collection], i) } } func (co *Corpus) buildCollectionDocs() { co.everyDoc = make([]*Document, len(co.archive.Docs)) co.collectionDocs = make(map[string][]*Document, len(validCollections)) co.sidtodayDocs = nil for i := range co.archive.Docs { d := &co.archive.Docs[i] co.everyDoc[i] = d co.collectionDocs[d.Collection] = append(co.collectionDocs[d.Collection], d) if d.Type == "SIDTODAY" { co.sidtodayDocs = append(co.sidtodayDocs, d) } } } func (co *Corpus) randomDoc(includeFOIA bool) *Document { pool := func(coll string) []int { if coll == collectionFOIA && !includeFOIA { return nil } return co.randomPools[coll] } total := 0 for _, rw := range randomWeights { if len(pool(rw.coll)) > 0 { total += rw.weight } } if total == 0 { return nil } n := rand.Intn(total) for _, rw := range randomWeights { p := pool(rw.coll) if len(p) == 0 { continue } if n < rw.weight { return &co.archive.Docs[p[rand.Intn(len(p))]] } n -= rw.weight } return nil } func (co *Corpus) handleRandom(w http.ResponseWriter, r *http.Request) { d := co.randomDoc(readPrefs(r).IncludeFOIA) if d == nil { notFound(w, r) return } http.Redirect(w, r, docPath(d.ID), http.StatusFound) } func handleAbout(w http.ResponseWriter, r *http.Request) { render(w, r, &aboutPage{}) } func paginate(total, page, perPage int) (start, end, clampedPage, totalPages int) { if perPage <= 0 { return 0, total, 1, 1 } totalPages = (total + perPage - 1) / perPage if page < 1 { page = 1 } maxPage := totalPages if maxPage < 1 { maxPage = 1 } if page > maxPage { page = maxPage } start = (page - 1) * perPage if start < 0 || start > total { start = total } end = start + perPage if end < start || end > total { end = total } return start, end, page, totalPages } // buildListView assembles everything the shared doc-cards / doc-table // layouts need: view, sort, pagination, the link bases their partials // build on, and the card display prefs. // // The page embeds what is returned and adds its own fields, including the // cards' ActiveCategory and ActiveValue when it has a filter value for them // to highlight. func (co *Corpus) buildListView(q url.Values, prefs Prefs, docs []*Document, basePath string, params url.Values, defaultSort string, explicitView bool) listView { return co.buildListViewBoosted(q, prefs, docs, basePath, params, defaultSort, explicitView, "") } func (co *Corpus) buildListViewBoosted(q url.Values, prefs Prefs, docs []*Document, basePath string, params url.Values, defaultSort string, explicitView bool, boost string) listView { view := q.Get("view") sortBy := sortToken(q.Get("sort")) order := sortToken(q.Get("order")) if docSortByKey[sortBy] == nil { sortBy = "" } effSort := cmp.Or(sortBy, defaultSort) sortDocs(docs, effSort, order) if sortBy == "" && boost != "" { boostTitleMatches(docs, boost) } page, _ := strconv.Atoi(q.Get("p")) finalDocs := docs totalPages := 1 if view == "table" { page = 1 } else { var start, end int start, end, page, totalPages = paginate(len(docs), page, prefs.cardsPerPage()) finalDocs = docs[start:end] } baseQ := cloneValues(params) sortQ := cloneValues(params) if view != "" && (explicitView || view == "table") { baseQ.Set("view", view) } lv := listView{ Docs: finalDocs, Total: len(docs), Page: page, TotalPages: totalPages, View: view, Sort: sortBy, Order: order, DefaultSort: defaultSort, BaseURL: queryBase(basePath, baseQ), SortURL: queryBase(basePath, sortQ), TableHeads: sortHeads(queryBase(basePath, baseQ), docTableCols, effSort, order == "desc"), Cards: newCardView(prefs, finalDocs), } if view == "table" { lv.RowsHTML = co.buildDocRows(finalDocs, false) } return lv } // selectedTypes reads the ?type= params into a set, and reports whether the // request actually filters (no params means everything, not nothing). func selectedTypes(v url.Values) (map[string]bool, bool) { sel := make(map[string]bool, len(v["type"])) for _, t := range v["type"] { if t = strings.ToUpper(strings.TrimSpace(t)); t != "" { sel[t] = true } } return sel, len(sel) > 0 } func typeSelected(d *Document, sel map[string]bool) bool { return len(sel) == 0 || sel[d.Type] } func typeCounts(docs []*Document) map[string]int { counts := make(map[string]int) for i := range docs { counts[docs[i].Type]++ } return counts } func filterByType(docs []*Document, sel map[string]bool) []*Document { if len(sel) == 0 { return docs } kept := docs[:0] for _, d := range docs { if typeSelected(d, sel) { kept = append(kept, d) } } return kept } func allTypesSelected(counts map[string]int) map[string]bool { sel := make(map[string]bool, len(counts)) for t := range counts { sel[t] = true } return sel } var validCollections = map[string]bool{ collectionNews: true, collectionArchive: true, collectionFOIA: true, } var collectionOrder = []string{collectionNews, collectionArchive, collectionFOIA} func collectionParams(sel map[string]bool) []string { var out []string for _, c := range collectionOrder { if sel[c] { out = append(out, c) } } return out } // defaultCollections is what a page shows when its URL names no // collection: FOIA releases are opt-in (Prefs.IncludeFOIA). var defaultCollections = map[string]bool{collectionNews: true, collectionArchive: true} // collScope is a page's collection scope: the documents it shows and what // its own links carry. Every link a page builds from its documents goes // through link, so the number on a link is the number its click returns. type collScope struct { filter map[string]bool // the documents shown; nil: every collection shown map[string]bool // the same as a set, never nil (the form's boxes) defaulted bool // the URL named no collection: the preference decided colls []string // the coll= the page's links carry; nil: none extra string // more scope for the links (a publisher page's year) } // scopeOf is an explicit scope: sel, with links that name it, so a count // holds whatever the next reader's FOIA preference. func scopeOf(sel map[string]bool) collScope { s := collScope{shown: sel, colls: collectionParams(sel)} if len(s.colls) < len(validCollections) { s.filter = sel } return s } // wholeArchive scopes links to all three collections: a count taken over // the whole archive must not change with the reader's FOIA preference. var wholeArchive = scopeOf(validCollections) // pageScope reads a page's scope: the coll= its URL names or, when it // names none, the reader's preference, FOIA releases only when opted in. func pageScope(v url.Values, p Prefs, pinDefault bool) collScope { if _, given := v["coll"]; given { if f := collectionFilter(v); f != nil { return scopeOf(f) } return wholeArchive } s := scopeOf(defaultCollections) if p.IncludeFOIA { s = wholeArchive } s.defaulted = true if !pinDefault { s.colls = nil } return s } func (s collScope) shows(d *Document) bool { return inCollections(d, s.filter) } func (s collScope) hasValue(co *Corpus, slug, value string) bool { c := co.archive.FacetsByColl[slug][co.categoryNames[slug][valueKey(value)]] if c == nil { return false } in := func(coll string, n int) bool { return n > 0 && (s.filter == nil || s.filter[coll]) } return in(collectionNews, c.News) || in(collectionArchive, c.Archive) || in(collectionFOIA, c.FOIA) } func (s collScope) docs(docs []*Document) []*Document { return filterByCollection(docs, s.filter) } func (s collScope) explicit() bool { return s.filter != nil && !s.defaulted } func (s collScope) including(coll string) collScope { if s.filter == nil || s.filter[coll] { return s } return wholeArchive } func (s collScope) and(qs string) collScope { if s.extra != "" { qs = s.extra + "&" + qs } s.extra = qs return s } func (s collScope) qs() string { q := "" if len(s.colls) > 0 { q = url.Values{"coll": s.colls}.Encode() } switch { case q == "": return s.extra case s.extra != "": return q + "&" + s.extra } return q } func (s collScope) link(u string) string { return withQuery(u, s.qs()) } func (s collScope) suffix() string { if q := s.qs(); q != "" { return "?" + q } return "" } func withQuery(link, qs string) string { switch { case qs == "" || link == "": return link case strings.Contains(link, "?"): return link + "&" + qs } return link + "?" + qs } func cloneValues(v url.Values) url.Values { out := make(url.Values, len(v)) for k, vs := range v { out[k] = append([]string(nil), vs...) } return out } func (co *Corpus) handleBrowseRoot(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() collection := q.Get("collection") if !validCollections[collection] { collection = collectionNews } if collection == collectionArchive { co.renderArchiveBrowse(w, r, q) return } prefs := readPrefs(r) selTypes, typeFiltered := selectedTypes(q) typeFiltered = typeFiltered && collection == collectionNews selectedTypeSet := map[string]bool{} if collection == collectionNews { selectedTypeSet = selTypes if !typeFiltered { selectedTypeSet = allTypesSelected(co.archive.NewsTypes) } } docs := append([]*Document(nil), co.collectionDocs[collection]...) if collection == collectionNews { docs = filterByType(docs, selectedTypeSet) } params := url.Values{} if collection != collectionNews { params.Set("collection", collection) } if typeFiltered { params["type"] = q["type"] } data := &browsePage{ Base: pageWith(prefs), listView: co.buildListView(q, prefs, docs, "/browse", params, "title", false), Collection: collection, CollectionCounts: co.archive.CollectionCounts, SIDTodayCount: co.archive.SIDTodayCount, SelectedTypes: selectedTypeSet, AllTypes: co.archive.NewsTypes, Params: q, } co.addFacetNav(&data.Base, prefs) render(w, r, data) } // --- SIDtoday archive: chronological view --- type archiveMonth struct { Name string Docs []*Document } type archiveYear struct { Year string Count int Months []archiveMonth } var monthNames = [...]string{"", "January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"} func (co *Corpus) buildArchiveTimeline() { co.archiveTimeline = nil docs := append([]*Document(nil), co.sidtodayDocs...) sort.SliceStable(docs, func(i, j int) bool { a, b := docs[i].DocMeta.DateCreated, docs[j].DocMeta.DateCreated if (a == "") != (b == "") { return b == "" } if a != b { return a < b } ta, tb := strings.ToLower(docs[i].Title), strings.ToLower(docs[j].Title) if ta == tb { return false } return ta < tb }) for _, d := range docs { y, m := "Undated", "" date := d.DocMeta.DateCreated if len(date) >= 4 { y = date[:4] } if len(date) >= 7 { if n, err := strconv.Atoi(date[5:7]); err == nil && n >= 1 && n <= 12 { m = monthNames[n] } } if len(co.archiveTimeline) == 0 || co.archiveTimeline[len(co.archiveTimeline)-1].Year != y { co.archiveTimeline = append(co.archiveTimeline, archiveYear{Year: y}) } yr := &co.archiveTimeline[len(co.archiveTimeline)-1] yr.Count++ if len(yr.Months) == 0 || yr.Months[len(yr.Months)-1].Name != m { yr.Months = append(yr.Months, archiveMonth{Name: m}) } mo := &yr.Months[len(yr.Months)-1] mo.Docs = append(mo.Docs, d) } } func (co *Corpus) renderArchiveBrowse(w http.ResponseWriter, r *http.Request, q url.Values) { prefs := readPrefs(r) view := q.Get("view") data := &browsePage{ Base: pageWith(prefs), Collection: collectionArchive, CollectionCounts: co.archive.CollectionCounts, SIDTodayCount: co.archive.SIDTodayCount, SeriesList: co.seriesBySize[prefs.IncludeFOIA], } co.addFacetNav(&data.Base, prefs) if view == "card" || view == "table" { docs := append([]*Document(nil), co.sidtodayDocs...) params := url.Values{"collection": {collectionArchive}} data.listView = co.buildListView(q, prefs, docs, "/browse", params, "date", true) data.MarkReported, data.Cards.MarkReported = true, true if data.RowsHTML != "" { data.RowsHTML = co.buildDocRows(data.Docs, true) } render(w, r, data) return } data.Total = co.archive.SIDTodayCount data.View = view data.ArchiveYears = co.archiveTimeline data.TimelineHTML = co.preHTML("archive-timeline") render(w, r, data) } func docPath(id string) string { return "/doc/" + url.PathEscape(id) } func filePath(id, name string) string { return "/files/" + url.PathEscape(id) + "/" + url.PathEscape(name) } func versionPath(id, slug string) string { return docPath(id) + "/versions/" + url.PathEscape(slug) } func versionFilePath(id, slug, name string) string { return "/files/" + url.PathEscape(id) + "/versions/" + url.PathEscape(slug) + "/" + url.PathEscape(name) } func imageBase(id, slug string) string { if slug == "" { return "/files/" + url.PathEscape(id) + "/images/" } return "/files/" + url.PathEscape(id) + "/versions/" + url.PathEscape(slug) + "/images/" } func (co *Corpus) publishedTogether(doc *Document) []*Document { var out []*Document seen := map[string]bool{doc.ID: true} for _, p := range doc.Publication { if p.Link == "" { continue } a := co.articleByLink[p.Link] if a == nil { continue } for _, id := range a.DocIDs { if seen[id] { continue } seen[id] = true if d := co.docIndex[id]; d != nil { out = append(out, d) } } } return out } // ogImageAbs is the document's first page image, or its thumbnail when it // has no page images. func ogImageAbs(r *http.Request, doc *Document) string { switch { case len(doc.Images) > 0: return requestBase(r) + imageBase(doc.ID, "") + url.PathEscape(doc.Images[0]) case doc.HasThumb: return requestBase(r) + filePath(doc.ID, "thumb.jpg") } return "" } // queryBase is path plus q, ready for one more parameter to be appended. func queryBase(path string, q url.Values) string { if enc := q.Encode(); enc != "" { return path + "?" + enc + "&" } return path + "?" } func (co *Corpus) handleDocPages(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") doc := co.docIndex[id] if doc == nil { notFound(w, r) return } prefs := readPrefs(r) render(w, r, &pagesPage{ Base: pageWith(prefs), viewer: viewer{Images: doc.Images, Sizes: doc.ImageSizes, ImageBase: imageBase(doc.ID, "")}, Doc: doc, TextOnly: prefs.TextOnly, }) } func (co *Corpus) handleDocView(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") doc := co.docIndex[id] if doc == nil { notFound(w, r) return } together := co.publishedTogether(doc) shownTogether := together var moreArticles []*Article if len(together) > pubTogetherCap { shownTogether = together[:pubTogetherCap] moreArticles = co.docArticles(doc) } prefs := readPrefs(r) scope := pageScope(nil, prefs, false).including(doc.Collection) var relatedDocs []*Document for _, relID := range co.archive.RelatedDocs[id] { if related := co.docIndex[relID]; related != nil && scope.shows(related) { relatedDocs = append(relatedDocs, related) } } if prefs.TrackRecent && !crossSiteFetch(r) { pushRecentDoc(w, r, id) } var serialPrevDoc, serialNextDoc *Document if doc.Type == "SIDTODAY" { if id, ok := co.serialPrev[doc.ID]; ok { serialPrevDoc = co.docIndex[id] } if id, ok := co.serialNext[doc.ID]; ok { serialNextDoc = co.docIndex[id] } } embedBase := requestBase(r) absURL := embedBase + docPath(doc.ID) pdfURL := embedBase + filePath(doc.ID, doc.PDFFile) render(w, r, &docPage{ Base: pageWith(prefs), viewer: viewer{ Images: doc.Images, Sizes: doc.ImageSizes, ImageBase: imageBase(doc.ID, ""), Content: doc.HTMLContent, }, Doc: doc, BrowseQS: scope.suffix(), OGImageURL: ogImageAbs(r, doc), SeriesGroup: co.seriesIndex[valueKey(doc.DocMeta.Series.Name)], Assemblies: co.assembliesByDoc[doc.ID], DocAbsURL: absURL, Embeds: buildDocEmbeds(doc, absURL, pdfURL), SerialPrev: serialPrevDoc, SerialNext: serialNextDoc, PublishedTogether: together, PublishedShown: len(shownTogether), PublishedMore: moreArticles, RelatedDocs: relatedDocs, PublishedCards: newCardView(prefs, shownTogether), RelatedCards: newCardView(prefs, relatedDocs), Pinned: isPinned(r, id), }) } func sortToken(s string) string { for i := 0; i < len(s); i++ { if s[i] < 'a' || s[i] > 'z' { return "" } } return s } // firstLetter returns the uppercase A-Z bucket for a facet value, folding // accents first ("Çağlar" -> "C"), with "#" for anything non-alphabetic. func firstLetter(s string) string { s = strings.ToUpper(strings.TrimSpace(foldAccents(s))) if s != "" && s[0] >= 'A' && s[0] <= 'Z' { return string(s[0]) } return "#" } // TODO split this huge ass file to separate files, browse_cateogry etc? // category describes one browsable dimension: its URL slug, the two page // headings, and the search field code its values map to type category struct { Slug string Title string // list page e.g. "Browse by Programs" Label string // value page e.g. "Program: XKEYSCORE" Field string // search field code (fqf), "" for date categories DateParam string // "date_doc" / "date_pub" for the date categories, else "" Blurb string } func (c *category) isDate() bool { return c.DateParam != "" } // facet = category's corpus counts (value -> documents) func (c *category) Facet(co *Corpus) map[string]int { return co.archive.Facets[c.Slug] } var categories = []category{ {"program", "Programs", "Program", "PR", "", "Codenames and programs recorded for each document."}, {"journalist", "Journalists", "Journalist", "RE", "", "Journalists bylined on the stories that published these documents. Contributors credited without a byline are searchable but not listed here."}, {"agency", "Agencies", "Agency", "CR", "", "The agency that WROTE each document."}, {"publisher", "Publishers", "Publisher", "PU", "", "Who published each document. Mostly a news outlet, but sometimes a book. A partner outlet that reported the story without publishing the documents keeps its byline and its article, but is not a publisher here."}, {"country", "Countries", "Country", "CO", "", "Countries mentioned in each document. A country may be mentioned directly (e.g., France) or inferred from a related reference, such as a city (Paris -> France) or a domain name (.fr -> France)."}, {"people", "People", "Person", "PE", "", "People who appear in these documents, named in the text or recognised in an image. An appearance implies nothing about the person beyond that appearance."}, {"classification", "Classification Levels", "Classification", "CL", "", "Each document's classification level, as component values. If a document contains multiple classifications, the highest is used here."}, {"dissemination", "Dissemination Controls", "Dissemination control", "DI", "", "Handling controls assessed for each document, as component values. If a document contains multiple disseminations, the most restrictive is used here."}, {"banner", "Classification Banners", "Banner", "BA", "", "The marking line each document prints."}, {"genre", "Genres", "Genre", "GE", "", "What kind of document it is: slides, report, wiki, manual, guide, newsletter."}, {"series", "Document Series", "Series", "SE", "", "Recurring columns and multi-part runs in SIDtoday documents. A series lists its parts in the order the documents print. A missing part shows as a gap."}, {"published", "Publication Years", "Publication year", "", "date_pub", "When a publisher first published each document. If a document is published multiple times, only the oldest one is used here."}, {"docdate", "Document Dates", "Document year", "", "date_doc", "When each document was written, from its own dateline. Undated documents don't appear here."}, {"tag", "Tags", "Tag", "TA", "", "Free-text subjects the curators added. Unlike the other axes these are not necessarily from the documents' own vocabulary, so they are the loosest thing to browse by (and the most forgiving)."}, } // pubTogetherCap = on doc view, how many published together docs are shown, max const pubTogetherCap = 10 func (co *Corpus) docArticles(doc *Document) []*Article { var out []*Article for _, p := range doc.Publication { a := co.articleByLink[p.Link] if a != nil && len(a.DocIDs) > 1 && !slices.Contains(out, a) { out = append(out, a) } } return out } func categoryValueURL(slug, value string) string { return "/browse/" + slug + "/" + url.PathEscape(value) } func inCollections(d *Document, sel map[string]bool) bool { return sel == nil || sel[d.Collection] } func filterByCollection(docs []*Document, sel map[string]bool) []*Document { if sel == nil { return docs } out := make([]*Document, 0, len(docs)) for _, d := range docs { if inCollections(d, sel) { out = append(out, d) } } return out } func categoryBySlug(slug string) *category { for i := range categories { if categories[i].Slug == slug { return &categories[i] } } return nil } const azMinEntries = 50 func (co *Corpus) prerenderCategoryRows() { t := tmplSet["category.html"] if t == nil { return } exec := func(block string, data interface{}) (template.HTML, bool) { var buf bytes.Buffer if err := t.ExecuteTemplate(&buf, block, data); err != nil { warnf("prerender category %s: %v - rendering rows per request", block, err) return "", false } return template.HTML(buf.String()), true } co.categoryRowHTML = make(map[string][]template.HTML, len(co.categoryTables)) co.categoryRowSize = make(map[string]int, len(co.categoryTables)) co.azRowHTML = map[string]template.HTML{} for ci := range categories { for _, key := range []string{categories[ci].Slug, noFOIAKey(categories[ci].Slug)} { entries := co.categoryTables[key].Entries rows := make([]template.HTML, len(entries)) for i, e := range entries { h, ok := exec("category-row", e) if !ok { return } rows[i] = h co.categoryRowSize[key] += len(h) if _, seen := co.azRowHTML[e.FirstLetter]; !seen { if h, ok := exec("category-az-row", e.FirstLetter); ok { co.azRowHTML[e.FirstLetter] = h } } } co.categoryRowSize[key] += 26 * 64 co.categoryRowHTML[key] = rows } } } func (co *Corpus) prerenderDocRows() { t := tmplSet["browse.html"] if t == nil { return } rows := make(map[string]template.HTML, len(co.archive.Docs)) marked := map[string]template.HTML{} var buf bytes.Buffer render := func(row docTableRow) (template.HTML, bool) { buf.Reset() if err := t.ExecuteTemplate(&buf, "doc-table-row", row); err != nil { warnf("prerender doc-table-row: %v - rendering table rows per request", err) return "", false } return template.HTML(buf.String()), true } for i := range co.archive.Docs { d := &co.archive.Docs[i] row, ok := render(docTableRow{Document: d}) if !ok { return } rows[d.ID] = row if d.ReportedIssue() { if marked[d.ID], ok = render(docTableRow{d, true}); !ok { return } } } co.docRowHTML, co.docRowMarkedHTML = rows, marked } // buildDocRows joins the documents' pre-rendered table rows; markReported // takes the rows that badge an individually reported SIDtoday issue. func (co *Corpus) buildDocRows(docs []*Document, markReported bool) template.HTML { rows := make([]template.HTML, len(docs)) n := 0 for i, d := range docs { row, ok := co.docRowMarkedHTML[d.ID] if !markReported || !ok { row, ok = co.docRowHTML[d.ID] } if !ok { return "" } rows[i] = row n += len(row) } var b strings.Builder b.Grow(n) for _, row := range rows { b.WriteString(string(row)) } return template.HTML(b.String()) } func azID(letter string) string { if letter == "#" { return "other" } return letter } func (co *Corpus) buildCategoryRows(key string, t *categoryTable, order []int32, showAZ bool) template.HTML { rows := co.categoryRowHTML[key] if len(rows) != len(t.Entries) { return "" } var b strings.Builder b.Grow(co.categoryRowSize[key]) prevL := "" for _, n := range order { if l := t.Entries[n].FirstLetter; showAZ && l != prevL { b.WriteString(string(co.azRowHTML[l])) prevL = l } b.WriteString(string(rows[n])) } return template.HTML(b.String()) } func (co *Corpus) handleCategoryList(w http.ResponseWriter, r *http.Request) { cat := r.PathValue("category") q := r.URL.Query() sortBy := sortToken(q.Get("sort")) order := sortToken(q.Get("order")) c := categoryBySlug(cat) if c == nil { notFound(w, r) return } prefs := readPrefs(r) collCols := cat != "series" listSort := "name" switch { case sortBy == "count": listSort = sortBy case !collCols: case sortBy == "reported", sortBy == "sidtoday": listSort = sortBy case sortBy == "foia": if prefs.IncludeFOIA { listSort = sortBy } } desc := order == "desc" if listSort != "name" { desc = order != "asc" } key := readerListKey(cat, prefs) table := co.categoryTables[key] rowOrder := table.order(listSort, desc) showAZ := listSort == "name" && len(rowOrder) >= azMinEntries var letters []string if showAZ { letters = table.letters[desc] } collSort := listSort == "reported" || listSort == "sidtoday" || listSort == "foia" nameCol := "NAME" if c.isDate() { nameCol = "YEAR" } cols := []sortCol{ {Key: "name", Label: nameCol}, {Key: "count", Label: "TOTAL", Class: "num-col", DescFirst: true}, } if collCols { cols = append(cols, sortCol{Key: "reported", Label: "REPORTED", Class: "num-col coll-col", DescFirst: true}, sortCol{Key: "sidtoday", Label: "SIDTODAY", Class: "num-col coll-col", DescFirst: true}) } if collCols && prefs.IncludeFOIA { cols = append(cols, sortCol{Key: "foia", Label: "FOIA", Class: "num-col coll-col", DescFirst: true}) } data := &categoryPage{ Base: pageWith(prefs), Title: c.Title, Blurb: c.Blurb, Heads: sortHeads("?", cols, listSort, desc), RowsHTML: co.buildCategoryRows(key, table, rowOrder, showAZ), CollCols: collCols, ShowCollCols: collCols && (prefs.ShowCollCols || collSort), ShowAZ: showAZ, Letters: letters, } data.ActiveCategory = cat if data.RowsHTML == "" { data.Entries = table.ordered(rowOrder) } co.addFacetNav(&data.Base, prefs) switch cat { case "classification", "dissemination": data.SeeAlso = []seeAlsoLink{{"/browse/banner", "full classification banners (verbatim marking lines)"}} case "banner": data.SeeAlso = []seeAlsoLink{ {"/browse/classification", "classification components"}, {"/browse/dissemination", "dissemination controls"}, } } render(w, r, data) } // categoryEntry = one row of a /browse/{category} list page type categoryEntry struct { Name string URL template.URL // /browse/{category}/{name}, path-escaped Note string // the agency expansion on the agency list, else "" Count int // total, all collections News int // type REPORTED (categoryTypeCounts), any collection Archive int // type SIDTODAY, incl. individually reported issues FOIA int // type FOIA NonFOIA int // in the Reported and SIDtoday collections: the total when FOIA releases are left out NoFOIA bool // a row of the FOIA-less table: no FOIA column FirstLetter string fold string // accent-folded lowercase name: the name sort key } // categoryTable = one category's list page type categoryTable struct { Entries []categoryEntry // by name, ascending orders map[string][]int32 // categoryListKey -> indices into Entries, in that order letters map[bool][]string // the two name orderings' A-Z letters as they come, by desc } func (t *categoryTable) order(sortBy string, desc bool) []int32 { return t.orders[categoryListKey(sortBy, desc)] } func (t *categoryTable) ordered(order []int32) []categoryEntry { out := make([]categoryEntry, len(order)) for i, n := range order { out[i] = t.Entries[n] } return out } // noFOIAKey is where a category's FOIA-less table is kept in categoryLists // and categoryRowHTML. func noFOIAKey(slug string) string { return slug + "|nofoia" } // readerListKey is the table a reader's category list comes from: the // FOIA-less one unless they include FOIA releases. func readerListKey(slug string, p Prefs) string { if p.IncludeFOIA { return slug } return noFOIAKey(slug) } // newCategoryTable sorts entries by name and works out the orderings the // sorts ask for, ascending and descending. func newCategoryTable(entries []categoryEntry, sorts ...string) *categoryTable { sortCategoryEntries(entries, "name", false) t := &categoryTable{Entries: entries, orders: map[string][]int32{}, letters: map[bool][]string{}} for _, sortBy := range sorts { for _, desc := range []bool{false, true} { order := make([]int32, len(entries)) for i := range order { order[i] = int32(i) } sort.Slice(order, func(i, j int) bool { return categoryLess(&entries[order[i]], &entries[order[j]], sortBy, desc) }) t.orders[categoryListKey(sortBy, desc)] = order if sortBy != "name" { continue } seen := map[string]bool{} for _, n := range order { if l := entries[n].FirstLetter; !seen[l] { seen[l] = true t.letters[desc] = append(t.letters[desc], l) } } } } return t } func categoryListKey(sortBy string, desc bool) string { if desc { return sortBy + "/desc" } return sortBy + "/asc" } func (co *Corpus) buildCategoryLists() { co.categoryTables = make(map[string]*categoryTable, 2*len(categories)) co.categoryValueCounts = map[bool]map[string]int{false: {}, true: {}} for ci := range categories { c := &categories[ci] byType := co.categoryTypeCounts[c.Slug] byColl := co.archive.FacetsByColl[c.Slug] facet := c.Facet(co) base := make([]categoryEntry, 0, len(facet)) for name, count := range facet { e := categoryEntry{ Name: name, URL: template.URL(categoryValueURL(c.Slug, name)), Count: count, FirstLetter: firstLetter(name), fold: normFoldString(name), } if c.Slug == "agency" { e.Note = agencyName(name) } if tc := byType[name]; tc != nil { e.News, e.Archive, e.FOIA = tc.News, tc.Archive, tc.FOIA } if cc := byColl[name]; cc != nil { e.NonFOIA = cc.News + cc.Archive } base = append(base, e) } noFOIA := make([]categoryEntry, 0, len(base)) for i := range base { e := base[i] if e.NonFOIA > 0 { e.Count = e.NonFOIA e.NoFOIA = true noFOIA = append(noFOIA, e) } } co.categoryTables[c.Slug] = newCategoryTable(base, "name", "count", "reported", "sidtoday", "foia") co.categoryTables[noFOIAKey(c.Slug)] = newCategoryTable(noFOIA, "name", "count", "reported", "sidtoday") co.categoryValueCounts[true][c.Slug] = len(base) co.categoryValueCounts[false][c.Slug] = len(noFOIA) } co.seriesBySize = map[bool][]categoryEntry{} for _, foia := range []bool{false, true} { t := co.categoryTables[readerListKey("series", Prefs{IncludeFOIA: foia})] co.seriesBySize[foia] = t.ordered(t.order("count", true)) } } func sortCategoryEntries(entries []categoryEntry, sortBy string, desc bool) { sort.Slice(entries, func(i, j int) bool { return categoryLess(&entries[i], &entries[j], sortBy, desc) }) } // categoryLess orders two rows of a category table: by the sorted column in // the direction asked, then the larger total first, then by name. func categoryLess(a, b *categoryEntry, sortBy string, desc bool) bool { col := func(e *categoryEntry) int { switch sortBy { case "count": return e.Count case "reported": return e.News case "sidtoday": return e.Archive case "foia": return e.FOIA } return 0 } byName := sortBy == "name" if !byName { if x, y := col(a), col(b); x != y { if desc { return x > y } return x < y } if a.Count != b.Count { return a.Count > b.Count } } if a.fold != b.fold { if byName && desc { return a.fold > b.fold } return a.fold < b.fold } return a.Name < b.Name } type seeAlsoLink struct { URL string Text string } type timelineBar struct { Year string Count int Pct int URL string } func listSortKey(vals []string) string { best := "" for _, v := range vals { lv := strings.ToLower(strings.TrimSpace(v)) if lv != "" && (best == "" || lv < best) { best = lv } } return best } var classAlias = map[string]string{ "ts": "top secret", "s": "secret", "c": "confidential", "u": "unclassified", "strap1": "strap 1", "strap2": "strap 2", "talent keyhole": "tk", } func foldClass(v string) string { lv := strings.ToLower(strings.TrimSpace(v)) if a, ok := classAlias[lv]; ok { return a } return lv } // classSortKey ranks a marking set per the curators' order: level first // (TOP SECRET > SECRET > CONFIDENTIAL > UNCLASSIFIED), a level WITH SI // above the bare level, and the restrictive compartments raising rank // cumulatively - STRAP 2 (weight 4) > STRAP 1 (2), TK and MR (1 each) - // so TOP SECRET SI TK outranks TOP SECRET SI. func classSortKey(vals []string) string { if len(vals) == 0 { return "" } level, si, raise := 0, 0, 0 for _, v := range vals { switch foldClass(v) { case "top secret": if level < 4 { level = 4 } case "secret": if level < 3 { level = 3 } case "confidential": if level < 2 { level = 2 } case "unclassified": if level < 1 { level = 1 } case "si", "comint": si = 1 case "strap 2": raise += 4 case "strap 1": raise += 2 case "tk", "mr": raise++ } } if level == 0 && si == 0 && raise == 0 { return "" } r := strconv.Itoa(raise) if len(r) < 2 { r = "0" + r } return strconv.Itoa(level) + strconv.Itoa(si) + r } // TODO this file really needs the split. // searchClause is one row of the search form: the field code, the value as // typed into it and how the row joins the ones before it. type searchClause struct{ Conj, Field, Value string } // searchURL builds an executed /search link func searchURL(clauses []searchClause, ranges ...[2]string) string { q := url.Values{} q.Set("do_search", "1") q.Set("action", "search") for _, c := range clauses { conj := c.Conj if conj == "" { conj = "and" } q.Add("fqc", conj) q.Add("fqv", c.Value) q.Add("fqf", c.Field) } for _, r := range ranges { q.Set(r[0]+"_start", r[1]) q.Set(r[0]+"_end", r[1]) } return "/search?" + q.Encode() } func dateRangeSearchURL(fqf, fqv string, ranges ...[2]string) string { return searchURL([]searchClause{{Field: fqf, Value: fqv}}, ranges...) } func appendClause(base, field, value string) string { return base + "fqf=" + url.QueryEscape(field) + "&fqv=" + url.QueryEscape(value) + "&fqc=and" } // entityTimelineBy is the per-year histogram behind both value-page // timelines. docs are the page's documents, already narrowed to its // collection scope, so every bar link carries that scope too. // yearsOf returns a document's distinct years on the bar axis. func entityTimelineBy(docs []*Document, c *category, val string, scope collScope, dateParam string, yearsOf func(*Document) []string) []timelineBar { counts := map[string]int{} for _, d := range docs { for _, y := range yearsOf(d) { counts[y]++ } } if len(counts) < 2 { return nil } pageParam := "" if c.isDate() && validDatePrefix(val) { pageParam = c.DateParam } return yearBars(counts, func(y string) string { switch { case c.Field != "": return scope.link(dateRangeSearchURL(c.Field, exactClause(val), [2]string{dateParam, y})) case pageParam == "date_doc" && dateParam == "date_pub": return scope.link(dateRangeSearchURL("TE", "", [2]string{pageParam, val}, [2]string{dateParam, y})) } return "" }) } func entityTimeline(docs []*Document, c *category, val string, scope collScope) []timelineBar { return entityTimelineBy(docs, c, val, scope, "date_doc", func(d *Document) []string { if y := docYear(d); y != "" { return []string{y} } return nil }) } func entityPubTimeline(docs []*Document, c *category, val string, scope collScope) []timelineBar { switch c.Slug { case "journalist", "publisher": return entityTimelineBy(docs, c, val, scope, "date_pub", func(d *Document) []string { return pubYearsMatching(d, c.Field, val) }) } return entityTimelineBy(docs, c, val, scope, "date_pub", pubYearsOf) } type monthNav struct { Months []monthNavEntry Year string YearURL string YearActive bool Monthless int } func (co *Corpus) buildMonthNav(c *category, val string, allMatching []*Document, scope collScope) *monthNav { cat := c.Slug if !c.isDate() || !validDatePrefix(val) { return nil } year := val[:4] yearDocs := allMatching if len(val) == 7 { yearDocs = scope.docs(co.categoryMatches(cat, year)) } counts := map[string]int{} for _, d := range yearDocs { ds := d.DocMeta.DateCreated if cat == "published" { ds = oldestPubDate(d) } if len(ds) >= 7 { counts[ds[5:7]]++ } } var months []monthNavEntry shown := 0 for m := 1; m <= 12; m++ { mm := strconv.Itoa(m) if m < 10 { mm = "0" + mm } c := counts[mm] if c == 0 { continue } shown += c months = append(months, monthNavEntry{ Label: monthNames[m][:3], URL: scope.link(categoryValueURL(cat, year+"-"+mm)), Count: c, Active: len(val) == 7 && val[5:7] == mm, }) } if len(months) == 0 { return nil } return &monthNav{ Months: months, Year: year, YearURL: scope.link(categoryValueURL(cat, year)), YearActive: len(val) == 4, Monthless: len(yearDocs) - shown, } } // bdGroup = one of a value page's breakdown charts (its banners, its // classification levels), each value a bar. type bdGroup struct { Label string Values []bdVal } type bdVal struct { Value string Count int URL string Pct int } func (co *Corpus) handleCategoryFilter(w http.ResponseWriter, r *http.Request) { cat := r.PathValue("category") val := r.PathValue("value") c := categoryBySlug(cat) if c == nil { notFound(w, r) return } if name, ok := co.categoryNames[cat][valueKey(val)]; ok && name != val { u := categoryValueURL(cat, name) if r.URL.RawQuery != "" { u += "?" + r.URL.RawQuery } http.Redirect(w, r, u, http.StatusMovedPermanently) return } prefs := readPrefs(r) q := r.URL.Query() selectedTypeSet, typeFiltered := selectedTypes(q) scope := pageScope(q, prefs, false) everything := co.categoryMatches(cat, val) allMatching := scope.docs(everything) pubYear := "" if y := q.Get("year"); cat == "publisher" && len(y) == 4 && allDigits(y) { pubYear = y allMatching = publishedBy(allMatching, val, y) } viewTypeCounts := typeCounts(allMatching) if !typeFiltered { selectedTypeSet = allTypesSelected(co.archive.Types) } results := append([]*Document(nil), allMatching...) results = filterByType(results, selectedTypeSet) basePath := categoryValueURL(cat, val) params := url.Values{} if typeFiltered { params["type"] = q["type"] } if scope.colls != nil { params["coll"] = scope.colls } if pubYear != "" { params.Set("year", pubYear) } defSort, boost := "title", val if cat == "series" { defSort = "series" } if cat == "series" || c.isDate() { boost = "" } data := &listPage{ Base: pageWith(prefs), listView: co.buildListViewBoosted(q, prefs, results, basePath, params, defSort, false, boost), } searchLink := "" if c.Field != "" { searchLink = searchURL([]searchClause{ {Field: "TE"}, {Field: c.Field, Value: exactClause(val)}, }) } else if c.isDate() && validDatePrefix(val) { searchLink = dateRangeSearchURL("TE", "", [2]string{c.DateParam, val}) } docScope := scope if pubYear != "" { docScope = scope.and(url.Values{"date_pub_start": {pubYear}, "date_pub_end": {pubYear}}.Encode()) } searchLink = docScope.link(searchLink) if mn := co.buildMonthNav(c, val, allMatching, scope); mn != nil { data.MonthNav = mn.Months data.MonthNavYear = mn.Year data.MonthNavYearURL = mn.YearURL data.MonthNavYearActive = mn.YearActive if mn.Monthless > 0 { data.MonthlessCount = mn.Monthless } } var gloss *GlossaryEntry if cat == "program" { gloss = co.glossaryLookup(val) } data.GlossaryEntry = gloss if pubYear != "" { data.YearScope = pubYear data.YearScopeAllURL = scope.link(basePath) } refs := categoryRefinements(allMatching, c, val, docScope, pubYear) data.Refinements = refs var breakdowns []bdGroup for _, g := range refs { if g.Label != "Banner" && g.Label != "Classification" { continue } top := 0 for _, v := range g.Values { if v.Count > top { top = v.Count } } bg := bdGroup{Label: g.Label} for _, v := range g.Values { bg.Values = append(bg.Values, bdVal{v.Name, v.Count, v.URL, barPct(v.Count, top, 2)}) } breakdowns = append(breakdowns, bg) } data.Breakdowns = breakdowns data.Timeline = entityTimeline(allMatching, c, val, docScope) if pubYear == "" { // a year-scoped page has one publication year: no timeline data.PubTimeline = entityPubTimeline(allMatching, c, val, scope) } co.addFacetNav(&data.Base, prefs) data.Title = val data.ActiveCategory = cat data.Cards.ActiveCategory, data.Cards.ActiveValue = cat, val data.SearchLink = searchLink data.SelectedTypes = selectedTypeSet data.AllTypes = viewTypeCounts data.Params = q data.CatLabel = c.Label data.Blurb = c.Blurb render(w, r, data) } // categoryRefinements builds the "combine with another filter" facet groups // for a category value page. For the docs matching this page's value, tally // the other browsable dimensions and link each value to a TWO-clause search // (this page's value AND that value). On a publisher page scoped to pubYear // the links carry that publication year, and the tallies read the // journalists the way that search will (refineValues). func categoryRefinements(docs []*Document, c *category, val string, scope collScope, pubYear string) []searchFacet { if c.Field == "" || len(docs) < 2 { return nil } base := scope.link(searchURL([]searchClause{{Field: c.Field, Value: exactClause(val)}})) + "&" groups := []facetGroup{ {"Banner", "BA"}, {"Classification", "CL"}, {"Dissemination", "DI"}, {"Genre", "GE"}, {"Programs", "PR"}, {"Agencies", "CR"}, {"Countries", "CO"}, {"Journalists", "RE"}, } kept := groups[:0] for _, g := range groups { if g.Code != c.Field { kept = append(kept, g) } } return facetGroups(docs, kept, base, refineValues(pubYear, pubYear)) } func (co *Corpus) buildCategoryDocs() { co.categoryDocs = make(map[string]map[string][]int32, len(categories)) co.categoryNames = make(map[string]map[string]string, len(categories)) co.archive.Facets = make(map[string]map[string]int, len(categories)) co.archive.FacetsByColl = make(map[string]map[string]*collCounts, len(categories)) co.categoryTypeCounts = make(map[string]map[string]*collCounts, len(categories)) for ci := range categories { c := &categories[ci] byValue := make(map[string][]int32) name := make(map[string]string) // index key -> display value add := func(key, value string, i int) { if _, ok := name[key]; !ok { name[key] = value } byValue[key] = append(byValue[key], int32(i)) } for i := range co.archive.Docs { d := &co.archive.Docs[i] if c.isDate() { date := d.DocMeta.DateCreated if c.DateParam == "date_pub" { date = oldestPubDate(d) } if len(date) >= 4 { add(date[:4], date[:4], i) } if len(date) >= 7 { add(date[:7], date[:7], i) } continue } for _, v := range facetValues(d, c.Field) { add(valueKey(v), v, i) } } counts := make(map[string]int) byColl := make(map[string]*collCounts) byType := make(map[string]*collCounts) for key, docs := range byValue { if c.isDate() && len(key) != 4 { continue } cc, tc := &collCounts{}, &collCounts{} for _, n := range docs { cc.add(co.archive.Docs[n].Collection) tc.addType(co.archive.Docs[n].Type) } counts[name[key]] = len(docs) byColl[name[key]] = cc byType[name[key]] = tc } co.categoryDocs[c.Slug] = byValue co.categoryNames[c.Slug] = name co.archive.Facets[c.Slug] = counts co.archive.FacetsByColl[c.Slug] = byColl co.categoryTypeCounts[c.Slug] = byType } } // categoryMatches returns the documents on a /browse/{category}/{value} // page, in corpus order, as a fresh slice of pointers the caller owns. func (co *Corpus) categoryMatches(cat, val string) []*Document { c := categoryBySlug(cat) if c == nil { return nil } if c.isDate() && !validDatePrefix(val) { return nil } idx := co.categoryDocs[cat][valueKey(val)] out := make([]*Document, len(idx)) for i, n := range idx { out[i] = &co.archive.Docs[n] } return out } func validDatePrefix(val string) bool { switch len(val) { case 4: return allDigits(val) case 7: return allDigits(val[:4]) && val[4] == '-' && allDigits(val[5:]) } return false } func validDate(val string) bool { if len(val) == 10 { return validDatePrefix(val[:7]) && val[7] == '-' && allDigits(val[8:]) } return validDatePrefix(val) } func allDigits(s string) bool { if s == "" { return false } for i := 0; i < len(s); i++ { if s[i] < '0' || s[i] > '9' { return false } } return true } func publishedBy(docs []*Document, publisher, year string) []*Document { var out []*Document for _, d := range docs { for _, p := range d.Publication { if !p.ReportingOnly && sameValue(p.Publisher, publisher) && strings.HasPrefix(p.DatePublished, year) { out = append(out, d) break } } } return out } func publisherYearURL(publisher, year string) string { return categoryValueURL("publisher", publisher) + "?year=" + url.QueryEscape(year) } func oldestPubDate(d *Document) string { oldest := "" for _, p := range d.Publication { if len(p.DatePublished) >= 4 && (oldest == "" || p.DatePublished < oldest) { oldest = p.DatePublished } } return oldest } func (d *Document) FirstPubYear() string { if date := oldestPubDate(d); len(date) >= 4 { return date[:4] } return "" } func docYear(d *Document) string { if len(d.DocMeta.DateCreated) >= 4 { return d.DocMeta.DateCreated[:4] } return "" } // pubYearsOf lists the years the document was published in, any entry's, // in the entries' order. func pubYearsOf(d *Document) []string { if d.fields != nil { return d.fields.pubYears } var out []string for _, p := range d.Publication { if len(p.DatePublished) < 4 { continue } if y := p.DatePublished[:4]; !slices.Contains(out, y) { out = append(out, y) } } return out } func pubYearsMatching(d *Document, field, val string) []string { var out []string for i := range d.Publication { date := d.Publication[i].DatePublished if len(date) < 4 || slices.Contains(out, date[:4]) { continue } for _, v := range entryValues(d, i, field) { if sameValue(v, val) { out = append(out, date[:4]) break } } } return out } type monthNavEntry struct { Label string URL string Count int Active bool } // docSort is one way to order a list of documents. The sort bar, the // doc-table's column headers, the list pages, search and /api/docs all read // this table, so a sort means the same thing everywhere. type docSort struct { Key string Label string Value func(d *Document) string EmptiesLast bool DescFirst bool InBar bool // offered in the sort bar } // titleSortKey sorts a title by its first letter or digit func titleSortKey(d *Document) string { return strings.TrimLeftFunc(strings.ToLower(d.Title), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) } var docSorts = []docSort{ {"title", "Title", titleSortKey, false, false, true}, {"agency", "Agency", func(d *Document) string { return strings.ToLower(d.DocMeta.Creator) }, false, false, true}, {"date", "Doc Date", func(d *Document) string { return d.DocMeta.DateCreated }, true, false, true}, {"pubdate", "Pub Date", oldestPubDate, true, false, true}, {"classification", "Classification", func(d *Document) string { return classSortKey(d.DocMeta.Classification) }, true, true, false}, {"dissemination", "Dissemination", func(d *Document) string { return listSortKey(d.DocMeta.Dissemination) }, true, false, false}, {"program", "Programs", func(d *Document) string { return listSortKey(d.DocMeta.Programs) }, true, false, false}, {"country", "Countries", func(d *Document) string { return listSortKey(d.DocMeta.Countries) }, true, false, false}, {"people", "People", func(d *Document) string { return listSortKey(d.DocMeta.People) }, true, false, false}, {"journalist", "Journalists", func(d *Document) string { return listSortKey(d.JournalistsDeduped()) }, true, false, false}, {"type", "Type", func(d *Document) string { return strings.ToLower(d.Type) }, false, false, false}, {"series", "Series order", func(d *Document) string { if p := d.DocMeta.Series.Part; p > 0 { return fmt.Sprintf("%06d", p) } return "" }, true, false, false}, } var docSortByKey = func() map[string]*docSort { m := make(map[string]*docSort, len(docSorts)) for i := range docSorts { m[docSorts[i].Key] = &docSorts[i] } return m }() var docTableCols = func() []sortCol { col := func(key string) sortCol { s := docSortByKey[key] return sortCol{Key: s.Key, Label: s.Label, DescFirst: s.DescFirst} } return []sortCol{ col("title"), col("agency"), col("date"), col("pubdate"), col("classification"), col("dissemination"), col("program"), col("country"), col("people"), col("journalist"), {Label: "Published"}, col("type"), } }() type sortCol struct { Key, Label, Class string DescFirst, Fixed bool } type sortHead struct { sortCol URL string Active, Desc bool } func sortHeads(base string, cols []sortCol, cur string, desc bool) []sortHead { heads := make([]sortHead, len(cols)) for i, c := range cols { h := sortHead{sortCol: c} if c.Key != "" { next := c.DescFirst if h.Active = c.Key == cur; h.Active { h.Desc = desc if !c.Fixed { next = !desc } } h.URL = base + "sort=" + c.Key if !c.Fixed { h.URL += "&order=" + orderParam(next) } } heads[i] = h } return heads } func orderParam(desc bool) string { if desc { return "desc" } return "asc" } // sortCmp orders two sort keys: in the direction asked, an empty key after // the others either way where the sort puts its empties last. func sortCmp(a, b string, desc, emptiesLast bool) int { if emptiesLast && (a == "") != (b == "") { if b == "" { return -1 } return 1 } if desc { return cmp.Compare(b, a) } return cmp.Compare(a, b) } // boostTitleMatches moves the documents whose title has word as a whole word, // in any case, to the front, keeping each group's order e.g. on // /browse/tag/Tor the documents about Tor come before the ones that only carry // the tag. func boostTitleMatches(docs []*Document, word string) { re, err := regexp.Compile(`(?i)\b` + regexp.QuoteMeta(word) + `\b`) if err != nil { return } var hit, rest []*Document for _, d := range docs { if re.MatchString(d.Title) { hit = append(hit, d) } else { rest = append(rest, d) } } copy(docs, append(hit, rest...)) } func sortDocs(docs []*Document, sortBy, order string) { desc := order == "desc" if docSortByKey[sortBy] == nil { sortBy = "title" } copy(docs, sortByField(docs, func(d *Document) *Document { return d }, sortBy, desc)) } func sortScored(scored []ScoredDoc, sortBy, order string) []ScoredDoc { if docSortByKey[sortBy] == nil { return scored } return sortByField(scored, func(s ScoredDoc) *Document { return s.Doc }, sortBy, order == "desc") } func sortByField[T any](items []T, doc func(T) *Document, sortBy string, desc bool) []T { ds := docSortByKey[sortBy] keys := make([]string, len(items)) for i := range items { keys[i] = ds.Value(doc(items[i])) } idx := make([]int, len(items)) for i := range idx { idx[i] = i } slices.SortStableFunc(idx, func(a, b int) int { return sortCmp(keys[a], keys[b], desc, ds.EmptiesLast) }) out := make([]T, len(items)) for i, o := range idx { out[i] = items[o] } return out } func (co *Corpus) docVersion(r *http.Request) (*Document, *DocVersion) { doc := co.docIndex[r.PathValue("id")] if doc == nil { return nil, nil } return doc, doc.VersionFor(r.PathValue("slug")) } func (co *Corpus) handleVersionView(w http.ResponseWriter, r *http.Request) { doc, v := co.docVersion(r) if v == nil { notFound(w, r) return } pubs := doc.Publication[:0:0] for _, i := range v.PubIndex { pubs = append(pubs, doc.Publication[i]) } prefs := readPrefs(r) render(w, r, &versionPage{ Base: pageWith(prefs), viewer: viewer{ Images: v.Images, Sizes: v.ImageSizes, ImageBase: imageBase(doc.ID, v.Slug), Content: v.HTMLContent, Version: v, }, Doc: doc, Publications: pubs, PDFURL: versionFilePath(doc.ID, v.Slug, v.PDFFile), }) } func (co *Corpus) handleVersionPages(w http.ResponseWriter, r *http.Request) { doc, v := co.docVersion(r) if v == nil { notFound(w, r) return } prefs := readPrefs(r) render(w, r, &pagesPage{ Base: pageWith(prefs), viewer: viewer{Images: v.Images, Sizes: v.ImageSizes, ImageBase: imageBase(doc.ID, v.Slug), Version: v}, Doc: doc, TextOnly: prefs.TextOnly, }) }