2273 lines
61 KiB
Go
2273 lines
61 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"cmp"
|
|
"fmt"
|
|
"html/template"
|
|
"math/rand"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"slices"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
type facetEntry struct {
|
|
Name string
|
|
Count int
|
|
URL string
|
|
}
|
|
|
|
// facetLinks renders a metadata field's values as links to their category
|
|
// pages, wherever a card, a table row, a search result or a document page
|
|
// lists them. It takes the template's dict:
|
|
//
|
|
// Vals the values ("-" stands for none)
|
|
// Cat the category's slug
|
|
// Sep between the links: "dot", "space", or a comma when not given
|
|
// AsList one <li> per link instead
|
|
// Class the links' class
|
|
// Active the value of the page the links are on, when it is this
|
|
// category's: its own link is marked active
|
|
// QS the collection scope the links carry ("?coll=..."), when the page
|
|
// they open would otherwise leave out the document they come from
|
|
//
|
|
// The output is what html/template made of the same markup: every value
|
|
// goes through htmlEscape.
|
|
func facetLinks(p map[string]interface{}) (template.HTML, error) {
|
|
vals, ok := p["Vals"].([]string)
|
|
if !ok && p["Vals"] != nil {
|
|
return "", fmt.Errorf("facetLinks: Vals is %T, not a list of values", p["Vals"])
|
|
}
|
|
if len(vals) == 0 {
|
|
return "-", nil
|
|
}
|
|
cat, _ := p["Cat"].(string)
|
|
sep, _ := p["Sep"].(string)
|
|
class, _ := p["Class"].(string)
|
|
active, _ := p["Active"].(string)
|
|
qs, _ := p["QS"].(string)
|
|
asList, _ := p["AsList"].(bool)
|
|
|
|
var b strings.Builder
|
|
if asList {
|
|
b.WriteString(`<ul class="meta-value-list">`)
|
|
}
|
|
for i, v := range vals {
|
|
switch {
|
|
case asList:
|
|
b.WriteString("<li>")
|
|
case i == 0:
|
|
case sep == "dot":
|
|
b.WriteString(` <span class="doc-dot">·</span> `)
|
|
case sep == "space":
|
|
b.WriteString(" ")
|
|
default:
|
|
b.WriteString(", ")
|
|
}
|
|
on := active != "" && sameValue(v, active)
|
|
b.WriteString(`<a href="`)
|
|
b.WriteString(htmlEscape(categoryValueURL(cat, v) + qs))
|
|
b.WriteString(`"`)
|
|
if class != "" || on {
|
|
b.WriteString(` class="`)
|
|
b.WriteString(htmlEscape(class))
|
|
if class != "" && on {
|
|
b.WriteString(" ")
|
|
}
|
|
if on {
|
|
b.WriteString("active")
|
|
}
|
|
b.WriteString(`"`)
|
|
}
|
|
b.WriteString(">")
|
|
b.WriteString(htmlEscape(v))
|
|
b.WriteString("</a>")
|
|
if asList {
|
|
b.WriteString("</li>")
|
|
}
|
|
}
|
|
if asList {
|
|
b.WriteString("</ul>")
|
|
}
|
|
return template.HTML(b.String()), nil
|
|
}
|
|
|
|
func htmlEscape(s string) string {
|
|
if !strings.ContainsAny(s, "\x00\"&'+<>") && isASCII(s) {
|
|
return s
|
|
}
|
|
var b strings.Builder
|
|
b.Grow(len(s) + 16)
|
|
for i := 0; i < len(s); {
|
|
r, w := utf8.DecodeRuneInString(s[i:])
|
|
switch {
|
|
case r == 0:
|
|
b.WriteString("\uFFFD")
|
|
case r == '"':
|
|
b.WriteString(""")
|
|
case r == '&':
|
|
b.WriteString("&")
|
|
case r == '\'':
|
|
b.WriteString("'")
|
|
case r == '+':
|
|
b.WriteString("+")
|
|
case r == '<':
|
|
b.WriteString("<")
|
|
case r == '>':
|
|
b.WriteString(">")
|
|
case r >= 0xfdd0 && r <= 0xfdef, r >= 0xfff0 && r <= 0xffff:
|
|
fmt.Fprintf(&b, "&#x%x;", r)
|
|
default:
|
|
b.WriteString(s[i : i+w])
|
|
}
|
|
i += w
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// rankCounts returns the keys of a count map in rank order (most counted
|
|
// first, ties alphabetically ignoring accents and case, then by the raw
|
|
// spelling)
|
|
func rankCounts(m map[string]int, n int) []string {
|
|
type keyed struct {
|
|
name, fold string
|
|
count int
|
|
}
|
|
all := make([]keyed, 0, len(m))
|
|
for k, v := range m {
|
|
all = append(all, keyed{k, normFoldString(k), v})
|
|
}
|
|
sort.Slice(all, func(i, j int) bool {
|
|
a, b := &all[i], &all[j]
|
|
if a.count != b.count {
|
|
return a.count > b.count
|
|
}
|
|
if a.fold != b.fold {
|
|
return a.fold < b.fold
|
|
}
|
|
return a.name < b.name
|
|
})
|
|
if n >= 0 && len(all) > n {
|
|
all = all[:n]
|
|
}
|
|
out := make([]string, len(all))
|
|
for i := range all {
|
|
out[i] = all[i].name
|
|
}
|
|
return out
|
|
}
|
|
|
|
func topFacets(m map[string]int, n int) []facetEntry {
|
|
names := rankCounts(m, n)
|
|
out := make([]facetEntry, len(names))
|
|
for i, k := range names {
|
|
out[i] = facetEntry{Name: k, Count: m[k]}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func barPct(count, denom, floor int) int {
|
|
if denom <= 0 || count <= 0 {
|
|
return 0
|
|
}
|
|
pct := count * 100 / denom
|
|
if pct < floor {
|
|
pct = floor
|
|
}
|
|
return pct
|
|
}
|
|
|
|
func sortedYears(m map[string]int) (years []string, maxCount int) {
|
|
years = make([]string, 0, len(m))
|
|
for y, c := range m {
|
|
years = append(years, y)
|
|
if c > maxCount {
|
|
maxCount = c
|
|
}
|
|
}
|
|
sort.Strings(years)
|
|
return years, maxCount
|
|
}
|
|
|
|
func yearBars(m map[string]int, urlFor func(y string) string) []timelineBar {
|
|
years, maxCount := sortedYears(m)
|
|
if maxCount == 0 {
|
|
return nil
|
|
}
|
|
bars := make([]timelineBar, 0, len(years))
|
|
for _, y := range years {
|
|
bars = append(bars, timelineBar{
|
|
Year: y,
|
|
Count: m[y],
|
|
Pct: barPct(m[y], maxCount, 1),
|
|
URL: urlFor(y),
|
|
})
|
|
}
|
|
return bars
|
|
}
|
|
|
|
type homeFigures struct {
|
|
TopAgencies, TopPrograms, TopCountries, TopPeople,
|
|
TopJournalists, TopPublishers, TopTags []facetEntry
|
|
DocHist, PubHist []timelineBar
|
|
}
|
|
|
|
func (co *Corpus) buildHomeStats() {
|
|
co.homeStats = co.homeFigures(false)
|
|
co.homeStatsSIDtoday = co.homeFigures(true)
|
|
co.buildRandomPools()
|
|
}
|
|
|
|
func (co *Corpus) homeFigures(sidtoday bool) homeFigures {
|
|
scope := scopeOf(map[string]bool{collectionNews: true, collectionArchive: sidtoday})
|
|
top := func(slug string, n int) []facetEntry {
|
|
entries := topFacets(co.homeFacet(slug, sidtoday), n)
|
|
for i := range entries {
|
|
entries[i].URL = scope.link(categoryValueURL(slug, entries[i].Name))
|
|
}
|
|
return entries
|
|
}
|
|
f := homeFigures{
|
|
TopAgencies: top("agency", 10),
|
|
TopPrograms: top("program", 20),
|
|
TopCountries: top("country", 15),
|
|
TopPeople: top("people", 15),
|
|
TopJournalists: top("journalist", 12),
|
|
TopPublishers: top("publisher", 10),
|
|
TopTags: top("tag", 25),
|
|
}
|
|
f.DocHist, f.PubHist = co.homeHistograms(scope)
|
|
return f
|
|
}
|
|
|
|
func (co *Corpus) homeFacet(slug string, sidtoday bool) map[string]int {
|
|
out := make(map[string]int)
|
|
for value, c := range co.archive.FacetsByColl[slug] {
|
|
n := c.News
|
|
if sidtoday {
|
|
n += c.Archive
|
|
}
|
|
if n > 0 {
|
|
out[value] = n
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (co *Corpus) homeHistograms(scope collScope) (docHist, pubHist []timelineBar) {
|
|
docYears := map[string]int{}
|
|
pubYears := map[string]int{}
|
|
for i := range co.archive.Docs {
|
|
d := &co.archive.Docs[i]
|
|
if !scope.shows(d) {
|
|
continue
|
|
}
|
|
if y := docYear(d); y != "" {
|
|
docYears[y]++
|
|
}
|
|
if y := d.FirstPubYear(); y != "" {
|
|
pubYears[y]++
|
|
}
|
|
}
|
|
link := func(slug string) func(string) string {
|
|
return func(y string) string { return scope.link(categoryValueURL(slug, y)) }
|
|
}
|
|
return yearBars(docYears, link("docdate")), yearBars(pubYears, link("published"))
|
|
}
|
|
|
|
func (co *Corpus) addFacetNav(b *Base, prefs Prefs) {
|
|
b.ShowCatBoxes = true
|
|
b.CatCounts = co.categoryValueCounts[prefs.IncludeFOIA]
|
|
}
|
|
|
|
func (co *Corpus) handleIndex(w http.ResponseWriter, r *http.Request) {
|
|
prefs := readPrefs(r)
|
|
var recentDocs []*Document
|
|
if prefs.TrackRecent {
|
|
for _, id := range readIDListCookie(r, recentDocsCookie, recentDocsMax) {
|
|
if d := co.docIndex[id]; d != nil {
|
|
recentDocs = append(recentDocs, d)
|
|
}
|
|
}
|
|
}
|
|
if len(recentDocs) > 0 {
|
|
noStore(w)
|
|
}
|
|
|
|
withSIDtoday := r.URL.Query().Get("sidtoday") == "1"
|
|
stats := co.homeStats
|
|
if withSIDtoday {
|
|
stats = co.homeStatsSIDtoday
|
|
}
|
|
render(w, r, &indexPage{
|
|
Base: pageWith(prefs),
|
|
homeFigures: stats,
|
|
TrackRecent: prefs.TrackRecent,
|
|
RecentDocs: recentDocs,
|
|
SnowdenTotal: co.archive.CollectionCounts[collectionNews] + co.archive.CollectionCounts[collectionArchive],
|
|
ArchiveTotal: co.archive.CollectionCounts[collectionArchive],
|
|
FOIATotal: co.archive.CollectionCounts[collectionFOIA],
|
|
IncludeFOIA: prefs.IncludeFOIA,
|
|
ReportedTotal: co.archive.CollectionCounts[collectionNews],
|
|
WithSIDtoday: withSIDtoday,
|
|
})
|
|
}
|
|
|
|
// randomWeights says how often /random lands in each collection
|
|
var randomWeights = []struct {
|
|
coll string
|
|
weight int
|
|
}{
|
|
{collectionNews, 65},
|
|
{collectionArchive, 25},
|
|
{collectionFOIA, 10},
|
|
}
|
|
|
|
func (co *Corpus) buildRandomPools() {
|
|
co.randomPools = map[string][]int{}
|
|
for i, d := range co.archive.Docs {
|
|
co.randomPools[d.Collection] = append(co.randomPools[d.Collection], i)
|
|
}
|
|
}
|
|
|
|
func (co *Corpus) buildCollectionDocs() {
|
|
co.everyDoc = make([]*Document, len(co.archive.Docs))
|
|
co.collectionDocs = make(map[string][]*Document, len(validCollections))
|
|
co.sidtodayDocs = nil
|
|
for i := range co.archive.Docs {
|
|
d := &co.archive.Docs[i]
|
|
co.everyDoc[i] = d
|
|
co.collectionDocs[d.Collection] = append(co.collectionDocs[d.Collection], d)
|
|
if d.Type == "SIDTODAY" {
|
|
co.sidtodayDocs = append(co.sidtodayDocs, d)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (co *Corpus) randomDoc(includeFOIA bool) *Document {
|
|
pool := func(coll string) []int {
|
|
if coll == collectionFOIA && !includeFOIA {
|
|
return nil
|
|
}
|
|
return co.randomPools[coll]
|
|
}
|
|
total := 0
|
|
for _, rw := range randomWeights {
|
|
if len(pool(rw.coll)) > 0 {
|
|
total += rw.weight
|
|
}
|
|
}
|
|
if total == 0 {
|
|
return nil
|
|
}
|
|
n := rand.Intn(total)
|
|
for _, rw := range randomWeights {
|
|
p := pool(rw.coll)
|
|
if len(p) == 0 {
|
|
continue
|
|
}
|
|
if n < rw.weight {
|
|
return &co.archive.Docs[p[rand.Intn(len(p))]]
|
|
}
|
|
n -= rw.weight
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (co *Corpus) handleRandom(w http.ResponseWriter, r *http.Request) {
|
|
d := co.randomDoc(readPrefs(r).IncludeFOIA)
|
|
if d == nil {
|
|
notFound(w, r)
|
|
return
|
|
}
|
|
http.Redirect(w, r, docPath(d.ID), http.StatusFound)
|
|
}
|
|
|
|
func handleAbout(w http.ResponseWriter, r *http.Request) {
|
|
render(w, r, &aboutPage{})
|
|
}
|
|
|
|
func paginate(total, page, perPage int) (start, end, clampedPage, totalPages int) {
|
|
if perPage <= 0 {
|
|
return 0, total, 1, 1
|
|
}
|
|
totalPages = (total + perPage - 1) / perPage
|
|
|
|
if page < 1 {
|
|
page = 1
|
|
}
|
|
maxPage := totalPages
|
|
if maxPage < 1 {
|
|
maxPage = 1
|
|
}
|
|
if page > maxPage {
|
|
page = maxPage
|
|
}
|
|
start = (page - 1) * perPage
|
|
if start < 0 || start > total {
|
|
start = total
|
|
}
|
|
end = start + perPage
|
|
if end < start || end > total {
|
|
end = total
|
|
}
|
|
return start, end, page, totalPages
|
|
}
|
|
|
|
// buildListView assembles everything the shared doc-cards / doc-table
|
|
// layouts need: view, sort, pagination, the link bases their partials
|
|
// build on, and the card display prefs.
|
|
//
|
|
// The page embeds what is returned and adds its own fields, including the
|
|
// cards' ActiveCategory and ActiveValue when it has a filter value for them
|
|
// to highlight.
|
|
func (co *Corpus) buildListView(q url.Values, prefs Prefs, docs []*Document, basePath string, params url.Values, defaultSort string, explicitView bool) listView {
|
|
return co.buildListViewBoosted(q, prefs, docs, basePath, params, defaultSort, explicitView, "")
|
|
}
|
|
|
|
func (co *Corpus) buildListViewBoosted(q url.Values, prefs Prefs, docs []*Document, basePath string, params url.Values, defaultSort string, explicitView bool, boost string) listView {
|
|
view := q.Get("view")
|
|
sortBy := sortToken(q.Get("sort"))
|
|
order := sortToken(q.Get("order"))
|
|
if docSortByKey[sortBy] == nil {
|
|
sortBy = ""
|
|
}
|
|
|
|
effSort := cmp.Or(sortBy, defaultSort)
|
|
sortDocs(docs, effSort, order)
|
|
if sortBy == "" && boost != "" {
|
|
boostTitleMatches(docs, boost)
|
|
}
|
|
|
|
page, _ := strconv.Atoi(q.Get("p"))
|
|
finalDocs := docs
|
|
totalPages := 1
|
|
if view == "table" {
|
|
page = 1
|
|
} else {
|
|
var start, end int
|
|
start, end, page, totalPages = paginate(len(docs), page, prefs.cardsPerPage())
|
|
finalDocs = docs[start:end]
|
|
}
|
|
|
|
baseQ := cloneValues(params)
|
|
sortQ := cloneValues(params)
|
|
if view != "" && (explicitView || view == "table") {
|
|
baseQ.Set("view", view)
|
|
}
|
|
|
|
lv := listView{
|
|
Docs: finalDocs,
|
|
Total: len(docs),
|
|
Page: page,
|
|
TotalPages: totalPages,
|
|
View: view,
|
|
Sort: sortBy,
|
|
Order: order,
|
|
DefaultSort: defaultSort,
|
|
BaseURL: queryBase(basePath, baseQ),
|
|
SortURL: queryBase(basePath, sortQ),
|
|
TableHeads: sortHeads(queryBase(basePath, baseQ), docTableCols, effSort, order == "desc"),
|
|
Cards: newCardView(prefs, finalDocs),
|
|
}
|
|
if view == "table" {
|
|
lv.RowsHTML = co.buildDocRows(finalDocs, false)
|
|
}
|
|
return lv
|
|
}
|
|
|
|
// selectedTypes reads the ?type= params into a set, and reports whether the
|
|
// request actually filters (no params means everything, not nothing).
|
|
func selectedTypes(v url.Values) (map[string]bool, bool) {
|
|
sel := make(map[string]bool, len(v["type"]))
|
|
for _, t := range v["type"] {
|
|
if t = strings.ToUpper(strings.TrimSpace(t)); t != "" {
|
|
sel[t] = true
|
|
}
|
|
}
|
|
return sel, len(sel) > 0
|
|
}
|
|
|
|
func typeSelected(d *Document, sel map[string]bool) bool {
|
|
return len(sel) == 0 || sel[d.Type]
|
|
}
|
|
|
|
func typeCounts(docs []*Document) map[string]int {
|
|
counts := make(map[string]int)
|
|
for i := range docs {
|
|
counts[docs[i].Type]++
|
|
}
|
|
return counts
|
|
}
|
|
|
|
func filterByType(docs []*Document, sel map[string]bool) []*Document {
|
|
if len(sel) == 0 {
|
|
return docs
|
|
}
|
|
kept := docs[:0]
|
|
for _, d := range docs {
|
|
if typeSelected(d, sel) {
|
|
kept = append(kept, d)
|
|
}
|
|
}
|
|
return kept
|
|
}
|
|
|
|
func allTypesSelected(counts map[string]int) map[string]bool {
|
|
sel := make(map[string]bool, len(counts))
|
|
for t := range counts {
|
|
sel[t] = true
|
|
}
|
|
return sel
|
|
}
|
|
|
|
var validCollections = map[string]bool{
|
|
collectionNews: true,
|
|
collectionArchive: true,
|
|
collectionFOIA: true,
|
|
}
|
|
|
|
var collectionOrder = []string{collectionNews, collectionArchive, collectionFOIA}
|
|
|
|
func collectionParams(sel map[string]bool) []string {
|
|
var out []string
|
|
for _, c := range collectionOrder {
|
|
if sel[c] {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// defaultCollections is what a page shows when its URL names no
|
|
// collection: FOIA releases are opt-in (Prefs.IncludeFOIA).
|
|
var defaultCollections = map[string]bool{collectionNews: true, collectionArchive: true}
|
|
|
|
// collScope is a page's collection scope: the documents it shows and what
|
|
// its own links carry. Every link a page builds from its documents goes
|
|
// through link, so the number on a link is the number its click returns.
|
|
type collScope struct {
|
|
filter map[string]bool // the documents shown; nil: every collection
|
|
shown map[string]bool // the same as a set, never nil (the form's boxes)
|
|
defaulted bool // the URL named no collection: the preference decided
|
|
colls []string // the coll= the page's links carry; nil: none
|
|
extra string // more scope for the links (a publisher page's year)
|
|
}
|
|
|
|
// scopeOf is an explicit scope: sel, with links that name it, so a count
|
|
// holds whatever the next reader's FOIA preference.
|
|
func scopeOf(sel map[string]bool) collScope {
|
|
s := collScope{shown: sel, colls: collectionParams(sel)}
|
|
if len(s.colls) < len(validCollections) {
|
|
s.filter = sel
|
|
}
|
|
return s
|
|
}
|
|
|
|
// wholeArchive scopes links to all three collections: a count taken over
|
|
// the whole archive must not change with the reader's FOIA preference.
|
|
var wholeArchive = scopeOf(validCollections)
|
|
|
|
// pageScope reads a page's scope: the coll= its URL names or, when it
|
|
// names none, the reader's preference, FOIA releases only when opted in.
|
|
func pageScope(v url.Values, p Prefs, pinDefault bool) collScope {
|
|
if _, given := v["coll"]; given {
|
|
if f := collectionFilter(v); f != nil {
|
|
return scopeOf(f)
|
|
}
|
|
return wholeArchive
|
|
}
|
|
s := scopeOf(defaultCollections)
|
|
if p.IncludeFOIA {
|
|
s = wholeArchive
|
|
}
|
|
s.defaulted = true
|
|
if !pinDefault {
|
|
s.colls = nil
|
|
}
|
|
return s
|
|
}
|
|
|
|
func (s collScope) shows(d *Document) bool { return inCollections(d, s.filter) }
|
|
|
|
func (s collScope) hasValue(co *Corpus, slug, value string) bool {
|
|
c := co.archive.FacetsByColl[slug][co.categoryNames[slug][valueKey(value)]]
|
|
if c == nil {
|
|
return false
|
|
}
|
|
in := func(coll string, n int) bool { return n > 0 && (s.filter == nil || s.filter[coll]) }
|
|
return in(collectionNews, c.News) || in(collectionArchive, c.Archive) || in(collectionFOIA, c.FOIA)
|
|
}
|
|
|
|
func (s collScope) docs(docs []*Document) []*Document { return filterByCollection(docs, s.filter) }
|
|
|
|
func (s collScope) explicit() bool { return s.filter != nil && !s.defaulted }
|
|
|
|
func (s collScope) including(coll string) collScope {
|
|
if s.filter == nil || s.filter[coll] {
|
|
return s
|
|
}
|
|
return wholeArchive
|
|
}
|
|
|
|
func (s collScope) and(qs string) collScope {
|
|
if s.extra != "" {
|
|
qs = s.extra + "&" + qs
|
|
}
|
|
s.extra = qs
|
|
return s
|
|
}
|
|
|
|
func (s collScope) qs() string {
|
|
q := ""
|
|
if len(s.colls) > 0 {
|
|
q = url.Values{"coll": s.colls}.Encode()
|
|
}
|
|
switch {
|
|
case q == "":
|
|
return s.extra
|
|
case s.extra != "":
|
|
return q + "&" + s.extra
|
|
}
|
|
return q
|
|
}
|
|
|
|
func (s collScope) link(u string) string { return withQuery(u, s.qs()) }
|
|
|
|
func (s collScope) suffix() string {
|
|
if q := s.qs(); q != "" {
|
|
return "?" + q
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func withQuery(link, qs string) string {
|
|
switch {
|
|
case qs == "" || link == "":
|
|
return link
|
|
case strings.Contains(link, "?"):
|
|
return link + "&" + qs
|
|
}
|
|
return link + "?" + qs
|
|
}
|
|
|
|
func cloneValues(v url.Values) url.Values {
|
|
out := make(url.Values, len(v))
|
|
for k, vs := range v {
|
|
out[k] = append([]string(nil), vs...)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (co *Corpus) handleBrowseRoot(w http.ResponseWriter, r *http.Request) {
|
|
q := r.URL.Query()
|
|
collection := q.Get("collection")
|
|
if !validCollections[collection] {
|
|
collection = collectionNews
|
|
}
|
|
|
|
if collection == collectionArchive {
|
|
co.renderArchiveBrowse(w, r, q)
|
|
return
|
|
}
|
|
|
|
prefs := readPrefs(r)
|
|
|
|
selTypes, typeFiltered := selectedTypes(q)
|
|
typeFiltered = typeFiltered && collection == collectionNews
|
|
selectedTypeSet := map[string]bool{}
|
|
if collection == collectionNews {
|
|
selectedTypeSet = selTypes
|
|
if !typeFiltered {
|
|
selectedTypeSet = allTypesSelected(co.archive.NewsTypes)
|
|
}
|
|
}
|
|
|
|
docs := append([]*Document(nil), co.collectionDocs[collection]...)
|
|
if collection == collectionNews {
|
|
docs = filterByType(docs, selectedTypeSet)
|
|
}
|
|
|
|
params := url.Values{}
|
|
if collection != collectionNews {
|
|
params.Set("collection", collection)
|
|
}
|
|
if typeFiltered {
|
|
params["type"] = q["type"]
|
|
}
|
|
|
|
data := &browsePage{
|
|
Base: pageWith(prefs),
|
|
listView: co.buildListView(q, prefs, docs, "/browse", params, "title", false),
|
|
Collection: collection,
|
|
CollectionCounts: co.archive.CollectionCounts,
|
|
SIDTodayCount: co.archive.SIDTodayCount,
|
|
SelectedTypes: selectedTypeSet,
|
|
AllTypes: co.archive.NewsTypes,
|
|
Params: q,
|
|
}
|
|
co.addFacetNav(&data.Base, prefs)
|
|
render(w, r, data)
|
|
}
|
|
|
|
// --- SIDtoday archive: chronological view ---
|
|
type archiveMonth struct {
|
|
Name string
|
|
Docs []*Document
|
|
}
|
|
|
|
type archiveYear struct {
|
|
Year string
|
|
Count int
|
|
Months []archiveMonth
|
|
}
|
|
|
|
var monthNames = [...]string{"", "January", "February", "March", "April",
|
|
"May", "June", "July", "August", "September", "October", "November",
|
|
"December"}
|
|
|
|
func (co *Corpus) buildArchiveTimeline() {
|
|
co.archiveTimeline = nil
|
|
docs := append([]*Document(nil), co.sidtodayDocs...)
|
|
|
|
sort.SliceStable(docs, func(i, j int) bool {
|
|
a, b := docs[i].DocMeta.DateCreated, docs[j].DocMeta.DateCreated
|
|
if (a == "") != (b == "") {
|
|
return b == ""
|
|
}
|
|
if a != b {
|
|
return a < b
|
|
}
|
|
ta, tb := strings.ToLower(docs[i].Title), strings.ToLower(docs[j].Title)
|
|
if ta == tb {
|
|
return false
|
|
}
|
|
return ta < tb
|
|
})
|
|
|
|
for _, d := range docs {
|
|
y, m := "Undated", ""
|
|
date := d.DocMeta.DateCreated
|
|
if len(date) >= 4 {
|
|
y = date[:4]
|
|
}
|
|
if len(date) >= 7 {
|
|
if n, err := strconv.Atoi(date[5:7]); err == nil && n >= 1 && n <= 12 {
|
|
m = monthNames[n]
|
|
}
|
|
}
|
|
|
|
if len(co.archiveTimeline) == 0 || co.archiveTimeline[len(co.archiveTimeline)-1].Year != y {
|
|
co.archiveTimeline = append(co.archiveTimeline, archiveYear{Year: y})
|
|
}
|
|
yr := &co.archiveTimeline[len(co.archiveTimeline)-1]
|
|
yr.Count++
|
|
if len(yr.Months) == 0 || yr.Months[len(yr.Months)-1].Name != m {
|
|
yr.Months = append(yr.Months, archiveMonth{Name: m})
|
|
}
|
|
mo := &yr.Months[len(yr.Months)-1]
|
|
mo.Docs = append(mo.Docs, d)
|
|
}
|
|
}
|
|
|
|
func (co *Corpus) renderArchiveBrowse(w http.ResponseWriter, r *http.Request, q url.Values) {
|
|
prefs := readPrefs(r)
|
|
view := q.Get("view")
|
|
|
|
data := &browsePage{
|
|
Base: pageWith(prefs),
|
|
Collection: collectionArchive,
|
|
CollectionCounts: co.archive.CollectionCounts,
|
|
SIDTodayCount: co.archive.SIDTodayCount,
|
|
SeriesList: co.seriesBySize[prefs.IncludeFOIA],
|
|
}
|
|
co.addFacetNav(&data.Base, prefs)
|
|
|
|
if view == "card" || view == "table" {
|
|
docs := append([]*Document(nil), co.sidtodayDocs...)
|
|
params := url.Values{"collection": {collectionArchive}}
|
|
data.listView = co.buildListView(q, prefs, docs, "/browse", params, "date", true)
|
|
data.MarkReported, data.Cards.MarkReported = true, true
|
|
if data.RowsHTML != "" {
|
|
data.RowsHTML = co.buildDocRows(data.Docs, true)
|
|
}
|
|
render(w, r, data)
|
|
return
|
|
}
|
|
|
|
data.Total = co.archive.SIDTodayCount
|
|
data.View = view
|
|
data.ArchiveYears = co.archiveTimeline
|
|
data.TimelineHTML = co.preHTML("archive-timeline")
|
|
render(w, r, data)
|
|
}
|
|
|
|
func docPath(id string) string { return "/doc/" + url.PathEscape(id) }
|
|
|
|
func filePath(id, name string) string {
|
|
return "/files/" + url.PathEscape(id) + "/" + url.PathEscape(name)
|
|
}
|
|
|
|
func versionPath(id, slug string) string {
|
|
return docPath(id) + "/versions/" + url.PathEscape(slug)
|
|
}
|
|
|
|
func versionFilePath(id, slug, name string) string {
|
|
return "/files/" + url.PathEscape(id) + "/versions/" + url.PathEscape(slug) + "/" + url.PathEscape(name)
|
|
}
|
|
|
|
func imageBase(id, slug string) string {
|
|
if slug == "" {
|
|
return "/files/" + url.PathEscape(id) + "/images/"
|
|
}
|
|
return "/files/" + url.PathEscape(id) + "/versions/" + url.PathEscape(slug) + "/images/"
|
|
}
|
|
|
|
func (co *Corpus) publishedTogether(doc *Document) []*Document {
|
|
var out []*Document
|
|
seen := map[string]bool{doc.ID: true}
|
|
for _, p := range doc.Publication {
|
|
if p.Link == "" {
|
|
continue
|
|
}
|
|
a := co.articleByLink[p.Link]
|
|
if a == nil {
|
|
continue
|
|
}
|
|
for _, id := range a.DocIDs {
|
|
if seen[id] {
|
|
continue
|
|
}
|
|
seen[id] = true
|
|
if d := co.docIndex[id]; d != nil {
|
|
out = append(out, d)
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ogImageAbs is the document's first page image, or its thumbnail when it
|
|
// has no page images.
|
|
func ogImageAbs(r *http.Request, doc *Document) string {
|
|
switch {
|
|
case len(doc.Images) > 0:
|
|
return requestBase(r) + imageBase(doc.ID, "") + url.PathEscape(doc.Images[0])
|
|
case doc.HasThumb:
|
|
return requestBase(r) + filePath(doc.ID, "thumb.jpg")
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// queryBase is path plus q, ready for one more parameter to be appended.
|
|
func queryBase(path string, q url.Values) string {
|
|
if enc := q.Encode(); enc != "" {
|
|
return path + "?" + enc + "&"
|
|
}
|
|
return path + "?"
|
|
}
|
|
|
|
func (co *Corpus) handleDocPages(w http.ResponseWriter, r *http.Request) {
|
|
id := r.PathValue("id")
|
|
doc := co.docIndex[id]
|
|
if doc == nil {
|
|
notFound(w, r)
|
|
return
|
|
}
|
|
prefs := readPrefs(r)
|
|
render(w, r, &pagesPage{
|
|
Base: pageWith(prefs),
|
|
viewer: viewer{Images: doc.Images, Sizes: doc.ImageSizes, ImageBase: imageBase(doc.ID, "")},
|
|
Doc: doc,
|
|
TextOnly: prefs.TextOnly,
|
|
})
|
|
}
|
|
|
|
func (co *Corpus) handleDocView(w http.ResponseWriter, r *http.Request) {
|
|
id := r.PathValue("id")
|
|
|
|
doc := co.docIndex[id]
|
|
if doc == nil {
|
|
notFound(w, r)
|
|
return
|
|
}
|
|
|
|
together := co.publishedTogether(doc)
|
|
shownTogether := together
|
|
var moreArticles []*Article
|
|
if len(together) > pubTogetherCap {
|
|
shownTogether = together[:pubTogetherCap]
|
|
moreArticles = co.docArticles(doc)
|
|
}
|
|
|
|
prefs := readPrefs(r)
|
|
scope := pageScope(nil, prefs, false).including(doc.Collection)
|
|
var relatedDocs []*Document
|
|
for _, relID := range co.archive.RelatedDocs[id] {
|
|
if related := co.docIndex[relID]; related != nil && scope.shows(related) {
|
|
relatedDocs = append(relatedDocs, related)
|
|
}
|
|
}
|
|
|
|
if prefs.TrackRecent && !crossSiteFetch(r) {
|
|
pushRecentDoc(w, r, id)
|
|
}
|
|
|
|
var serialPrevDoc, serialNextDoc *Document
|
|
if doc.Type == "SIDTODAY" {
|
|
if id, ok := co.serialPrev[doc.ID]; ok {
|
|
serialPrevDoc = co.docIndex[id]
|
|
}
|
|
if id, ok := co.serialNext[doc.ID]; ok {
|
|
serialNextDoc = co.docIndex[id]
|
|
}
|
|
}
|
|
|
|
embedBase := requestBase(r)
|
|
absURL := embedBase + docPath(doc.ID)
|
|
pdfURL := embedBase + filePath(doc.ID, doc.PDFFile)
|
|
|
|
render(w, r, &docPage{
|
|
Base: pageWith(prefs),
|
|
viewer: viewer{
|
|
Images: doc.Images,
|
|
Sizes: doc.ImageSizes,
|
|
ImageBase: imageBase(doc.ID, ""),
|
|
Content: doc.HTMLContent,
|
|
},
|
|
Doc: doc,
|
|
BrowseQS: scope.suffix(),
|
|
OGImageURL: ogImageAbs(r, doc),
|
|
SeriesGroup: co.seriesIndex[valueKey(doc.DocMeta.Series.Name)],
|
|
Assemblies: co.assembliesByDoc[doc.ID],
|
|
DocAbsURL: absURL,
|
|
Embeds: buildDocEmbeds(doc, absURL, pdfURL),
|
|
SerialPrev: serialPrevDoc,
|
|
SerialNext: serialNextDoc,
|
|
PublishedTogether: together,
|
|
PublishedShown: len(shownTogether),
|
|
PublishedMore: moreArticles,
|
|
RelatedDocs: relatedDocs,
|
|
PublishedCards: newCardView(prefs, shownTogether),
|
|
RelatedCards: newCardView(prefs, relatedDocs),
|
|
Pinned: isPinned(r, id),
|
|
})
|
|
}
|
|
|
|
func sortToken(s string) string {
|
|
for i := 0; i < len(s); i++ {
|
|
if s[i] < 'a' || s[i] > 'z' {
|
|
return ""
|
|
}
|
|
}
|
|
return s
|
|
}
|
|
|
|
// firstLetter returns the uppercase A-Z bucket for a facet value, folding
|
|
// accents first ("Çağlar" -> "C"), with "#" for anything non-alphabetic.
|
|
func firstLetter(s string) string {
|
|
s = strings.ToUpper(strings.TrimSpace(foldAccents(s)))
|
|
if s != "" && s[0] >= 'A' && s[0] <= 'Z' {
|
|
return string(s[0])
|
|
}
|
|
return "#"
|
|
}
|
|
|
|
// TODO split this huge ass file to separate files, browse_cateogry etc?
|
|
|
|
// category describes one browsable dimension: its URL slug, the two page
|
|
// headings, and the search field code its values map to
|
|
type category struct {
|
|
Slug string
|
|
Title string // list page e.g. "Browse by Programs"
|
|
Label string // value page e.g. "Program: XKEYSCORE"
|
|
Field string // search field code (fqf), "" for date categories
|
|
DateParam string // "date_doc" / "date_pub" for the date categories, else ""
|
|
Blurb string
|
|
}
|
|
|
|
func (c *category) isDate() bool { return c.DateParam != "" }
|
|
|
|
// facet = category's corpus counts (value -> documents)
|
|
func (c *category) Facet(co *Corpus) map[string]int { return co.archive.Facets[c.Slug] }
|
|
|
|
var categories = []category{
|
|
{"program", "Programs", "Program", "PR", "",
|
|
"Codenames and programs recorded for each document."},
|
|
{"journalist", "Journalists", "Journalist", "RE", "",
|
|
"Journalists bylined on the stories that published these documents. Contributors credited without a byline are searchable but not listed here."},
|
|
{"agency", "Agencies", "Agency", "CR", "",
|
|
"The agency that WROTE each document."},
|
|
{"publisher", "Publishers", "Publisher", "PU", "",
|
|
"Who published each document. Mostly a news outlet, but sometimes a book. A partner outlet that reported the story without publishing the documents keeps its byline and its article, but is not a publisher here."},
|
|
{"country", "Countries", "Country", "CO", "",
|
|
"Countries mentioned in each document. A country may be mentioned directly (e.g., France) or inferred from a related reference, such as a city (Paris -> France) or a domain name (.fr -> France)."},
|
|
{"people", "People", "Person", "PE", "",
|
|
"People who appear in these documents, named in the text or recognised in an image. An appearance implies nothing about the person beyond that appearance."},
|
|
{"classification", "Classification Levels", "Classification", "CL", "",
|
|
"Each document's classification level, as component values. If a document contains multiple classifications, the highest is used here."},
|
|
{"dissemination", "Dissemination Controls", "Dissemination control", "DI", "",
|
|
"Handling controls assessed for each document, as component values. If a document contains multiple disseminations, the most restrictive is used here."},
|
|
{"banner", "Classification Banners", "Banner", "BA", "",
|
|
"The marking line each document prints."},
|
|
{"genre", "Genres", "Genre", "GE", "",
|
|
"What kind of document it is: slides, report, wiki, manual, guide, newsletter."},
|
|
{"series", "Document Series", "Series", "SE", "",
|
|
"Recurring columns and multi-part runs in SIDtoday documents. A series lists its parts in the order the documents print. A missing part shows as a gap."},
|
|
{"published", "Publication Years", "Publication year", "", "date_pub",
|
|
"When a publisher first published each document. If a document is published multiple times, only the oldest one is used here."},
|
|
{"docdate", "Document Dates", "Document year", "", "date_doc",
|
|
"When each document was written, from its own dateline. Undated documents don't appear here."},
|
|
{"tag", "Tags", "Tag", "TA", "",
|
|
"Free-text subjects the curators added. Unlike the other axes these are not necessarily from the documents' own vocabulary, so they are the loosest thing to browse by (and the most forgiving)."},
|
|
}
|
|
|
|
// pubTogetherCap = on doc view, how many published together docs are shown, max
|
|
const pubTogetherCap = 10
|
|
|
|
func (co *Corpus) docArticles(doc *Document) []*Article {
|
|
var out []*Article
|
|
for _, p := range doc.Publication {
|
|
a := co.articleByLink[p.Link]
|
|
if a != nil && len(a.DocIDs) > 1 && !slices.Contains(out, a) {
|
|
out = append(out, a)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func categoryValueURL(slug, value string) string {
|
|
return "/browse/" + slug + "/" + url.PathEscape(value)
|
|
}
|
|
|
|
func inCollections(d *Document, sel map[string]bool) bool {
|
|
return sel == nil || sel[d.Collection]
|
|
}
|
|
|
|
func filterByCollection(docs []*Document, sel map[string]bool) []*Document {
|
|
if sel == nil {
|
|
return docs
|
|
}
|
|
out := make([]*Document, 0, len(docs))
|
|
for _, d := range docs {
|
|
if inCollections(d, sel) {
|
|
out = append(out, d)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func categoryBySlug(slug string) *category {
|
|
for i := range categories {
|
|
if categories[i].Slug == slug {
|
|
return &categories[i]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
const azMinEntries = 50
|
|
|
|
func (co *Corpus) prerenderCategoryRows() {
|
|
t := tmplSet["category.html"]
|
|
if t == nil {
|
|
return
|
|
}
|
|
exec := func(block string, data interface{}) (template.HTML, bool) {
|
|
var buf bytes.Buffer
|
|
if err := t.ExecuteTemplate(&buf, block, data); err != nil {
|
|
warnf("prerender category %s: %v - rendering rows per request", block, err)
|
|
return "", false
|
|
}
|
|
return template.HTML(buf.String()), true
|
|
}
|
|
co.categoryRowHTML = make(map[string][]template.HTML, len(co.categoryTables))
|
|
co.categoryRowSize = make(map[string]int, len(co.categoryTables))
|
|
co.azRowHTML = map[string]template.HTML{}
|
|
for ci := range categories {
|
|
for _, key := range []string{categories[ci].Slug, noFOIAKey(categories[ci].Slug)} {
|
|
entries := co.categoryTables[key].Entries
|
|
rows := make([]template.HTML, len(entries))
|
|
for i, e := range entries {
|
|
h, ok := exec("category-row", e)
|
|
if !ok {
|
|
return
|
|
}
|
|
rows[i] = h
|
|
co.categoryRowSize[key] += len(h)
|
|
if _, seen := co.azRowHTML[e.FirstLetter]; !seen {
|
|
if h, ok := exec("category-az-row", e.FirstLetter); ok {
|
|
co.azRowHTML[e.FirstLetter] = h
|
|
}
|
|
}
|
|
}
|
|
co.categoryRowSize[key] += 26 * 64
|
|
co.categoryRowHTML[key] = rows
|
|
}
|
|
}
|
|
}
|
|
|
|
func (co *Corpus) prerenderDocRows() {
|
|
t := tmplSet["browse.html"]
|
|
if t == nil {
|
|
return
|
|
}
|
|
rows := make(map[string]template.HTML, len(co.archive.Docs))
|
|
marked := map[string]template.HTML{}
|
|
var buf bytes.Buffer
|
|
render := func(row docTableRow) (template.HTML, bool) {
|
|
buf.Reset()
|
|
if err := t.ExecuteTemplate(&buf, "doc-table-row", row); err != nil {
|
|
warnf("prerender doc-table-row: %v - rendering table rows per request", err)
|
|
return "", false
|
|
}
|
|
return template.HTML(buf.String()), true
|
|
}
|
|
for i := range co.archive.Docs {
|
|
d := &co.archive.Docs[i]
|
|
row, ok := render(docTableRow{Document: d})
|
|
if !ok {
|
|
return
|
|
}
|
|
rows[d.ID] = row
|
|
if d.ReportedIssue() {
|
|
if marked[d.ID], ok = render(docTableRow{d, true}); !ok {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
co.docRowHTML, co.docRowMarkedHTML = rows, marked
|
|
}
|
|
|
|
// buildDocRows joins the documents' pre-rendered table rows; markReported
|
|
// takes the rows that badge an individually reported SIDtoday issue.
|
|
func (co *Corpus) buildDocRows(docs []*Document, markReported bool) template.HTML {
|
|
rows := make([]template.HTML, len(docs))
|
|
n := 0
|
|
for i, d := range docs {
|
|
row, ok := co.docRowMarkedHTML[d.ID]
|
|
if !markReported || !ok {
|
|
row, ok = co.docRowHTML[d.ID]
|
|
}
|
|
if !ok {
|
|
return ""
|
|
}
|
|
rows[i] = row
|
|
n += len(row)
|
|
}
|
|
var b strings.Builder
|
|
b.Grow(n)
|
|
for _, row := range rows {
|
|
b.WriteString(string(row))
|
|
}
|
|
return template.HTML(b.String())
|
|
}
|
|
|
|
func azID(letter string) string {
|
|
if letter == "#" {
|
|
return "other"
|
|
}
|
|
return letter
|
|
}
|
|
|
|
func (co *Corpus) buildCategoryRows(key string, t *categoryTable, order []int32, showAZ bool) template.HTML {
|
|
rows := co.categoryRowHTML[key]
|
|
if len(rows) != len(t.Entries) {
|
|
return ""
|
|
}
|
|
var b strings.Builder
|
|
b.Grow(co.categoryRowSize[key])
|
|
prevL := ""
|
|
for _, n := range order {
|
|
if l := t.Entries[n].FirstLetter; showAZ && l != prevL {
|
|
b.WriteString(string(co.azRowHTML[l]))
|
|
prevL = l
|
|
}
|
|
b.WriteString(string(rows[n]))
|
|
}
|
|
return template.HTML(b.String())
|
|
}
|
|
|
|
func (co *Corpus) handleCategoryList(w http.ResponseWriter, r *http.Request) {
|
|
cat := r.PathValue("category")
|
|
q := r.URL.Query()
|
|
sortBy := sortToken(q.Get("sort"))
|
|
order := sortToken(q.Get("order"))
|
|
|
|
c := categoryBySlug(cat)
|
|
if c == nil {
|
|
notFound(w, r)
|
|
return
|
|
}
|
|
|
|
prefs := readPrefs(r)
|
|
collCols := cat != "series"
|
|
listSort := "name"
|
|
switch {
|
|
case sortBy == "count":
|
|
listSort = sortBy
|
|
case !collCols:
|
|
case sortBy == "reported", sortBy == "sidtoday":
|
|
listSort = sortBy
|
|
case sortBy == "foia":
|
|
if prefs.IncludeFOIA {
|
|
listSort = sortBy
|
|
}
|
|
}
|
|
desc := order == "desc"
|
|
if listSort != "name" {
|
|
desc = order != "asc"
|
|
}
|
|
key := readerListKey(cat, prefs)
|
|
table := co.categoryTables[key]
|
|
rowOrder := table.order(listSort, desc)
|
|
|
|
showAZ := listSort == "name" && len(rowOrder) >= azMinEntries
|
|
var letters []string
|
|
if showAZ {
|
|
letters = table.letters[desc]
|
|
}
|
|
|
|
collSort := listSort == "reported" || listSort == "sidtoday" || listSort == "foia"
|
|
|
|
nameCol := "NAME"
|
|
if c.isDate() {
|
|
nameCol = "YEAR"
|
|
}
|
|
|
|
cols := []sortCol{
|
|
{Key: "name", Label: nameCol},
|
|
{Key: "count", Label: "TOTAL", Class: "num-col", DescFirst: true},
|
|
}
|
|
if collCols {
|
|
cols = append(cols,
|
|
sortCol{Key: "reported", Label: "REPORTED", Class: "num-col coll-col", DescFirst: true},
|
|
sortCol{Key: "sidtoday", Label: "SIDTODAY", Class: "num-col coll-col", DescFirst: true})
|
|
}
|
|
if collCols && prefs.IncludeFOIA {
|
|
cols = append(cols, sortCol{Key: "foia", Label: "FOIA", Class: "num-col coll-col", DescFirst: true})
|
|
}
|
|
|
|
data := &categoryPage{
|
|
Base: pageWith(prefs),
|
|
Title: c.Title,
|
|
Blurb: c.Blurb,
|
|
Heads: sortHeads("?", cols, listSort, desc),
|
|
RowsHTML: co.buildCategoryRows(key, table, rowOrder, showAZ),
|
|
CollCols: collCols,
|
|
ShowCollCols: collCols && (prefs.ShowCollCols || collSort),
|
|
ShowAZ: showAZ,
|
|
Letters: letters,
|
|
}
|
|
data.ActiveCategory = cat
|
|
if data.RowsHTML == "" {
|
|
data.Entries = table.ordered(rowOrder)
|
|
}
|
|
co.addFacetNav(&data.Base, prefs)
|
|
|
|
switch cat {
|
|
case "classification", "dissemination":
|
|
data.SeeAlso = []seeAlsoLink{{"/browse/banner", "full classification banners (verbatim marking lines)"}}
|
|
case "banner":
|
|
data.SeeAlso = []seeAlsoLink{
|
|
{"/browse/classification", "classification components"},
|
|
{"/browse/dissemination", "dissemination controls"},
|
|
}
|
|
}
|
|
|
|
render(w, r, data)
|
|
}
|
|
|
|
// categoryEntry = one row of a /browse/{category} list page
|
|
type categoryEntry struct {
|
|
Name string
|
|
URL template.URL // /browse/{category}/{name}, path-escaped
|
|
Note string // the agency expansion on the agency list, else ""
|
|
Count int // total, all collections
|
|
News int // type REPORTED (categoryTypeCounts), any collection
|
|
Archive int // type SIDTODAY, incl. individually reported issues
|
|
FOIA int // type FOIA
|
|
NonFOIA int // in the Reported and SIDtoday collections: the total when FOIA releases are left out
|
|
NoFOIA bool // a row of the FOIA-less table: no FOIA column
|
|
FirstLetter string
|
|
fold string // accent-folded lowercase name: the name sort key
|
|
}
|
|
|
|
// categoryTable = one category's list page
|
|
type categoryTable struct {
|
|
Entries []categoryEntry // by name, ascending
|
|
orders map[string][]int32 // categoryListKey -> indices into Entries, in that order
|
|
letters map[bool][]string // the two name orderings' A-Z letters as they come, by desc
|
|
}
|
|
|
|
func (t *categoryTable) order(sortBy string, desc bool) []int32 {
|
|
return t.orders[categoryListKey(sortBy, desc)]
|
|
}
|
|
|
|
func (t *categoryTable) ordered(order []int32) []categoryEntry {
|
|
out := make([]categoryEntry, len(order))
|
|
for i, n := range order {
|
|
out[i] = t.Entries[n]
|
|
}
|
|
return out
|
|
}
|
|
|
|
// noFOIAKey is where a category's FOIA-less table is kept in categoryLists
|
|
// and categoryRowHTML.
|
|
func noFOIAKey(slug string) string { return slug + "|nofoia" }
|
|
|
|
// readerListKey is the table a reader's category list comes from: the
|
|
// FOIA-less one unless they include FOIA releases.
|
|
func readerListKey(slug string, p Prefs) string {
|
|
if p.IncludeFOIA {
|
|
return slug
|
|
}
|
|
return noFOIAKey(slug)
|
|
}
|
|
|
|
// newCategoryTable sorts entries by name and works out the orderings the
|
|
// sorts ask for, ascending and descending.
|
|
func newCategoryTable(entries []categoryEntry, sorts ...string) *categoryTable {
|
|
sortCategoryEntries(entries, "name", false)
|
|
t := &categoryTable{Entries: entries, orders: map[string][]int32{}, letters: map[bool][]string{}}
|
|
for _, sortBy := range sorts {
|
|
for _, desc := range []bool{false, true} {
|
|
order := make([]int32, len(entries))
|
|
for i := range order {
|
|
order[i] = int32(i)
|
|
}
|
|
sort.Slice(order, func(i, j int) bool {
|
|
return categoryLess(&entries[order[i]], &entries[order[j]], sortBy, desc)
|
|
})
|
|
t.orders[categoryListKey(sortBy, desc)] = order
|
|
if sortBy != "name" {
|
|
continue
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, n := range order {
|
|
if l := entries[n].FirstLetter; !seen[l] {
|
|
seen[l] = true
|
|
t.letters[desc] = append(t.letters[desc], l)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return t
|
|
}
|
|
|
|
func categoryListKey(sortBy string, desc bool) string {
|
|
if desc {
|
|
return sortBy + "/desc"
|
|
}
|
|
return sortBy + "/asc"
|
|
}
|
|
|
|
func (co *Corpus) buildCategoryLists() {
|
|
co.categoryTables = make(map[string]*categoryTable, 2*len(categories))
|
|
co.categoryValueCounts = map[bool]map[string]int{false: {}, true: {}}
|
|
for ci := range categories {
|
|
c := &categories[ci]
|
|
byType := co.categoryTypeCounts[c.Slug]
|
|
byColl := co.archive.FacetsByColl[c.Slug]
|
|
facet := c.Facet(co)
|
|
base := make([]categoryEntry, 0, len(facet))
|
|
for name, count := range facet {
|
|
e := categoryEntry{
|
|
Name: name,
|
|
URL: template.URL(categoryValueURL(c.Slug, name)),
|
|
Count: count,
|
|
FirstLetter: firstLetter(name),
|
|
fold: normFoldString(name),
|
|
}
|
|
if c.Slug == "agency" {
|
|
e.Note = agencyName(name)
|
|
}
|
|
if tc := byType[name]; tc != nil {
|
|
e.News, e.Archive, e.FOIA = tc.News, tc.Archive, tc.FOIA
|
|
}
|
|
if cc := byColl[name]; cc != nil {
|
|
e.NonFOIA = cc.News + cc.Archive
|
|
}
|
|
base = append(base, e)
|
|
}
|
|
noFOIA := make([]categoryEntry, 0, len(base))
|
|
for i := range base {
|
|
e := base[i]
|
|
if e.NonFOIA > 0 {
|
|
e.Count = e.NonFOIA
|
|
e.NoFOIA = true
|
|
noFOIA = append(noFOIA, e)
|
|
}
|
|
}
|
|
co.categoryTables[c.Slug] = newCategoryTable(base, "name", "count", "reported", "sidtoday", "foia")
|
|
co.categoryTables[noFOIAKey(c.Slug)] = newCategoryTable(noFOIA, "name", "count", "reported", "sidtoday")
|
|
co.categoryValueCounts[true][c.Slug] = len(base)
|
|
co.categoryValueCounts[false][c.Slug] = len(noFOIA)
|
|
}
|
|
co.seriesBySize = map[bool][]categoryEntry{}
|
|
for _, foia := range []bool{false, true} {
|
|
t := co.categoryTables[readerListKey("series", Prefs{IncludeFOIA: foia})]
|
|
co.seriesBySize[foia] = t.ordered(t.order("count", true))
|
|
}
|
|
}
|
|
|
|
func sortCategoryEntries(entries []categoryEntry, sortBy string, desc bool) {
|
|
sort.Slice(entries, func(i, j int) bool {
|
|
return categoryLess(&entries[i], &entries[j], sortBy, desc)
|
|
})
|
|
}
|
|
|
|
// categoryLess orders two rows of a category table: by the sorted column in
|
|
// the direction asked, then the larger total first, then by name.
|
|
func categoryLess(a, b *categoryEntry, sortBy string, desc bool) bool {
|
|
col := func(e *categoryEntry) int {
|
|
switch sortBy {
|
|
case "count":
|
|
return e.Count
|
|
case "reported":
|
|
return e.News
|
|
case "sidtoday":
|
|
return e.Archive
|
|
case "foia":
|
|
return e.FOIA
|
|
}
|
|
return 0
|
|
}
|
|
byName := sortBy == "name"
|
|
if !byName {
|
|
if x, y := col(a), col(b); x != y {
|
|
if desc {
|
|
return x > y
|
|
}
|
|
return x < y
|
|
}
|
|
if a.Count != b.Count {
|
|
return a.Count > b.Count
|
|
}
|
|
}
|
|
if a.fold != b.fold {
|
|
if byName && desc {
|
|
return a.fold > b.fold
|
|
}
|
|
return a.fold < b.fold
|
|
}
|
|
return a.Name < b.Name
|
|
}
|
|
|
|
type seeAlsoLink struct {
|
|
URL string
|
|
Text string
|
|
}
|
|
|
|
type timelineBar struct {
|
|
Year string
|
|
Count int
|
|
Pct int
|
|
URL string
|
|
}
|
|
|
|
func listSortKey(vals []string) string {
|
|
best := ""
|
|
for _, v := range vals {
|
|
lv := strings.ToLower(strings.TrimSpace(v))
|
|
if lv != "" && (best == "" || lv < best) {
|
|
best = lv
|
|
}
|
|
}
|
|
return best
|
|
}
|
|
|
|
var classAlias = map[string]string{
|
|
"ts": "top secret", "s": "secret", "c": "confidential", "u": "unclassified",
|
|
"strap1": "strap 1", "strap2": "strap 2", "talent keyhole": "tk",
|
|
}
|
|
|
|
func foldClass(v string) string {
|
|
lv := strings.ToLower(strings.TrimSpace(v))
|
|
if a, ok := classAlias[lv]; ok {
|
|
return a
|
|
}
|
|
return lv
|
|
}
|
|
|
|
// classSortKey ranks a marking set per the curators' order: level first
|
|
// (TOP SECRET > SECRET > CONFIDENTIAL > UNCLASSIFIED), a level WITH SI
|
|
// above the bare level, and the restrictive compartments raising rank
|
|
// cumulatively - STRAP 2 (weight 4) > STRAP 1 (2), TK and MR (1 each) -
|
|
// so TOP SECRET SI TK outranks TOP SECRET SI.
|
|
func classSortKey(vals []string) string {
|
|
if len(vals) == 0 {
|
|
return ""
|
|
}
|
|
level, si, raise := 0, 0, 0
|
|
for _, v := range vals {
|
|
switch foldClass(v) {
|
|
case "top secret":
|
|
if level < 4 {
|
|
level = 4
|
|
}
|
|
case "secret":
|
|
if level < 3 {
|
|
level = 3
|
|
}
|
|
case "confidential":
|
|
if level < 2 {
|
|
level = 2
|
|
}
|
|
case "unclassified":
|
|
if level < 1 {
|
|
level = 1
|
|
}
|
|
case "si", "comint":
|
|
si = 1
|
|
case "strap 2":
|
|
raise += 4
|
|
case "strap 1":
|
|
raise += 2
|
|
case "tk", "mr":
|
|
raise++
|
|
}
|
|
}
|
|
if level == 0 && si == 0 && raise == 0 {
|
|
return ""
|
|
}
|
|
r := strconv.Itoa(raise)
|
|
if len(r) < 2 {
|
|
r = "0" + r
|
|
}
|
|
return strconv.Itoa(level) + strconv.Itoa(si) + r
|
|
}
|
|
|
|
// TODO this file really needs the split.
|
|
|
|
// searchClause is one row of the search form: the field code, the value as
|
|
// typed into it and how the row joins the ones before it.
|
|
type searchClause struct{ Conj, Field, Value string }
|
|
|
|
// searchURL builds an executed /search link
|
|
func searchURL(clauses []searchClause, ranges ...[2]string) string {
|
|
q := url.Values{}
|
|
q.Set("do_search", "1")
|
|
q.Set("action", "search")
|
|
for _, c := range clauses {
|
|
conj := c.Conj
|
|
if conj == "" {
|
|
conj = "and"
|
|
}
|
|
q.Add("fqc", conj)
|
|
q.Add("fqv", c.Value)
|
|
q.Add("fqf", c.Field)
|
|
}
|
|
for _, r := range ranges {
|
|
q.Set(r[0]+"_start", r[1])
|
|
q.Set(r[0]+"_end", r[1])
|
|
}
|
|
return "/search?" + q.Encode()
|
|
}
|
|
|
|
func dateRangeSearchURL(fqf, fqv string, ranges ...[2]string) string {
|
|
return searchURL([]searchClause{{Field: fqf, Value: fqv}}, ranges...)
|
|
}
|
|
|
|
func appendClause(base, field, value string) string {
|
|
return base + "fqf=" + url.QueryEscape(field) +
|
|
"&fqv=" + url.QueryEscape(value) + "&fqc=and"
|
|
}
|
|
|
|
// entityTimelineBy is the per-year histogram behind both value-page
|
|
// timelines. docs are the page's documents, already narrowed to its
|
|
// collection scope, so every bar link carries that scope too.
|
|
// yearsOf returns a document's distinct years on the bar axis.
|
|
func entityTimelineBy(docs []*Document, c *category, val string, scope collScope, dateParam string, yearsOf func(*Document) []string) []timelineBar {
|
|
counts := map[string]int{}
|
|
for _, d := range docs {
|
|
for _, y := range yearsOf(d) {
|
|
counts[y]++
|
|
}
|
|
}
|
|
if len(counts) < 2 {
|
|
return nil
|
|
}
|
|
pageParam := ""
|
|
if c.isDate() && validDatePrefix(val) {
|
|
pageParam = c.DateParam
|
|
}
|
|
return yearBars(counts, func(y string) string {
|
|
switch {
|
|
case c.Field != "":
|
|
return scope.link(dateRangeSearchURL(c.Field, exactClause(val), [2]string{dateParam, y}))
|
|
case pageParam == "date_doc" && dateParam == "date_pub":
|
|
return scope.link(dateRangeSearchURL("TE", "", [2]string{pageParam, val}, [2]string{dateParam, y}))
|
|
}
|
|
return ""
|
|
})
|
|
}
|
|
|
|
func entityTimeline(docs []*Document, c *category, val string, scope collScope) []timelineBar {
|
|
return entityTimelineBy(docs, c, val, scope, "date_doc", func(d *Document) []string {
|
|
if y := docYear(d); y != "" {
|
|
return []string{y}
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func entityPubTimeline(docs []*Document, c *category, val string, scope collScope) []timelineBar {
|
|
switch c.Slug {
|
|
case "journalist", "publisher":
|
|
return entityTimelineBy(docs, c, val, scope, "date_pub", func(d *Document) []string {
|
|
return pubYearsMatching(d, c.Field, val)
|
|
})
|
|
}
|
|
return entityTimelineBy(docs, c, val, scope, "date_pub", pubYearsOf)
|
|
}
|
|
|
|
type monthNav struct {
|
|
Months []monthNavEntry
|
|
Year string
|
|
YearURL string
|
|
YearActive bool
|
|
Monthless int
|
|
}
|
|
|
|
func (co *Corpus) buildMonthNav(c *category, val string, allMatching []*Document, scope collScope) *monthNav {
|
|
cat := c.Slug
|
|
if !c.isDate() || !validDatePrefix(val) {
|
|
return nil
|
|
}
|
|
year := val[:4]
|
|
yearDocs := allMatching
|
|
if len(val) == 7 {
|
|
yearDocs = scope.docs(co.categoryMatches(cat, year))
|
|
}
|
|
counts := map[string]int{}
|
|
for _, d := range yearDocs {
|
|
ds := d.DocMeta.DateCreated
|
|
if cat == "published" {
|
|
ds = oldestPubDate(d)
|
|
}
|
|
if len(ds) >= 7 {
|
|
counts[ds[5:7]]++
|
|
}
|
|
}
|
|
var months []monthNavEntry
|
|
shown := 0
|
|
for m := 1; m <= 12; m++ {
|
|
mm := strconv.Itoa(m)
|
|
if m < 10 {
|
|
mm = "0" + mm
|
|
}
|
|
c := counts[mm]
|
|
if c == 0 {
|
|
continue
|
|
}
|
|
shown += c
|
|
months = append(months, monthNavEntry{
|
|
Label: monthNames[m][:3],
|
|
URL: scope.link(categoryValueURL(cat, year+"-"+mm)),
|
|
Count: c,
|
|
Active: len(val) == 7 && val[5:7] == mm,
|
|
})
|
|
}
|
|
if len(months) == 0 {
|
|
return nil
|
|
}
|
|
return &monthNav{
|
|
Months: months,
|
|
Year: year,
|
|
YearURL: scope.link(categoryValueURL(cat, year)),
|
|
YearActive: len(val) == 4,
|
|
Monthless: len(yearDocs) - shown,
|
|
}
|
|
}
|
|
|
|
// bdGroup = one of a value page's breakdown charts (its banners, its
|
|
// classification levels), each value a bar.
|
|
type bdGroup struct {
|
|
Label string
|
|
Values []bdVal
|
|
}
|
|
|
|
type bdVal struct {
|
|
Value string
|
|
Count int
|
|
URL string
|
|
Pct int
|
|
}
|
|
|
|
func (co *Corpus) handleCategoryFilter(w http.ResponseWriter, r *http.Request) {
|
|
cat := r.PathValue("category")
|
|
val := r.PathValue("value")
|
|
c := categoryBySlug(cat)
|
|
if c == nil {
|
|
notFound(w, r)
|
|
return
|
|
}
|
|
if name, ok := co.categoryNames[cat][valueKey(val)]; ok && name != val {
|
|
u := categoryValueURL(cat, name)
|
|
if r.URL.RawQuery != "" {
|
|
u += "?" + r.URL.RawQuery
|
|
}
|
|
http.Redirect(w, r, u, http.StatusMovedPermanently)
|
|
return
|
|
}
|
|
prefs := readPrefs(r)
|
|
q := r.URL.Query()
|
|
|
|
selectedTypeSet, typeFiltered := selectedTypes(q)
|
|
|
|
scope := pageScope(q, prefs, false)
|
|
|
|
everything := co.categoryMatches(cat, val)
|
|
allMatching := scope.docs(everything)
|
|
pubYear := ""
|
|
if y := q.Get("year"); cat == "publisher" && len(y) == 4 && allDigits(y) {
|
|
pubYear = y
|
|
allMatching = publishedBy(allMatching, val, y)
|
|
}
|
|
viewTypeCounts := typeCounts(allMatching)
|
|
if !typeFiltered {
|
|
selectedTypeSet = allTypesSelected(co.archive.Types)
|
|
}
|
|
results := append([]*Document(nil), allMatching...)
|
|
results = filterByType(results, selectedTypeSet)
|
|
|
|
basePath := categoryValueURL(cat, val)
|
|
params := url.Values{}
|
|
if typeFiltered {
|
|
params["type"] = q["type"]
|
|
}
|
|
if scope.colls != nil {
|
|
params["coll"] = scope.colls
|
|
}
|
|
if pubYear != "" {
|
|
params.Set("year", pubYear)
|
|
}
|
|
defSort, boost := "title", val
|
|
if cat == "series" {
|
|
defSort = "series"
|
|
}
|
|
if cat == "series" || c.isDate() {
|
|
boost = ""
|
|
}
|
|
data := &listPage{
|
|
Base: pageWith(prefs),
|
|
listView: co.buildListViewBoosted(q, prefs, results, basePath, params, defSort, false, boost),
|
|
}
|
|
|
|
searchLink := ""
|
|
if c.Field != "" {
|
|
searchLink = searchURL([]searchClause{
|
|
{Field: "TE"},
|
|
{Field: c.Field, Value: exactClause(val)},
|
|
})
|
|
} else if c.isDate() && validDatePrefix(val) {
|
|
searchLink = dateRangeSearchURL("TE", "", [2]string{c.DateParam, val})
|
|
}
|
|
|
|
docScope := scope
|
|
if pubYear != "" {
|
|
docScope = scope.and(url.Values{"date_pub_start": {pubYear}, "date_pub_end": {pubYear}}.Encode())
|
|
}
|
|
searchLink = docScope.link(searchLink)
|
|
|
|
if mn := co.buildMonthNav(c, val, allMatching, scope); mn != nil {
|
|
data.MonthNav = mn.Months
|
|
data.MonthNavYear = mn.Year
|
|
data.MonthNavYearURL = mn.YearURL
|
|
data.MonthNavYearActive = mn.YearActive
|
|
if mn.Monthless > 0 {
|
|
data.MonthlessCount = mn.Monthless
|
|
}
|
|
}
|
|
|
|
var gloss *GlossaryEntry
|
|
if cat == "program" {
|
|
gloss = co.glossaryLookup(val)
|
|
}
|
|
|
|
data.GlossaryEntry = gloss
|
|
if pubYear != "" {
|
|
data.YearScope = pubYear
|
|
data.YearScopeAllURL = scope.link(basePath)
|
|
}
|
|
refs := categoryRefinements(allMatching, c, val, docScope, pubYear)
|
|
data.Refinements = refs
|
|
|
|
var breakdowns []bdGroup
|
|
for _, g := range refs {
|
|
if g.Label != "Banner" && g.Label != "Classification" {
|
|
continue
|
|
}
|
|
top := 0
|
|
for _, v := range g.Values {
|
|
if v.Count > top {
|
|
top = v.Count
|
|
}
|
|
}
|
|
bg := bdGroup{Label: g.Label}
|
|
for _, v := range g.Values {
|
|
bg.Values = append(bg.Values, bdVal{v.Name, v.Count, v.URL, barPct(v.Count, top, 2)})
|
|
}
|
|
breakdowns = append(breakdowns, bg)
|
|
}
|
|
data.Breakdowns = breakdowns
|
|
data.Timeline = entityTimeline(allMatching, c, val, docScope)
|
|
if pubYear == "" {
|
|
// a year-scoped page has one publication year: no timeline
|
|
data.PubTimeline = entityPubTimeline(allMatching, c, val, scope)
|
|
}
|
|
co.addFacetNav(&data.Base, prefs)
|
|
data.Title = val
|
|
data.ActiveCategory = cat
|
|
data.Cards.ActiveCategory, data.Cards.ActiveValue = cat, val
|
|
data.SearchLink = searchLink
|
|
data.SelectedTypes = selectedTypeSet
|
|
data.AllTypes = viewTypeCounts
|
|
data.Params = q
|
|
data.CatLabel = c.Label
|
|
data.Blurb = c.Blurb
|
|
render(w, r, data)
|
|
}
|
|
|
|
// categoryRefinements builds the "combine with another filter" facet groups
|
|
// for a category value page. For the docs matching this page's value, tally
|
|
// the other browsable dimensions and link each value to a TWO-clause search
|
|
// (this page's value AND that value). On a publisher page scoped to pubYear
|
|
// the links carry that publication year, and the tallies read the
|
|
// journalists the way that search will (refineValues).
|
|
func categoryRefinements(docs []*Document, c *category, val string, scope collScope, pubYear string) []searchFacet {
|
|
if c.Field == "" || len(docs) < 2 {
|
|
return nil
|
|
}
|
|
base := scope.link(searchURL([]searchClause{{Field: c.Field, Value: exactClause(val)}})) + "&"
|
|
|
|
groups := []facetGroup{
|
|
{"Banner", "BA"}, {"Classification", "CL"}, {"Dissemination", "DI"}, {"Genre", "GE"},
|
|
{"Programs", "PR"}, {"Agencies", "CR"}, {"Countries", "CO"}, {"Journalists", "RE"},
|
|
}
|
|
kept := groups[:0]
|
|
for _, g := range groups {
|
|
if g.Code != c.Field {
|
|
kept = append(kept, g)
|
|
}
|
|
}
|
|
return facetGroups(docs, kept, base, refineValues(pubYear, pubYear))
|
|
}
|
|
|
|
func (co *Corpus) buildCategoryDocs() {
|
|
co.categoryDocs = make(map[string]map[string][]int32, len(categories))
|
|
co.categoryNames = make(map[string]map[string]string, len(categories))
|
|
co.archive.Facets = make(map[string]map[string]int, len(categories))
|
|
co.archive.FacetsByColl = make(map[string]map[string]*collCounts, len(categories))
|
|
co.categoryTypeCounts = make(map[string]map[string]*collCounts, len(categories))
|
|
for ci := range categories {
|
|
c := &categories[ci]
|
|
byValue := make(map[string][]int32)
|
|
name := make(map[string]string) // index key -> display value
|
|
add := func(key, value string, i int) {
|
|
if _, ok := name[key]; !ok {
|
|
name[key] = value
|
|
}
|
|
byValue[key] = append(byValue[key], int32(i))
|
|
}
|
|
for i := range co.archive.Docs {
|
|
d := &co.archive.Docs[i]
|
|
if c.isDate() {
|
|
date := d.DocMeta.DateCreated
|
|
if c.DateParam == "date_pub" {
|
|
date = oldestPubDate(d)
|
|
}
|
|
if len(date) >= 4 {
|
|
add(date[:4], date[:4], i)
|
|
}
|
|
if len(date) >= 7 {
|
|
add(date[:7], date[:7], i)
|
|
}
|
|
continue
|
|
}
|
|
for _, v := range facetValues(d, c.Field) {
|
|
add(valueKey(v), v, i)
|
|
}
|
|
}
|
|
counts := make(map[string]int)
|
|
byColl := make(map[string]*collCounts)
|
|
byType := make(map[string]*collCounts)
|
|
for key, docs := range byValue {
|
|
if c.isDate() && len(key) != 4 {
|
|
continue
|
|
}
|
|
cc, tc := &collCounts{}, &collCounts{}
|
|
for _, n := range docs {
|
|
cc.add(co.archive.Docs[n].Collection)
|
|
tc.addType(co.archive.Docs[n].Type)
|
|
}
|
|
counts[name[key]] = len(docs)
|
|
byColl[name[key]] = cc
|
|
byType[name[key]] = tc
|
|
}
|
|
co.categoryDocs[c.Slug] = byValue
|
|
co.categoryNames[c.Slug] = name
|
|
co.archive.Facets[c.Slug] = counts
|
|
co.archive.FacetsByColl[c.Slug] = byColl
|
|
co.categoryTypeCounts[c.Slug] = byType
|
|
}
|
|
}
|
|
|
|
// categoryMatches returns the documents on a /browse/{category}/{value}
|
|
// page, in corpus order, as a fresh slice of pointers the caller owns.
|
|
func (co *Corpus) categoryMatches(cat, val string) []*Document {
|
|
c := categoryBySlug(cat)
|
|
if c == nil {
|
|
return nil
|
|
}
|
|
if c.isDate() && !validDatePrefix(val) {
|
|
return nil
|
|
}
|
|
idx := co.categoryDocs[cat][valueKey(val)]
|
|
out := make([]*Document, len(idx))
|
|
for i, n := range idx {
|
|
out[i] = &co.archive.Docs[n]
|
|
}
|
|
return out
|
|
}
|
|
|
|
func validDatePrefix(val string) bool {
|
|
switch len(val) {
|
|
case 4:
|
|
return allDigits(val)
|
|
case 7:
|
|
return allDigits(val[:4]) && val[4] == '-' && allDigits(val[5:])
|
|
}
|
|
return false
|
|
}
|
|
|
|
func validDate(val string) bool {
|
|
if len(val) == 10 {
|
|
return validDatePrefix(val[:7]) && val[7] == '-' && allDigits(val[8:])
|
|
}
|
|
return validDatePrefix(val)
|
|
}
|
|
|
|
func allDigits(s string) bool {
|
|
if s == "" {
|
|
return false
|
|
}
|
|
for i := 0; i < len(s); i++ {
|
|
if s[i] < '0' || s[i] > '9' {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func publishedBy(docs []*Document, publisher, year string) []*Document {
|
|
var out []*Document
|
|
for _, d := range docs {
|
|
for _, p := range d.Publication {
|
|
if !p.ReportingOnly && sameValue(p.Publisher, publisher) &&
|
|
strings.HasPrefix(p.DatePublished, year) {
|
|
out = append(out, d)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func publisherYearURL(publisher, year string) string {
|
|
return categoryValueURL("publisher", publisher) + "?year=" + url.QueryEscape(year)
|
|
}
|
|
|
|
func oldestPubDate(d *Document) string {
|
|
oldest := ""
|
|
for _, p := range d.Publication {
|
|
if len(p.DatePublished) >= 4 && (oldest == "" || p.DatePublished < oldest) {
|
|
oldest = p.DatePublished
|
|
}
|
|
}
|
|
return oldest
|
|
}
|
|
|
|
func (d *Document) FirstPubYear() string {
|
|
if date := oldestPubDate(d); len(date) >= 4 {
|
|
return date[:4]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func docYear(d *Document) string {
|
|
if len(d.DocMeta.DateCreated) >= 4 {
|
|
return d.DocMeta.DateCreated[:4]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// pubYearsOf lists the years the document was published in, any entry's,
|
|
// in the entries' order.
|
|
func pubYearsOf(d *Document) []string {
|
|
if d.fields != nil {
|
|
return d.fields.pubYears
|
|
}
|
|
var out []string
|
|
for _, p := range d.Publication {
|
|
if len(p.DatePublished) < 4 {
|
|
continue
|
|
}
|
|
if y := p.DatePublished[:4]; !slices.Contains(out, y) {
|
|
out = append(out, y)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func pubYearsMatching(d *Document, field, val string) []string {
|
|
var out []string
|
|
for i := range d.Publication {
|
|
date := d.Publication[i].DatePublished
|
|
if len(date) < 4 || slices.Contains(out, date[:4]) {
|
|
continue
|
|
}
|
|
for _, v := range entryValues(d, i, field) {
|
|
if sameValue(v, val) {
|
|
out = append(out, date[:4])
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
type monthNavEntry struct {
|
|
Label string
|
|
URL string
|
|
Count int
|
|
Active bool
|
|
}
|
|
|
|
// docSort is one way to order a list of documents. The sort bar, the
|
|
// doc-table's column headers, the list pages, search and /api/docs all read
|
|
// this table, so a sort means the same thing everywhere.
|
|
type docSort struct {
|
|
Key string
|
|
Label string
|
|
Value func(d *Document) string
|
|
EmptiesLast bool
|
|
DescFirst bool
|
|
InBar bool // offered in the sort bar
|
|
}
|
|
|
|
// titleSortKey sorts a title by its first letter or digit
|
|
func titleSortKey(d *Document) string {
|
|
return strings.TrimLeftFunc(strings.ToLower(d.Title), func(r rune) bool {
|
|
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
|
})
|
|
}
|
|
|
|
var docSorts = []docSort{
|
|
{"title", "Title", titleSortKey, false, false, true},
|
|
{"agency", "Agency", func(d *Document) string { return strings.ToLower(d.DocMeta.Creator) }, false, false, true},
|
|
{"date", "Doc Date", func(d *Document) string { return d.DocMeta.DateCreated }, true, false, true},
|
|
{"pubdate", "Pub Date", oldestPubDate, true, false, true},
|
|
{"classification", "Classification", func(d *Document) string { return classSortKey(d.DocMeta.Classification) }, true, true, false},
|
|
{"dissemination", "Dissemination", func(d *Document) string { return listSortKey(d.DocMeta.Dissemination) }, true, false, false},
|
|
{"program", "Programs", func(d *Document) string { return listSortKey(d.DocMeta.Programs) }, true, false, false},
|
|
{"country", "Countries", func(d *Document) string { return listSortKey(d.DocMeta.Countries) }, true, false, false},
|
|
{"people", "People", func(d *Document) string { return listSortKey(d.DocMeta.People) }, true, false, false},
|
|
{"journalist", "Journalists", func(d *Document) string { return listSortKey(d.JournalistsDeduped()) }, true, false, false},
|
|
{"type", "Type", func(d *Document) string { return strings.ToLower(d.Type) }, false, false, false},
|
|
{"series", "Series order", func(d *Document) string {
|
|
if p := d.DocMeta.Series.Part; p > 0 {
|
|
return fmt.Sprintf("%06d", p)
|
|
}
|
|
return ""
|
|
}, true, false, false},
|
|
}
|
|
|
|
var docSortByKey = func() map[string]*docSort {
|
|
m := make(map[string]*docSort, len(docSorts))
|
|
for i := range docSorts {
|
|
m[docSorts[i].Key] = &docSorts[i]
|
|
}
|
|
return m
|
|
}()
|
|
|
|
var docTableCols = func() []sortCol {
|
|
col := func(key string) sortCol {
|
|
s := docSortByKey[key]
|
|
return sortCol{Key: s.Key, Label: s.Label, DescFirst: s.DescFirst}
|
|
}
|
|
return []sortCol{
|
|
col("title"), col("agency"), col("date"), col("pubdate"),
|
|
col("classification"), col("dissemination"), col("program"),
|
|
col("country"), col("people"), col("journalist"),
|
|
{Label: "Published"},
|
|
col("type"),
|
|
}
|
|
}()
|
|
|
|
type sortCol struct {
|
|
Key, Label, Class string
|
|
DescFirst, Fixed bool
|
|
}
|
|
|
|
type sortHead struct {
|
|
sortCol
|
|
URL string
|
|
Active, Desc bool
|
|
}
|
|
|
|
func sortHeads(base string, cols []sortCol, cur string, desc bool) []sortHead {
|
|
heads := make([]sortHead, len(cols))
|
|
for i, c := range cols {
|
|
h := sortHead{sortCol: c}
|
|
if c.Key != "" {
|
|
next := c.DescFirst
|
|
if h.Active = c.Key == cur; h.Active {
|
|
h.Desc = desc
|
|
if !c.Fixed {
|
|
next = !desc
|
|
}
|
|
}
|
|
h.URL = base + "sort=" + c.Key
|
|
if !c.Fixed {
|
|
h.URL += "&order=" + orderParam(next)
|
|
}
|
|
}
|
|
heads[i] = h
|
|
}
|
|
return heads
|
|
}
|
|
|
|
func orderParam(desc bool) string {
|
|
if desc {
|
|
return "desc"
|
|
}
|
|
return "asc"
|
|
}
|
|
|
|
// sortCmp orders two sort keys: in the direction asked, an empty key after
|
|
// the others either way where the sort puts its empties last.
|
|
func sortCmp(a, b string, desc, emptiesLast bool) int {
|
|
if emptiesLast && (a == "") != (b == "") {
|
|
if b == "" {
|
|
return -1
|
|
}
|
|
return 1
|
|
}
|
|
if desc {
|
|
return cmp.Compare(b, a)
|
|
}
|
|
return cmp.Compare(a, b)
|
|
}
|
|
|
|
// boostTitleMatches moves the documents whose title has word as a whole word,
|
|
// in any case, to the front, keeping each group's order e.g. on
|
|
// /browse/tag/Tor the documents about Tor come before the ones that only carry
|
|
// the tag.
|
|
func boostTitleMatches(docs []*Document, word string) {
|
|
re, err := regexp.Compile(`(?i)\b` + regexp.QuoteMeta(word) + `\b`)
|
|
if err != nil {
|
|
return
|
|
}
|
|
var hit, rest []*Document
|
|
for _, d := range docs {
|
|
if re.MatchString(d.Title) {
|
|
hit = append(hit, d)
|
|
} else {
|
|
rest = append(rest, d)
|
|
}
|
|
}
|
|
copy(docs, append(hit, rest...))
|
|
}
|
|
|
|
func sortDocs(docs []*Document, sortBy, order string) {
|
|
desc := order == "desc"
|
|
if docSortByKey[sortBy] == nil {
|
|
sortBy = "title"
|
|
}
|
|
|
|
copy(docs, sortByField(docs, func(d *Document) *Document { return d }, sortBy, desc))
|
|
}
|
|
|
|
func sortScored(scored []ScoredDoc, sortBy, order string) []ScoredDoc {
|
|
if docSortByKey[sortBy] == nil {
|
|
return scored
|
|
}
|
|
return sortByField(scored, func(s ScoredDoc) *Document { return s.Doc }, sortBy, order == "desc")
|
|
}
|
|
|
|
func sortByField[T any](items []T, doc func(T) *Document, sortBy string, desc bool) []T {
|
|
ds := docSortByKey[sortBy]
|
|
keys := make([]string, len(items))
|
|
for i := range items {
|
|
keys[i] = ds.Value(doc(items[i]))
|
|
}
|
|
idx := make([]int, len(items))
|
|
for i := range idx {
|
|
idx[i] = i
|
|
}
|
|
slices.SortStableFunc(idx, func(a, b int) int {
|
|
return sortCmp(keys[a], keys[b], desc, ds.EmptiesLast)
|
|
})
|
|
out := make([]T, len(items))
|
|
for i, o := range idx {
|
|
out[i] = items[o]
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (co *Corpus) docVersion(r *http.Request) (*Document, *DocVersion) {
|
|
doc := co.docIndex[r.PathValue("id")]
|
|
if doc == nil {
|
|
return nil, nil
|
|
}
|
|
return doc, doc.VersionFor(r.PathValue("slug"))
|
|
}
|
|
|
|
func (co *Corpus) handleVersionView(w http.ResponseWriter, r *http.Request) {
|
|
doc, v := co.docVersion(r)
|
|
if v == nil {
|
|
notFound(w, r)
|
|
return
|
|
}
|
|
pubs := doc.Publication[:0:0]
|
|
for _, i := range v.PubIndex {
|
|
pubs = append(pubs, doc.Publication[i])
|
|
}
|
|
prefs := readPrefs(r)
|
|
render(w, r, &versionPage{
|
|
Base: pageWith(prefs),
|
|
viewer: viewer{
|
|
Images: v.Images,
|
|
Sizes: v.ImageSizes,
|
|
ImageBase: imageBase(doc.ID, v.Slug),
|
|
Content: v.HTMLContent,
|
|
Version: v,
|
|
},
|
|
Doc: doc,
|
|
Publications: pubs,
|
|
PDFURL: versionFilePath(doc.ID, v.Slug, v.PDFFile),
|
|
})
|
|
}
|
|
|
|
func (co *Corpus) handleVersionPages(w http.ResponseWriter, r *http.Request) {
|
|
doc, v := co.docVersion(r)
|
|
if v == nil {
|
|
notFound(w, r)
|
|
return
|
|
}
|
|
prefs := readPrefs(r)
|
|
render(w, r, &pagesPage{
|
|
Base: pageWith(prefs),
|
|
viewer: viewer{Images: v.Images, Sizes: v.ImageSizes, ImageBase: imageBase(doc.ID, v.Slug), Version: v},
|
|
Doc: doc,
|
|
TextOnly: prefs.TextOnly,
|
|
})
|
|
}
|