377 lines
8.9 KiB
Go
377 lines
8.9 KiB
Go
package main
|
|
|
|
import (
|
|
"log"
|
|
"net/http"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// The Explore section (/explore).
|
|
|
|
type exploreCol struct {
|
|
Header string
|
|
Key string
|
|
Ratio bool
|
|
}
|
|
|
|
type exploreMetric struct {
|
|
Slug string
|
|
Title string
|
|
Blurb string
|
|
Cols []exploreCol
|
|
Vals func(d *Document) ([]float64, bool)
|
|
}
|
|
|
|
const exploreRows = 200 // rows shown per page ("top 200 of N")
|
|
|
|
var exploreMetrics = []exploreMetric{
|
|
{
|
|
Slug: "programs",
|
|
Title: "Most programs mentioned",
|
|
Blurb: "Documents referencing the most surveillance programs.",
|
|
Cols: []exploreCol{
|
|
{Header: "Programs", Key: "programs"},
|
|
{Header: "Pages", Key: "pages"},
|
|
{Header: "Programs / page", Key: "ratio", Ratio: true},
|
|
},
|
|
Vals: func(d *Document) ([]float64, bool) {
|
|
n := len(d.DocMeta.Programs)
|
|
if n == 0 {
|
|
return nil, false
|
|
}
|
|
pages, ratio := float64(d.Pages), -1.0
|
|
if d.Pages > 0 {
|
|
ratio = float64(n) / float64(d.Pages)
|
|
}
|
|
return []float64{float64(n), pages, ratio}, true
|
|
},
|
|
},
|
|
{
|
|
Slug: "people",
|
|
Title: "Most people mentioned",
|
|
Blurb: "Documents whose metadata names the most individuals.",
|
|
Cols: []exploreCol{{Header: "People", Key: "people"}},
|
|
Vals: func(d *Document) ([]float64, bool) {
|
|
n := len(d.DocMeta.People)
|
|
return []float64{float64(n)}, n > 0
|
|
},
|
|
},
|
|
{
|
|
Slug: "countries",
|
|
Title: "Most countries mentioned",
|
|
Blurb: "Documents spanning the most countries.",
|
|
Cols: []exploreCol{{Header: "Countries", Key: "countries"}},
|
|
Vals: func(d *Document) ([]float64, bool) {
|
|
n := len(d.DocMeta.Countries)
|
|
return []float64{float64(n)}, n > 0
|
|
},
|
|
},
|
|
{
|
|
Slug: "pages",
|
|
Title: "Longest documents",
|
|
Blurb: "Documents with the most pages.",
|
|
Cols: []exploreCol{{Header: "Pages", Key: "pages"}},
|
|
Vals: func(d *Document) ([]float64, bool) {
|
|
return []float64{float64(d.Pages)}, d.Pages > 0
|
|
},
|
|
},
|
|
}
|
|
|
|
func exploreMetricBySlug(slug string) *exploreMetric {
|
|
for i := range exploreMetrics {
|
|
if exploreMetrics[i].Slug == slug {
|
|
return &exploreMetrics[i]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (co *Corpus) handleExploreIndex(w http.ResponseWriter, r *http.Request) {
|
|
render(w, r, &explorePage{
|
|
Metrics: exploreMetrics,
|
|
HasAssemblies: len(co.assemblies) > 0,
|
|
})
|
|
}
|
|
|
|
type exploreRow struct {
|
|
Rank int
|
|
Doc Document
|
|
Vals []string
|
|
}
|
|
|
|
type exploreTable struct {
|
|
Rows []exploreRow
|
|
Total int
|
|
}
|
|
|
|
func exploreSet(slug string, foia bool) string {
|
|
if foia {
|
|
return slug
|
|
}
|
|
return slug + ":nofoia"
|
|
}
|
|
|
|
func (co *Corpus) buildExploreTables() {
|
|
co.exploreTables = make(map[string][]exploreTable, 2*len(exploreMetrics))
|
|
for mi := range exploreMetrics {
|
|
m := &exploreMetrics[mi]
|
|
if _, dup := co.exploreTables[m.Slug]; dup {
|
|
log.Fatalf("explore: duplicate metric slug %q", m.Slug)
|
|
}
|
|
for _, foia := range []bool{true, false} {
|
|
co.exploreTables[exploreSet(m.Slug, foia)] = co.buildExploreTable(m, foia)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (co *Corpus) buildExploreTable(m *exploreMetric, foia bool) []exploreTable {
|
|
type scored struct {
|
|
doc *Document
|
|
vals []float64
|
|
}
|
|
var rows []scored
|
|
for i := range co.archive.Docs {
|
|
d := &co.archive.Docs[i]
|
|
if !foia && d.Collection == collectionFOIA {
|
|
continue
|
|
}
|
|
if vals, ok := m.Vals(d); ok {
|
|
rows = append(rows, scored{d, vals})
|
|
}
|
|
}
|
|
|
|
tables := make([]exploreTable, len(m.Cols))
|
|
for sortIdx := range m.Cols {
|
|
sorted := append([]scored(nil), rows...)
|
|
sort.SliceStable(sorted, func(i, j int) bool {
|
|
if a, b := sorted[i].vals[sortIdx], sorted[j].vals[sortIdx]; a != b {
|
|
return a > b
|
|
}
|
|
if a, b := sorted[i].vals[0], sorted[j].vals[0]; a != b {
|
|
return a > b
|
|
}
|
|
return sorted[i].doc.Title < sorted[j].doc.Title
|
|
})
|
|
if len(sorted) > exploreRows {
|
|
sorted = sorted[:exploreRows]
|
|
}
|
|
out := make([]exploreRow, len(sorted))
|
|
for i, rw := range sorted {
|
|
vals := make([]string, len(rw.vals))
|
|
for k, v := range rw.vals {
|
|
switch {
|
|
case v < 0:
|
|
vals[k] = "-"
|
|
case m.Cols[k].Ratio:
|
|
vals[k] = trimRatio(v)
|
|
default:
|
|
vals[k] = strconv.Itoa(int(v))
|
|
}
|
|
}
|
|
out[i] = exploreRow{Rank: i + 1, Doc: *rw.doc, Vals: vals}
|
|
}
|
|
tables[sortIdx] = exploreTable{Rows: out, Total: len(rows)}
|
|
}
|
|
return tables
|
|
}
|
|
|
|
func exploreKey(set string, sortIdx int) string {
|
|
return "explore:" + set + ":" + strconv.Itoa(sortIdx)
|
|
}
|
|
|
|
func (co *Corpus) handleExploreMetric(w http.ResponseWriter, r *http.Request) {
|
|
m := exploreMetricBySlug(r.PathValue("metric"))
|
|
if m == nil {
|
|
notFound(w, r)
|
|
return
|
|
}
|
|
|
|
sortIdx := 0
|
|
sortBy := r.URL.Query().Get("sort")
|
|
for i, c := range m.Cols {
|
|
if c.Key == sortBy {
|
|
sortIdx = i
|
|
break
|
|
}
|
|
}
|
|
set := exploreSet(m.Slug, readPrefs(r).IncludeFOIA)
|
|
t := co.exploreTables[set][sortIdx]
|
|
|
|
cols := []sortCol{{Label: "#", Class: "num-col"}, {Label: "Document"}, {Label: "Date"}}
|
|
for _, c := range m.Cols {
|
|
cols = append(cols, sortCol{Key: c.Key, Label: c.Header, Class: "num-col", DescFirst: true, Fixed: true})
|
|
}
|
|
|
|
render(w, r, &exploreTablePage{
|
|
Metric: m,
|
|
Rows: t.Rows,
|
|
RowsHTML: co.preHTML(exploreKey(set, sortIdx)),
|
|
Total: t.Total,
|
|
Shown: len(t.Rows),
|
|
Heads: sortHeads("?", cols, m.Cols[sortIdx].Key, true),
|
|
})
|
|
}
|
|
|
|
func trimRatio(v float64) string {
|
|
s := strconv.FormatFloat(v, 'f', 2, 64)
|
|
s = strings.TrimRight(s, "0")
|
|
return strings.TrimRight(s, ".")
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// /explore/classification
|
|
|
|
type clsSeg struct {
|
|
Value string
|
|
Count int
|
|
Pct int
|
|
URL string
|
|
Class string // seg-0..seg-3, one per stripLevels entry
|
|
}
|
|
|
|
type clsYearRow struct {
|
|
Year string
|
|
URL string // the year's page, over every collection as the counts are
|
|
Total int
|
|
Segs []clsSeg
|
|
}
|
|
|
|
type heatCell struct {
|
|
Count int
|
|
URL string
|
|
}
|
|
|
|
type heatRow struct {
|
|
Name string
|
|
URL string
|
|
Cells []heatCell
|
|
}
|
|
|
|
var crossCols = []string{"TOP SECRET", "SECRET", "CONFIDENTIAL", "UNCLASSIFIED", "SI", "COMINT", "STRAP 1", "STRAP 2", "TK"}
|
|
var stripLevels = []string{"TOP SECRET", "SECRET", "CONFIDENTIAL", "UNCLASSIFIED"}
|
|
|
|
const stripMinMarks = 5
|
|
|
|
func (co *Corpus) buildClassPatterns() {
|
|
top := crossCols
|
|
rank := make(map[string]int, len(top))
|
|
for i, v := range top {
|
|
rank[valueKey(v)] = i
|
|
}
|
|
levelRank := make(map[string]int, len(stripLevels))
|
|
for i, v := range stripLevels {
|
|
levelRank[valueKey(v)] = i
|
|
}
|
|
|
|
yearCounts := map[string][]int{}
|
|
agCounts := map[string][]int{}
|
|
agName := map[string]string{}
|
|
for i := range co.archive.Docs {
|
|
d := &co.archive.Docs[i]
|
|
year := docYear(d)
|
|
agency := d.DocMeta.Creator
|
|
if n, ok := agName[valueKey(agency)]; ok {
|
|
agency = n
|
|
} else {
|
|
agName[valueKey(agency)] = agency
|
|
}
|
|
for _, cv := range d.DocMeta.Classification {
|
|
if lvl, isLevel := levelRank[valueKey(cv)]; isLevel && year != "" {
|
|
yc := yearCounts[year]
|
|
if yc == nil {
|
|
yc = make([]int, len(stripLevels))
|
|
yearCounts[year] = yc
|
|
}
|
|
yc[lvl]++
|
|
}
|
|
idx, known := rank[valueKey(cv)]
|
|
if known && agency != "" {
|
|
ac := agCounts[agency]
|
|
if ac == nil {
|
|
ac = make([]int, len(top))
|
|
agCounts[agency] = ac
|
|
}
|
|
ac[idx]++
|
|
}
|
|
}
|
|
}
|
|
|
|
years := make([]string, 0, len(yearCounts))
|
|
for y := range yearCounts {
|
|
years = append(years, y)
|
|
}
|
|
sort.Strings(years)
|
|
strips := make([]clsYearRow, 0, len(years))
|
|
var sparse []clsYearRow
|
|
for _, y := range years {
|
|
yc := yearCounts[y]
|
|
total := 0
|
|
for _, n := range yc {
|
|
total += n
|
|
}
|
|
row := clsYearRow{Year: y, URL: wholeArchive.link(categoryValueURL("docdate", y)), Total: total}
|
|
if total < stripMinMarks {
|
|
sparse = append(sparse, row)
|
|
continue
|
|
}
|
|
for i, n := range yc {
|
|
if n == 0 {
|
|
continue
|
|
}
|
|
row.Segs = append(row.Segs, clsSeg{
|
|
Value: stripLevels[i],
|
|
Count: n,
|
|
Pct: barPct(n, total, 1),
|
|
URL: wholeArchive.link(dateRangeSearchURL("CL", exactClause(stripLevels[i]), [2]string{"date_doc", y})),
|
|
Class: "seg-" + strconv.Itoa(i),
|
|
})
|
|
}
|
|
strips = append(strips, row)
|
|
}
|
|
|
|
mass := make(map[string]int, len(agCounts))
|
|
for a, cs := range agCounts {
|
|
for _, n := range cs {
|
|
mass[a] += n
|
|
}
|
|
}
|
|
ags := rankCounts(mass, -1)
|
|
rows := make([]heatRow, 0, len(ags))
|
|
for _, a := range ags {
|
|
hr := heatRow{Name: a, URL: wholeArchive.link(categoryValueURL("agency", a))}
|
|
for i, n := range agCounts[a] {
|
|
cell := heatCell{Count: n}
|
|
if n > 0 {
|
|
cell.URL = wholeArchive.link(searchURL([]searchClause{
|
|
{Field: "CR", Value: exactClause(a)},
|
|
{Field: "CL", Value: exactClause(top[i])},
|
|
}))
|
|
}
|
|
hr.Cells = append(hr.Cells, cell)
|
|
}
|
|
rows = append(rows, hr)
|
|
}
|
|
|
|
cols := make([]facetEntry, len(top))
|
|
for i, v := range top {
|
|
cols[i] = facetEntry{Name: v, URL: wholeArchive.link(categoryValueURL("classification", v))}
|
|
}
|
|
|
|
co.classPatterns.Strips = strips
|
|
co.classPatterns.Sparse = sparse
|
|
co.classPatterns.Cols = cols
|
|
co.classPatterns.HeatRows = rows
|
|
}
|
|
|
|
func (co *Corpus) handleClassPatterns(w http.ResponseWriter, r *http.Request) {
|
|
render(w, r, &classPatternsPage{
|
|
Cols: co.classPatterns.Cols,
|
|
Levels: stripLevels,
|
|
Strips: co.classPatterns.Strips,
|
|
Sparse: co.classPatterns.Sparse,
|
|
MinMarks: stripMinMarks,
|
|
HeatRows: co.classPatterns.HeatRows,
|
|
})
|
|
}
|