snowchive-server/explore.go
2026-11-19 04:40:59 +02:00

377 lines
8.9 KiB
Go

package main
import (
"log"
"net/http"
"sort"
"strconv"
"strings"
)
// The Explore section (/explore).
type exploreCol struct {
Header string
Key string
Ratio bool
}
type exploreMetric struct {
Slug string
Title string
Blurb string
Cols []exploreCol
Vals func(d *Document) ([]float64, bool)
}
const exploreRows = 200 // rows shown per page ("top 200 of N")
var exploreMetrics = []exploreMetric{
{
Slug: "programs",
Title: "Most programs mentioned",
Blurb: "Documents referencing the most surveillance programs.",
Cols: []exploreCol{
{Header: "Programs", Key: "programs"},
{Header: "Pages", Key: "pages"},
{Header: "Programs / page", Key: "ratio", Ratio: true},
},
Vals: func(d *Document) ([]float64, bool) {
n := len(d.DocMeta.Programs)
if n == 0 {
return nil, false
}
pages, ratio := float64(d.Pages), -1.0
if d.Pages > 0 {
ratio = float64(n) / float64(d.Pages)
}
return []float64{float64(n), pages, ratio}, true
},
},
{
Slug: "people",
Title: "Most people mentioned",
Blurb: "Documents whose metadata names the most individuals.",
Cols: []exploreCol{{Header: "People", Key: "people"}},
Vals: func(d *Document) ([]float64, bool) {
n := len(d.DocMeta.People)
return []float64{float64(n)}, n > 0
},
},
{
Slug: "countries",
Title: "Most countries mentioned",
Blurb: "Documents spanning the most countries.",
Cols: []exploreCol{{Header: "Countries", Key: "countries"}},
Vals: func(d *Document) ([]float64, bool) {
n := len(d.DocMeta.Countries)
return []float64{float64(n)}, n > 0
},
},
{
Slug: "pages",
Title: "Longest documents",
Blurb: "Documents with the most pages.",
Cols: []exploreCol{{Header: "Pages", Key: "pages"}},
Vals: func(d *Document) ([]float64, bool) {
return []float64{float64(d.Pages)}, d.Pages > 0
},
},
}
func exploreMetricBySlug(slug string) *exploreMetric {
for i := range exploreMetrics {
if exploreMetrics[i].Slug == slug {
return &exploreMetrics[i]
}
}
return nil
}
func (co *Corpus) handleExploreIndex(w http.ResponseWriter, r *http.Request) {
render(w, r, &explorePage{
Metrics: exploreMetrics,
HasAssemblies: len(co.assemblies) > 0,
})
}
type exploreRow struct {
Rank int
Doc Document
Vals []string
}
type exploreTable struct {
Rows []exploreRow
Total int
}
func exploreSet(slug string, foia bool) string {
if foia {
return slug
}
return slug + ":nofoia"
}
func (co *Corpus) buildExploreTables() {
co.exploreTables = make(map[string][]exploreTable, 2*len(exploreMetrics))
for mi := range exploreMetrics {
m := &exploreMetrics[mi]
if _, dup := co.exploreTables[m.Slug]; dup {
log.Fatalf("explore: duplicate metric slug %q", m.Slug)
}
for _, foia := range []bool{true, false} {
co.exploreTables[exploreSet(m.Slug, foia)] = co.buildExploreTable(m, foia)
}
}
}
func (co *Corpus) buildExploreTable(m *exploreMetric, foia bool) []exploreTable {
type scored struct {
doc *Document
vals []float64
}
var rows []scored
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
if !foia && d.Collection == collectionFOIA {
continue
}
if vals, ok := m.Vals(d); ok {
rows = append(rows, scored{d, vals})
}
}
tables := make([]exploreTable, len(m.Cols))
for sortIdx := range m.Cols {
sorted := append([]scored(nil), rows...)
sort.SliceStable(sorted, func(i, j int) bool {
if a, b := sorted[i].vals[sortIdx], sorted[j].vals[sortIdx]; a != b {
return a > b
}
if a, b := sorted[i].vals[0], sorted[j].vals[0]; a != b {
return a > b
}
return sorted[i].doc.Title < sorted[j].doc.Title
})
if len(sorted) > exploreRows {
sorted = sorted[:exploreRows]
}
out := make([]exploreRow, len(sorted))
for i, rw := range sorted {
vals := make([]string, len(rw.vals))
for k, v := range rw.vals {
switch {
case v < 0:
vals[k] = "-"
case m.Cols[k].Ratio:
vals[k] = trimRatio(v)
default:
vals[k] = strconv.Itoa(int(v))
}
}
out[i] = exploreRow{Rank: i + 1, Doc: *rw.doc, Vals: vals}
}
tables[sortIdx] = exploreTable{Rows: out, Total: len(rows)}
}
return tables
}
func exploreKey(set string, sortIdx int) string {
return "explore:" + set + ":" + strconv.Itoa(sortIdx)
}
func (co *Corpus) handleExploreMetric(w http.ResponseWriter, r *http.Request) {
m := exploreMetricBySlug(r.PathValue("metric"))
if m == nil {
notFound(w, r)
return
}
sortIdx := 0
sortBy := r.URL.Query().Get("sort")
for i, c := range m.Cols {
if c.Key == sortBy {
sortIdx = i
break
}
}
set := exploreSet(m.Slug, readPrefs(r).IncludeFOIA)
t := co.exploreTables[set][sortIdx]
cols := []sortCol{{Label: "#", Class: "num-col"}, {Label: "Document"}, {Label: "Date"}}
for _, c := range m.Cols {
cols = append(cols, sortCol{Key: c.Key, Label: c.Header, Class: "num-col", DescFirst: true, Fixed: true})
}
render(w, r, &exploreTablePage{
Metric: m,
Rows: t.Rows,
RowsHTML: co.preHTML(exploreKey(set, sortIdx)),
Total: t.Total,
Shown: len(t.Rows),
Heads: sortHeads("?", cols, m.Cols[sortIdx].Key, true),
})
}
func trimRatio(v float64) string {
s := strconv.FormatFloat(v, 'f', 2, 64)
s = strings.TrimRight(s, "0")
return strings.TrimRight(s, ".")
}
// ---------------------------------------------------------------------------
// /explore/classification
type clsSeg struct {
Value string
Count int
Pct int
URL string
Class string // seg-0..seg-3, one per stripLevels entry
}
type clsYearRow struct {
Year string
URL string // the year's page, over every collection as the counts are
Total int
Segs []clsSeg
}
type heatCell struct {
Count int
URL string
}
type heatRow struct {
Name string
URL string
Cells []heatCell
}
var crossCols = []string{"TOP SECRET", "SECRET", "CONFIDENTIAL", "UNCLASSIFIED", "SI", "COMINT", "STRAP 1", "STRAP 2", "TK"}
var stripLevels = []string{"TOP SECRET", "SECRET", "CONFIDENTIAL", "UNCLASSIFIED"}
const stripMinMarks = 5
func (co *Corpus) buildClassPatterns() {
top := crossCols
rank := make(map[string]int, len(top))
for i, v := range top {
rank[valueKey(v)] = i
}
levelRank := make(map[string]int, len(stripLevels))
for i, v := range stripLevels {
levelRank[valueKey(v)] = i
}
yearCounts := map[string][]int{}
agCounts := map[string][]int{}
agName := map[string]string{}
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
year := docYear(d)
agency := d.DocMeta.Creator
if n, ok := agName[valueKey(agency)]; ok {
agency = n
} else {
agName[valueKey(agency)] = agency
}
for _, cv := range d.DocMeta.Classification {
if lvl, isLevel := levelRank[valueKey(cv)]; isLevel && year != "" {
yc := yearCounts[year]
if yc == nil {
yc = make([]int, len(stripLevels))
yearCounts[year] = yc
}
yc[lvl]++
}
idx, known := rank[valueKey(cv)]
if known && agency != "" {
ac := agCounts[agency]
if ac == nil {
ac = make([]int, len(top))
agCounts[agency] = ac
}
ac[idx]++
}
}
}
years := make([]string, 0, len(yearCounts))
for y := range yearCounts {
years = append(years, y)
}
sort.Strings(years)
strips := make([]clsYearRow, 0, len(years))
var sparse []clsYearRow
for _, y := range years {
yc := yearCounts[y]
total := 0
for _, n := range yc {
total += n
}
row := clsYearRow{Year: y, URL: wholeArchive.link(categoryValueURL("docdate", y)), Total: total}
if total < stripMinMarks {
sparse = append(sparse, row)
continue
}
for i, n := range yc {
if n == 0 {
continue
}
row.Segs = append(row.Segs, clsSeg{
Value: stripLevels[i],
Count: n,
Pct: barPct(n, total, 1),
URL: wholeArchive.link(dateRangeSearchURL("CL", exactClause(stripLevels[i]), [2]string{"date_doc", y})),
Class: "seg-" + strconv.Itoa(i),
})
}
strips = append(strips, row)
}
mass := make(map[string]int, len(agCounts))
for a, cs := range agCounts {
for _, n := range cs {
mass[a] += n
}
}
ags := rankCounts(mass, -1)
rows := make([]heatRow, 0, len(ags))
for _, a := range ags {
hr := heatRow{Name: a, URL: wholeArchive.link(categoryValueURL("agency", a))}
for i, n := range agCounts[a] {
cell := heatCell{Count: n}
if n > 0 {
cell.URL = wholeArchive.link(searchURL([]searchClause{
{Field: "CR", Value: exactClause(a)},
{Field: "CL", Value: exactClause(top[i])},
}))
}
hr.Cells = append(hr.Cells, cell)
}
rows = append(rows, hr)
}
cols := make([]facetEntry, len(top))
for i, v := range top {
cols[i] = facetEntry{Name: v, URL: wholeArchive.link(categoryValueURL("classification", v))}
}
co.classPatterns.Strips = strips
co.classPatterns.Sparse = sparse
co.classPatterns.Cols = cols
co.classPatterns.HeatRows = rows
}
func (co *Corpus) handleClassPatterns(w http.ResponseWriter, r *http.Request) {
render(w, r, &classPatternsPage{
Cols: co.classPatterns.Cols,
Levels: stripLevels,
Strips: co.classPatterns.Strips,
Sparse: co.classPatterns.Sparse,
MinMarks: stripMinMarks,
HeatRows: co.classPatterns.HeatRows,
})
}