first commit
This commit is contained in:
commit
8c4dcae475
89 changed files with 24623 additions and 0 deletions
377
explore.go
Normal file
377
explore.go
Normal file
|
|
@ -0,0 +1,377 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The Explore section (/explore).
|
||||
|
||||
type exploreCol struct {
|
||||
Header string
|
||||
Key string
|
||||
Ratio bool
|
||||
}
|
||||
|
||||
type exploreMetric struct {
|
||||
Slug string
|
||||
Title string
|
||||
Blurb string
|
||||
Cols []exploreCol
|
||||
Vals func(d *Document) ([]float64, bool)
|
||||
}
|
||||
|
||||
const exploreRows = 200 // rows shown per page ("top 200 of N")
|
||||
|
||||
var exploreMetrics = []exploreMetric{
|
||||
{
|
||||
Slug: "programs",
|
||||
Title: "Most programs mentioned",
|
||||
Blurb: "Documents referencing the most surveillance programs.",
|
||||
Cols: []exploreCol{
|
||||
{Header: "Programs", Key: "programs"},
|
||||
{Header: "Pages", Key: "pages"},
|
||||
{Header: "Programs / page", Key: "ratio", Ratio: true},
|
||||
},
|
||||
Vals: func(d *Document) ([]float64, bool) {
|
||||
n := len(d.DocMeta.Programs)
|
||||
if n == 0 {
|
||||
return nil, false
|
||||
}
|
||||
pages, ratio := float64(d.Pages), -1.0
|
||||
if d.Pages > 0 {
|
||||
ratio = float64(n) / float64(d.Pages)
|
||||
}
|
||||
return []float64{float64(n), pages, ratio}, true
|
||||
},
|
||||
},
|
||||
{
|
||||
Slug: "people",
|
||||
Title: "Most people mentioned",
|
||||
Blurb: "Documents whose metadata names the most individuals.",
|
||||
Cols: []exploreCol{{Header: "People", Key: "people"}},
|
||||
Vals: func(d *Document) ([]float64, bool) {
|
||||
n := len(d.DocMeta.People)
|
||||
return []float64{float64(n)}, n > 0
|
||||
},
|
||||
},
|
||||
{
|
||||
Slug: "countries",
|
||||
Title: "Most countries mentioned",
|
||||
Blurb: "Documents spanning the most countries.",
|
||||
Cols: []exploreCol{{Header: "Countries", Key: "countries"}},
|
||||
Vals: func(d *Document) ([]float64, bool) {
|
||||
n := len(d.DocMeta.Countries)
|
||||
return []float64{float64(n)}, n > 0
|
||||
},
|
||||
},
|
||||
{
|
||||
Slug: "pages",
|
||||
Title: "Longest documents",
|
||||
Blurb: "Documents with the most pages.",
|
||||
Cols: []exploreCol{{Header: "Pages", Key: "pages"}},
|
||||
Vals: func(d *Document) ([]float64, bool) {
|
||||
return []float64{float64(d.Pages)}, d.Pages > 0
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func exploreMetricBySlug(slug string) *exploreMetric {
|
||||
for i := range exploreMetrics {
|
||||
if exploreMetrics[i].Slug == slug {
|
||||
return &exploreMetrics[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (co *Corpus) handleExploreIndex(w http.ResponseWriter, r *http.Request) {
|
||||
render(w, r, &explorePage{
|
||||
Metrics: exploreMetrics,
|
||||
HasAssemblies: len(co.assemblies) > 0,
|
||||
})
|
||||
}
|
||||
|
||||
type exploreRow struct {
|
||||
Rank int
|
||||
Doc Document
|
||||
Vals []string
|
||||
}
|
||||
|
||||
type exploreTable struct {
|
||||
Rows []exploreRow
|
||||
Total int
|
||||
}
|
||||
|
||||
func exploreSet(slug string, foia bool) string {
|
||||
if foia {
|
||||
return slug
|
||||
}
|
||||
return slug + ":nofoia"
|
||||
}
|
||||
|
||||
func (co *Corpus) buildExploreTables() {
|
||||
co.exploreTables = make(map[string][]exploreTable, 2*len(exploreMetrics))
|
||||
for mi := range exploreMetrics {
|
||||
m := &exploreMetrics[mi]
|
||||
if _, dup := co.exploreTables[m.Slug]; dup {
|
||||
log.Fatalf("explore: duplicate metric slug %q", m.Slug)
|
||||
}
|
||||
for _, foia := range []bool{true, false} {
|
||||
co.exploreTables[exploreSet(m.Slug, foia)] = co.buildExploreTable(m, foia)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (co *Corpus) buildExploreTable(m *exploreMetric, foia bool) []exploreTable {
|
||||
type scored struct {
|
||||
doc *Document
|
||||
vals []float64
|
||||
}
|
||||
var rows []scored
|
||||
for i := range co.archive.Docs {
|
||||
d := &co.archive.Docs[i]
|
||||
if !foia && d.Collection == collectionFOIA {
|
||||
continue
|
||||
}
|
||||
if vals, ok := m.Vals(d); ok {
|
||||
rows = append(rows, scored{d, vals})
|
||||
}
|
||||
}
|
||||
|
||||
tables := make([]exploreTable, len(m.Cols))
|
||||
for sortIdx := range m.Cols {
|
||||
sorted := append([]scored(nil), rows...)
|
||||
sort.SliceStable(sorted, func(i, j int) bool {
|
||||
if a, b := sorted[i].vals[sortIdx], sorted[j].vals[sortIdx]; a != b {
|
||||
return a > b
|
||||
}
|
||||
if a, b := sorted[i].vals[0], sorted[j].vals[0]; a != b {
|
||||
return a > b
|
||||
}
|
||||
return sorted[i].doc.Title < sorted[j].doc.Title
|
||||
})
|
||||
if len(sorted) > exploreRows {
|
||||
sorted = sorted[:exploreRows]
|
||||
}
|
||||
out := make([]exploreRow, len(sorted))
|
||||
for i, rw := range sorted {
|
||||
vals := make([]string, len(rw.vals))
|
||||
for k, v := range rw.vals {
|
||||
switch {
|
||||
case v < 0:
|
||||
vals[k] = "-"
|
||||
case m.Cols[k].Ratio:
|
||||
vals[k] = trimRatio(v)
|
||||
default:
|
||||
vals[k] = strconv.Itoa(int(v))
|
||||
}
|
||||
}
|
||||
out[i] = exploreRow{Rank: i + 1, Doc: *rw.doc, Vals: vals}
|
||||
}
|
||||
tables[sortIdx] = exploreTable{Rows: out, Total: len(rows)}
|
||||
}
|
||||
return tables
|
||||
}
|
||||
|
||||
func exploreKey(set string, sortIdx int) string {
|
||||
return "explore:" + set + ":" + strconv.Itoa(sortIdx)
|
||||
}
|
||||
|
||||
func (co *Corpus) handleExploreMetric(w http.ResponseWriter, r *http.Request) {
|
||||
m := exploreMetricBySlug(r.PathValue("metric"))
|
||||
if m == nil {
|
||||
notFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
sortIdx := 0
|
||||
sortBy := r.URL.Query().Get("sort")
|
||||
for i, c := range m.Cols {
|
||||
if c.Key == sortBy {
|
||||
sortIdx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
set := exploreSet(m.Slug, readPrefs(r).IncludeFOIA)
|
||||
t := co.exploreTables[set][sortIdx]
|
||||
|
||||
cols := []sortCol{{Label: "#", Class: "num-col"}, {Label: "Document"}, {Label: "Date"}}
|
||||
for _, c := range m.Cols {
|
||||
cols = append(cols, sortCol{Key: c.Key, Label: c.Header, Class: "num-col", DescFirst: true, Fixed: true})
|
||||
}
|
||||
|
||||
render(w, r, &exploreTablePage{
|
||||
Metric: m,
|
||||
Rows: t.Rows,
|
||||
RowsHTML: co.preHTML(exploreKey(set, sortIdx)),
|
||||
Total: t.Total,
|
||||
Shown: len(t.Rows),
|
||||
Heads: sortHeads("?", cols, m.Cols[sortIdx].Key, true),
|
||||
})
|
||||
}
|
||||
|
||||
func trimRatio(v float64) string {
|
||||
s := strconv.FormatFloat(v, 'f', 2, 64)
|
||||
s = strings.TrimRight(s, "0")
|
||||
return strings.TrimRight(s, ".")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /explore/classification
|
||||
|
||||
type clsSeg struct {
|
||||
Value string
|
||||
Count int
|
||||
Pct int
|
||||
URL string
|
||||
Class string // seg-0..seg-3, one per stripLevels entry
|
||||
}
|
||||
|
||||
type clsYearRow struct {
|
||||
Year string
|
||||
URL string // the year's page, over every collection as the counts are
|
||||
Total int
|
||||
Segs []clsSeg
|
||||
}
|
||||
|
||||
type heatCell struct {
|
||||
Count int
|
||||
URL string
|
||||
}
|
||||
|
||||
type heatRow struct {
|
||||
Name string
|
||||
URL string
|
||||
Cells []heatCell
|
||||
}
|
||||
|
||||
var crossCols = []string{"TOP SECRET", "SECRET", "CONFIDENTIAL", "UNCLASSIFIED", "SI", "COMINT", "STRAP 1", "STRAP 2", "TK"}
|
||||
var stripLevels = []string{"TOP SECRET", "SECRET", "CONFIDENTIAL", "UNCLASSIFIED"}
|
||||
|
||||
const stripMinMarks = 5
|
||||
|
||||
func (co *Corpus) buildClassPatterns() {
|
||||
top := crossCols
|
||||
rank := make(map[string]int, len(top))
|
||||
for i, v := range top {
|
||||
rank[valueKey(v)] = i
|
||||
}
|
||||
levelRank := make(map[string]int, len(stripLevels))
|
||||
for i, v := range stripLevels {
|
||||
levelRank[valueKey(v)] = i
|
||||
}
|
||||
|
||||
yearCounts := map[string][]int{}
|
||||
agCounts := map[string][]int{}
|
||||
agName := map[string]string{}
|
||||
for i := range co.archive.Docs {
|
||||
d := &co.archive.Docs[i]
|
||||
year := docYear(d)
|
||||
agency := d.DocMeta.Creator
|
||||
if n, ok := agName[valueKey(agency)]; ok {
|
||||
agency = n
|
||||
} else {
|
||||
agName[valueKey(agency)] = agency
|
||||
}
|
||||
for _, cv := range d.DocMeta.Classification {
|
||||
if lvl, isLevel := levelRank[valueKey(cv)]; isLevel && year != "" {
|
||||
yc := yearCounts[year]
|
||||
if yc == nil {
|
||||
yc = make([]int, len(stripLevels))
|
||||
yearCounts[year] = yc
|
||||
}
|
||||
yc[lvl]++
|
||||
}
|
||||
idx, known := rank[valueKey(cv)]
|
||||
if known && agency != "" {
|
||||
ac := agCounts[agency]
|
||||
if ac == nil {
|
||||
ac = make([]int, len(top))
|
||||
agCounts[agency] = ac
|
||||
}
|
||||
ac[idx]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
years := make([]string, 0, len(yearCounts))
|
||||
for y := range yearCounts {
|
||||
years = append(years, y)
|
||||
}
|
||||
sort.Strings(years)
|
||||
strips := make([]clsYearRow, 0, len(years))
|
||||
var sparse []clsYearRow
|
||||
for _, y := range years {
|
||||
yc := yearCounts[y]
|
||||
total := 0
|
||||
for _, n := range yc {
|
||||
total += n
|
||||
}
|
||||
row := clsYearRow{Year: y, URL: wholeArchive.link(categoryValueURL("docdate", y)), Total: total}
|
||||
if total < stripMinMarks {
|
||||
sparse = append(sparse, row)
|
||||
continue
|
||||
}
|
||||
for i, n := range yc {
|
||||
if n == 0 {
|
||||
continue
|
||||
}
|
||||
row.Segs = append(row.Segs, clsSeg{
|
||||
Value: stripLevels[i],
|
||||
Count: n,
|
||||
Pct: barPct(n, total, 1),
|
||||
URL: wholeArchive.link(dateRangeSearchURL("CL", exactClause(stripLevels[i]), [2]string{"date_doc", y})),
|
||||
Class: "seg-" + strconv.Itoa(i),
|
||||
})
|
||||
}
|
||||
strips = append(strips, row)
|
||||
}
|
||||
|
||||
mass := make(map[string]int, len(agCounts))
|
||||
for a, cs := range agCounts {
|
||||
for _, n := range cs {
|
||||
mass[a] += n
|
||||
}
|
||||
}
|
||||
ags := rankCounts(mass, -1)
|
||||
rows := make([]heatRow, 0, len(ags))
|
||||
for _, a := range ags {
|
||||
hr := heatRow{Name: a, URL: wholeArchive.link(categoryValueURL("agency", a))}
|
||||
for i, n := range agCounts[a] {
|
||||
cell := heatCell{Count: n}
|
||||
if n > 0 {
|
||||
cell.URL = wholeArchive.link(searchURL([]searchClause{
|
||||
{Field: "CR", Value: exactClause(a)},
|
||||
{Field: "CL", Value: exactClause(top[i])},
|
||||
}))
|
||||
}
|
||||
hr.Cells = append(hr.Cells, cell)
|
||||
}
|
||||
rows = append(rows, hr)
|
||||
}
|
||||
|
||||
cols := make([]facetEntry, len(top))
|
||||
for i, v := range top {
|
||||
cols[i] = facetEntry{Name: v, URL: wholeArchive.link(categoryValueURL("classification", v))}
|
||||
}
|
||||
|
||||
co.classPatterns.Strips = strips
|
||||
co.classPatterns.Sparse = sparse
|
||||
co.classPatterns.Cols = cols
|
||||
co.classPatterns.HeatRows = rows
|
||||
}
|
||||
|
||||
func (co *Corpus) handleClassPatterns(w http.ResponseWriter, r *http.Request) {
|
||||
render(w, r, &classPatternsPage{
|
||||
Cols: co.classPatterns.Cols,
|
||||
Levels: stripLevels,
|
||||
Strips: co.classPatterns.Strips,
|
||||
Sparse: co.classPatterns.Sparse,
|
||||
MinMarks: stripMinMarks,
|
||||
HeatRows: co.classPatterns.HeatRows,
|
||||
})
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue