package main import ( "log" "net/http" "sort" "strconv" "strings" ) // The Explore section (/explore). type exploreCol struct { Header string Key string Ratio bool } type exploreMetric struct { Slug string Title string Blurb string Cols []exploreCol Vals func(d *Document) ([]float64, bool) } const exploreRows = 200 // rows shown per page ("top 200 of N") var exploreMetrics = []exploreMetric{ { Slug: "programs", Title: "Most programs mentioned", Blurb: "Documents referencing the most surveillance programs.", Cols: []exploreCol{ {Header: "Programs", Key: "programs"}, {Header: "Pages", Key: "pages"}, {Header: "Programs / page", Key: "ratio", Ratio: true}, }, Vals: func(d *Document) ([]float64, bool) { n := len(d.DocMeta.Programs) if n == 0 { return nil, false } pages, ratio := float64(d.Pages), -1.0 if d.Pages > 0 { ratio = float64(n) / float64(d.Pages) } return []float64{float64(n), pages, ratio}, true }, }, { Slug: "people", Title: "Most people mentioned", Blurb: "Documents whose metadata names the most individuals.", Cols: []exploreCol{{Header: "People", Key: "people"}}, Vals: func(d *Document) ([]float64, bool) { n := len(d.DocMeta.People) return []float64{float64(n)}, n > 0 }, }, { Slug: "countries", Title: "Most countries mentioned", Blurb: "Documents spanning the most countries.", Cols: []exploreCol{{Header: "Countries", Key: "countries"}}, Vals: func(d *Document) ([]float64, bool) { n := len(d.DocMeta.Countries) return []float64{float64(n)}, n > 0 }, }, { Slug: "pages", Title: "Longest documents", Blurb: "Documents with the most pages.", Cols: []exploreCol{{Header: "Pages", Key: "pages"}}, Vals: func(d *Document) ([]float64, bool) { return []float64{float64(d.Pages)}, d.Pages > 0 }, }, } func exploreMetricBySlug(slug string) *exploreMetric { for i := range exploreMetrics { if exploreMetrics[i].Slug == slug { return &exploreMetrics[i] } } return nil } func (co *Corpus) handleExploreIndex(w http.ResponseWriter, r *http.Request) { render(w, r, &explorePage{ Metrics: exploreMetrics, HasAssemblies: len(co.assemblies) > 0, }) } type exploreRow struct { Rank int Doc Document Vals []string } type exploreTable struct { Rows []exploreRow Total int } func exploreSet(slug string, foia bool) string { if foia { return slug } return slug + ":nofoia" } func (co *Corpus) buildExploreTables() { co.exploreTables = make(map[string][]exploreTable, 2*len(exploreMetrics)) for mi := range exploreMetrics { m := &exploreMetrics[mi] if _, dup := co.exploreTables[m.Slug]; dup { log.Fatalf("explore: duplicate metric slug %q", m.Slug) } for _, foia := range []bool{true, false} { co.exploreTables[exploreSet(m.Slug, foia)] = co.buildExploreTable(m, foia) } } } func (co *Corpus) buildExploreTable(m *exploreMetric, foia bool) []exploreTable { type scored struct { doc *Document vals []float64 } var rows []scored for i := range co.archive.Docs { d := &co.archive.Docs[i] if !foia && d.Collection == collectionFOIA { continue } if vals, ok := m.Vals(d); ok { rows = append(rows, scored{d, vals}) } } tables := make([]exploreTable, len(m.Cols)) for sortIdx := range m.Cols { sorted := append([]scored(nil), rows...) sort.SliceStable(sorted, func(i, j int) bool { if a, b := sorted[i].vals[sortIdx], sorted[j].vals[sortIdx]; a != b { return a > b } if a, b := sorted[i].vals[0], sorted[j].vals[0]; a != b { return a > b } return sorted[i].doc.Title < sorted[j].doc.Title }) if len(sorted) > exploreRows { sorted = sorted[:exploreRows] } out := make([]exploreRow, len(sorted)) for i, rw := range sorted { vals := make([]string, len(rw.vals)) for k, v := range rw.vals { switch { case v < 0: vals[k] = "-" case m.Cols[k].Ratio: vals[k] = trimRatio(v) default: vals[k] = strconv.Itoa(int(v)) } } out[i] = exploreRow{Rank: i + 1, Doc: *rw.doc, Vals: vals} } tables[sortIdx] = exploreTable{Rows: out, Total: len(rows)} } return tables } func exploreKey(set string, sortIdx int) string { return "explore:" + set + ":" + strconv.Itoa(sortIdx) } func (co *Corpus) handleExploreMetric(w http.ResponseWriter, r *http.Request) { m := exploreMetricBySlug(r.PathValue("metric")) if m == nil { notFound(w, r) return } sortIdx := 0 sortBy := r.URL.Query().Get("sort") for i, c := range m.Cols { if c.Key == sortBy { sortIdx = i break } } set := exploreSet(m.Slug, readPrefs(r).IncludeFOIA) t := co.exploreTables[set][sortIdx] cols := []sortCol{{Label: "#", Class: "num-col"}, {Label: "Document"}, {Label: "Date"}} for _, c := range m.Cols { cols = append(cols, sortCol{Key: c.Key, Label: c.Header, Class: "num-col", DescFirst: true, Fixed: true}) } render(w, r, &exploreTablePage{ Metric: m, Rows: t.Rows, RowsHTML: co.preHTML(exploreKey(set, sortIdx)), Total: t.Total, Shown: len(t.Rows), Heads: sortHeads("?", cols, m.Cols[sortIdx].Key, true), }) } func trimRatio(v float64) string { s := strconv.FormatFloat(v, 'f', 2, 64) s = strings.TrimRight(s, "0") return strings.TrimRight(s, ".") } // --------------------------------------------------------------------------- // /explore/classification type clsSeg struct { Value string Count int Pct int URL string Class string // seg-0..seg-3, one per stripLevels entry } type clsYearRow struct { Year string URL string // the year's page, over every collection as the counts are Total int Segs []clsSeg } type heatCell struct { Count int URL string } type heatRow struct { Name string URL string Cells []heatCell } var crossCols = []string{"TOP SECRET", "SECRET", "CONFIDENTIAL", "UNCLASSIFIED", "SI", "COMINT", "STRAP 1", "STRAP 2", "TK"} var stripLevels = []string{"TOP SECRET", "SECRET", "CONFIDENTIAL", "UNCLASSIFIED"} const stripMinMarks = 5 func (co *Corpus) buildClassPatterns() { top := crossCols rank := make(map[string]int, len(top)) for i, v := range top { rank[valueKey(v)] = i } levelRank := make(map[string]int, len(stripLevels)) for i, v := range stripLevels { levelRank[valueKey(v)] = i } yearCounts := map[string][]int{} agCounts := map[string][]int{} agName := map[string]string{} for i := range co.archive.Docs { d := &co.archive.Docs[i] year := docYear(d) agency := d.DocMeta.Creator if n, ok := agName[valueKey(agency)]; ok { agency = n } else { agName[valueKey(agency)] = agency } for _, cv := range d.DocMeta.Classification { if lvl, isLevel := levelRank[valueKey(cv)]; isLevel && year != "" { yc := yearCounts[year] if yc == nil { yc = make([]int, len(stripLevels)) yearCounts[year] = yc } yc[lvl]++ } idx, known := rank[valueKey(cv)] if known && agency != "" { ac := agCounts[agency] if ac == nil { ac = make([]int, len(top)) agCounts[agency] = ac } ac[idx]++ } } } years := make([]string, 0, len(yearCounts)) for y := range yearCounts { years = append(years, y) } sort.Strings(years) strips := make([]clsYearRow, 0, len(years)) var sparse []clsYearRow for _, y := range years { yc := yearCounts[y] total := 0 for _, n := range yc { total += n } row := clsYearRow{Year: y, URL: wholeArchive.link(categoryValueURL("docdate", y)), Total: total} if total < stripMinMarks { sparse = append(sparse, row) continue } for i, n := range yc { if n == 0 { continue } row.Segs = append(row.Segs, clsSeg{ Value: stripLevels[i], Count: n, Pct: barPct(n, total, 1), URL: wholeArchive.link(dateRangeSearchURL("CL", exactClause(stripLevels[i]), [2]string{"date_doc", y})), Class: "seg-" + strconv.Itoa(i), }) } strips = append(strips, row) } mass := make(map[string]int, len(agCounts)) for a, cs := range agCounts { for _, n := range cs { mass[a] += n } } ags := rankCounts(mass, -1) rows := make([]heatRow, 0, len(ags)) for _, a := range ags { hr := heatRow{Name: a, URL: wholeArchive.link(categoryValueURL("agency", a))} for i, n := range agCounts[a] { cell := heatCell{Count: n} if n > 0 { cell.URL = wholeArchive.link(searchURL([]searchClause{ {Field: "CR", Value: exactClause(a)}, {Field: "CL", Value: exactClause(top[i])}, })) } hr.Cells = append(hr.Cells, cell) } rows = append(rows, hr) } cols := make([]facetEntry, len(top)) for i, v := range top { cols[i] = facetEntry{Name: v, URL: wholeArchive.link(categoryValueURL("classification", v))} } co.classPatterns.Strips = strips co.classPatterns.Sparse = sparse co.classPatterns.Cols = cols co.classPatterns.HeatRows = rows } func (co *Corpus) handleClassPatterns(w http.ResponseWriter, r *http.Request) { render(w, r, &classPatternsPage{ Cols: co.classPatterns.Cols, Levels: stripLevels, Strips: co.classPatterns.Strips, Sparse: co.classPatterns.Sparse, MinMarks: stripMinMarks, HeatRows: co.classPatterns.HeatRows, }) }