230 lines
6.6 KiB
Go
230 lines
6.6 KiB
Go
// portions.go - the portion markings inside the transcripts.
|
|
package main
|
|
|
|
import (
|
|
"html/template"
|
|
"log"
|
|
"net/http"
|
|
"regexp"
|
|
"strings"
|
|
)
|
|
|
|
// A marking opens a line, optionally behind a bullet, a heading hash or a
|
|
// list number: "(U)", "* (S//SI) ...", "## (U//FOUO) Heading".
|
|
var rePortion = regexp.MustCompile(`^[ \t]{0,6}(?:[-*+][ \t]+|#{1,6}[ \t]+|\d+[.)][ \t]+)?\(([^)\n]{1,42})\)`)
|
|
|
|
var portionLevels = map[string]string{
|
|
"U": "U", "UNCLASSIFIED": "U", "FOUO": "U",
|
|
"C": "C", "CONFIDENTIAL": "C",
|
|
"S": "S", "SECRET": "S",
|
|
"TS": "TS", "TOP SECRET": "TS",
|
|
}
|
|
|
|
var portionControls = map[string]bool{
|
|
"SI": true, "TK": true, "GAMMA": true, "SI-GAMMA": true, "STLW": true, "TSP": true,
|
|
"HCS": true, "KLONDIKE": true, "ECI": true, "SI-ECI": true, "COMINT": true,
|
|
"FOUO": true, "NF": true, "NOFORN": true, "OC": true, "ORCON": true,
|
|
"PROPIN": true, "PR": true, "RELIDO": true, "RSEN": true, "FISA": true,
|
|
"IMCON": true, "DSEN": true, "REL": true, "FVEY": true, "NATO": true,
|
|
"X1": true,
|
|
}
|
|
|
|
var portionLevelNames = []string{"U", "C", "S", "TS"}
|
|
|
|
var portionLevelLabels = map[string]string{
|
|
"U": "UNCLASSIFIED", "C": "CONFIDENTIAL", "S": "SECRET", "TS": "TOP SECRET",
|
|
}
|
|
|
|
// reSlashRun folds "/", "//" and "///" to one separator.
|
|
var reSlashRun = regexp.MustCompile(`[ \t]*/+[ \t]*`)
|
|
|
|
// portionMark is one marking, canonicalised.
|
|
type portionMark struct {
|
|
Raw string // as the transcript writes it, without the parentheses
|
|
Canonical string // "TS//SI//NF"
|
|
Level string // "TS"
|
|
Controls []string // ["SI", "NF"], in the order the marking writes them
|
|
OK bool // every part recognised
|
|
}
|
|
|
|
func parsePortion(raw string) portionMark {
|
|
m := portionMark{Raw: strings.TrimSpace(raw)}
|
|
written := strings.Split(reSlashRun.ReplaceAllString(strings.Join(strings.Fields(m.Raw), " "), "//"), "//")
|
|
if lvl := strings.TrimSpace(written[0]); lvl == "" || lvl != strings.ToUpper(lvl) {
|
|
return m
|
|
}
|
|
parts := make([]string, len(written))
|
|
for i := range written {
|
|
parts[i] = strings.ToUpper(strings.TrimSpace(written[i]))
|
|
}
|
|
lvl, isLevel := portionLevels[parts[0]]
|
|
if !isLevel {
|
|
return m
|
|
}
|
|
m.Level = lvl
|
|
if parts[0] == "FOUO" {
|
|
m.Controls = append(m.Controls, "FOUO")
|
|
}
|
|
known := true
|
|
for _, p := range parts[1:] {
|
|
if p == "" {
|
|
continue
|
|
}
|
|
m.Controls = append(m.Controls, p)
|
|
if !portionControlKnown(p) {
|
|
known = false
|
|
}
|
|
}
|
|
m.OK = known
|
|
m.Canonical = m.Level
|
|
if len(m.Controls) > 0 {
|
|
m.Canonical += "//" + strings.Join(m.Controls, "//")
|
|
}
|
|
return m
|
|
}
|
|
|
|
func portionControlKnown(p string) bool {
|
|
if portionControls[p] {
|
|
return true
|
|
}
|
|
if strings.HasPrefix(p, "REL") {
|
|
return true
|
|
}
|
|
if strings.HasPrefix(p, "ECI ") || strings.HasPrefix(p, "SI-ECI") {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func extractPortions(text string) []portionMark {
|
|
var out []portionMark
|
|
fenced := false
|
|
for _, line := range strings.Split(text, "\n") {
|
|
t := strings.TrimSpace(line)
|
|
if strings.HasPrefix(t, "```") || strings.HasPrefix(t, "~~~") {
|
|
fenced = !fenced
|
|
continue
|
|
}
|
|
if fenced || strings.HasPrefix(t, ">") {
|
|
continue
|
|
}
|
|
m := rePortion.FindStringSubmatch(line)
|
|
if m == nil {
|
|
continue
|
|
}
|
|
if p := parsePortion(m[1]); p.Level != "" {
|
|
out = append(out, p)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// --- the corpus view -------------------------------------------------------
|
|
|
|
type portionRow struct {
|
|
Name string // the canonical marking, a level, or a control
|
|
Label string // the spelled-out level, where there is one
|
|
Marks int // occurrences across the corpus
|
|
Docs int // documents carrying it
|
|
SearchURL template.URL
|
|
}
|
|
|
|
type portionSummary struct {
|
|
Docs int // documents carrying at least one marking
|
|
Marks int // markings found
|
|
Distinct int // distinct canonical markings
|
|
Unrecog int // markings whose parts are not all known
|
|
UnrecogEg []portionRow
|
|
Levels []portionRow
|
|
Markings []portionRow
|
|
Controls []portionRow
|
|
}
|
|
|
|
const portionRows = 200
|
|
|
|
func (co *Corpus) buildPortions() {
|
|
co.portionStats = portionSummary{}
|
|
markMarks, lvlMarks, ctlMarks := map[string]int{}, map[string]int{}, map[string]int{}
|
|
docsBy := map[string]map[string]int{"PL": {}, "PM": {}, "PC": {}}
|
|
unrecogMarks := map[string]int{}
|
|
|
|
for i := range co.archive.Docs {
|
|
d := &co.archive.Docs[i]
|
|
d.PortionLevels, d.PortionMarkings, d.PortionControls = nil, nil, nil
|
|
marks := extractPortions(d.FullText)
|
|
if len(marks) == 0 {
|
|
continue
|
|
}
|
|
co.portionStats.Docs++
|
|
co.portionStats.Marks += len(marks)
|
|
for _, m := range marks {
|
|
if !m.OK {
|
|
co.portionStats.Unrecog++
|
|
unrecogMarks[m.Raw]++
|
|
continue
|
|
}
|
|
markMarks[m.Canonical]++
|
|
lvlMarks[m.Level]++
|
|
d.PortionMarkings = append(d.PortionMarkings, m.Canonical)
|
|
d.PortionLevels = append(d.PortionLevels, m.Level)
|
|
for _, c := range m.Controls {
|
|
ctlMarks[c]++
|
|
d.PortionControls = append(d.PortionControls, c)
|
|
}
|
|
}
|
|
d.PortionLevels = normalizeList(d.PortionLevels)
|
|
d.PortionMarkings = normalizeList(d.PortionMarkings)
|
|
d.PortionControls = normalizeList(d.PortionControls)
|
|
for code, docs := range docsBy {
|
|
for _, v := range facetValues(d, code) {
|
|
docs[v]++
|
|
}
|
|
}
|
|
}
|
|
|
|
co.portionStats.Distinct = len(markMarks)
|
|
for _, l := range portionLevelNames {
|
|
if lvlMarks[l] == 0 {
|
|
continue
|
|
}
|
|
co.portionStats.Levels = append(co.portionStats.Levels, portionRow{
|
|
Name: l, Label: portionLevelLabels[l], Marks: lvlMarks[l], Docs: docsBy["PL"][l],
|
|
SearchURL: portionSearchURL("PL", l),
|
|
})
|
|
}
|
|
co.portionStats.Markings = portionRowsOf(markMarks, docsBy["PM"], portionRows)
|
|
for i := range co.portionStats.Markings {
|
|
r := &co.portionStats.Markings[i]
|
|
r.SearchURL = portionSearchURL("PM", r.Name)
|
|
}
|
|
co.portionStats.Controls = portionRowsOf(ctlMarks, docsBy["PC"], portionRows)
|
|
for i := range co.portionStats.Controls {
|
|
r := &co.portionStats.Controls[i]
|
|
r.SearchURL = portionSearchURL("PC", r.Name)
|
|
}
|
|
co.portionStats.UnrecogEg = portionRowsOf(unrecogMarks, nil, 25)
|
|
|
|
log.Printf("Portion markings: %d in %d documents, %d distinct (%d unrecognised)",
|
|
co.portionStats.Marks, co.portionStats.Docs, co.portionStats.Distinct, co.portionStats.Unrecog)
|
|
}
|
|
|
|
func portionSearchURL(code, value string) template.URL {
|
|
return template.URL(wholeArchive.link(searchURL([]searchClause{{Field: code, Value: exactClause(value)}})))
|
|
}
|
|
|
|
func portionRowsOf(marks, docs map[string]int, n int) []portionRow {
|
|
names := rankCounts(marks, n)
|
|
out := make([]portionRow, len(names))
|
|
for i, k := range names {
|
|
out[i] = portionRow{Name: k, Marks: marks[k], Docs: docs[k]}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (co *Corpus) handlePortions(w http.ResponseWriter, r *http.Request) {
|
|
render(w, r, &portionsPage{
|
|
Title: "Portion markings",
|
|
Stats: co.portionStats,
|
|
RowLimit: portionRows,
|
|
})
|
|
}
|