// portions.go - the portion markings inside the transcripts. package main import ( "html/template" "log" "net/http" "regexp" "strings" ) // A marking opens a line, optionally behind a bullet, a heading hash or a // list number: "(U)", "* (S//SI) ...", "## (U//FOUO) Heading". var rePortion = regexp.MustCompile(`^[ \t]{0,6}(?:[-*+][ \t]+|#{1,6}[ \t]+|\d+[.)][ \t]+)?\(([^)\n]{1,42})\)`) var portionLevels = map[string]string{ "U": "U", "UNCLASSIFIED": "U", "FOUO": "U", "C": "C", "CONFIDENTIAL": "C", "S": "S", "SECRET": "S", "TS": "TS", "TOP SECRET": "TS", } var portionControls = map[string]bool{ "SI": true, "TK": true, "GAMMA": true, "SI-GAMMA": true, "STLW": true, "TSP": true, "HCS": true, "KLONDIKE": true, "ECI": true, "SI-ECI": true, "COMINT": true, "FOUO": true, "NF": true, "NOFORN": true, "OC": true, "ORCON": true, "PROPIN": true, "PR": true, "RELIDO": true, "RSEN": true, "FISA": true, "IMCON": true, "DSEN": true, "REL": true, "FVEY": true, "NATO": true, "X1": true, } var portionLevelNames = []string{"U", "C", "S", "TS"} var portionLevelLabels = map[string]string{ "U": "UNCLASSIFIED", "C": "CONFIDENTIAL", "S": "SECRET", "TS": "TOP SECRET", } // reSlashRun folds "/", "//" and "///" to one separator. var reSlashRun = regexp.MustCompile(`[ \t]*/+[ \t]*`) // portionMark is one marking, canonicalised. type portionMark struct { Raw string // as the transcript writes it, without the parentheses Canonical string // "TS//SI//NF" Level string // "TS" Controls []string // ["SI", "NF"], in the order the marking writes them OK bool // every part recognised } func parsePortion(raw string) portionMark { m := portionMark{Raw: strings.TrimSpace(raw)} written := strings.Split(reSlashRun.ReplaceAllString(strings.Join(strings.Fields(m.Raw), " "), "//"), "//") if lvl := strings.TrimSpace(written[0]); lvl == "" || lvl != strings.ToUpper(lvl) { return m } parts := make([]string, len(written)) for i := range written { parts[i] = strings.ToUpper(strings.TrimSpace(written[i])) } lvl, isLevel := portionLevels[parts[0]] if !isLevel { return m } m.Level = lvl if parts[0] == "FOUO" { m.Controls = append(m.Controls, "FOUO") } known := true for _, p := range parts[1:] { if p == "" { continue } m.Controls = append(m.Controls, p) if !portionControlKnown(p) { known = false } } m.OK = known m.Canonical = m.Level if len(m.Controls) > 0 { m.Canonical += "//" + strings.Join(m.Controls, "//") } return m } func portionControlKnown(p string) bool { if portionControls[p] { return true } if strings.HasPrefix(p, "REL") { return true } if strings.HasPrefix(p, "ECI ") || strings.HasPrefix(p, "SI-ECI") { return true } return false } func extractPortions(text string) []portionMark { var out []portionMark fenced := false for _, line := range strings.Split(text, "\n") { t := strings.TrimSpace(line) if strings.HasPrefix(t, "```") || strings.HasPrefix(t, "~~~") { fenced = !fenced continue } if fenced || strings.HasPrefix(t, ">") { continue } m := rePortion.FindStringSubmatch(line) if m == nil { continue } if p := parsePortion(m[1]); p.Level != "" { out = append(out, p) } } return out } // --- the corpus view ------------------------------------------------------- type portionRow struct { Name string // the canonical marking, a level, or a control Label string // the spelled-out level, where there is one Marks int // occurrences across the corpus Docs int // documents carrying it SearchURL template.URL } type portionSummary struct { Docs int // documents carrying at least one marking Marks int // markings found Distinct int // distinct canonical markings Unrecog int // markings whose parts are not all known UnrecogEg []portionRow Levels []portionRow Markings []portionRow Controls []portionRow } const portionRows = 200 func (co *Corpus) buildPortions() { co.portionStats = portionSummary{} markMarks, lvlMarks, ctlMarks := map[string]int{}, map[string]int{}, map[string]int{} docsBy := map[string]map[string]int{"PL": {}, "PM": {}, "PC": {}} unrecogMarks := map[string]int{} for i := range co.archive.Docs { d := &co.archive.Docs[i] d.PortionLevels, d.PortionMarkings, d.PortionControls = nil, nil, nil marks := extractPortions(d.FullText) if len(marks) == 0 { continue } co.portionStats.Docs++ co.portionStats.Marks += len(marks) for _, m := range marks { if !m.OK { co.portionStats.Unrecog++ unrecogMarks[m.Raw]++ continue } markMarks[m.Canonical]++ lvlMarks[m.Level]++ d.PortionMarkings = append(d.PortionMarkings, m.Canonical) d.PortionLevels = append(d.PortionLevels, m.Level) for _, c := range m.Controls { ctlMarks[c]++ d.PortionControls = append(d.PortionControls, c) } } d.PortionLevels = normalizeList(d.PortionLevels) d.PortionMarkings = normalizeList(d.PortionMarkings) d.PortionControls = normalizeList(d.PortionControls) for code, docs := range docsBy { for _, v := range facetValues(d, code) { docs[v]++ } } } co.portionStats.Distinct = len(markMarks) for _, l := range portionLevelNames { if lvlMarks[l] == 0 { continue } co.portionStats.Levels = append(co.portionStats.Levels, portionRow{ Name: l, Label: portionLevelLabels[l], Marks: lvlMarks[l], Docs: docsBy["PL"][l], SearchURL: portionSearchURL("PL", l), }) } co.portionStats.Markings = portionRowsOf(markMarks, docsBy["PM"], portionRows) for i := range co.portionStats.Markings { r := &co.portionStats.Markings[i] r.SearchURL = portionSearchURL("PM", r.Name) } co.portionStats.Controls = portionRowsOf(ctlMarks, docsBy["PC"], portionRows) for i := range co.portionStats.Controls { r := &co.portionStats.Controls[i] r.SearchURL = portionSearchURL("PC", r.Name) } co.portionStats.UnrecogEg = portionRowsOf(unrecogMarks, nil, 25) log.Printf("Portion markings: %d in %d documents, %d distinct (%d unrecognised)", co.portionStats.Marks, co.portionStats.Docs, co.portionStats.Distinct, co.portionStats.Unrecog) } func portionSearchURL(code, value string) template.URL { return template.URL(wholeArchive.link(searchURL([]searchClause{{Field: code, Value: exactClause(value)}}))) } func portionRowsOf(marks, docs map[string]int, n int) []portionRow { names := rankCounts(marks, n) out := make([]portionRow, len(names)) for i, k := range names { out[i] = portionRow{Name: k, Marks: marks[k], Docs: docs[k]} } return out } func (co *Corpus) handlePortions(w http.ResponseWriter, r *http.Request) { render(w, r, &portionsPage{ Title: "Portion markings", Stats: co.portionStats, RowLimit: portionRows, }) }