119 lines
4.1 KiB
Go
119 lines
4.1 KiB
Go
package main
|
|
|
|
import (
|
|
"reflect"
|
|
"testing"
|
|
)
|
|
|
|
func TestParsePortion(t *testing.T) {
|
|
cases := []struct {
|
|
in string
|
|
canonical string
|
|
level string
|
|
ok bool
|
|
}{
|
|
{"U", "U", "U", true},
|
|
{"U//FOUO", "U//FOUO", "U", true},
|
|
{"TS//SI//NF", "TS//SI//NF", "TS", true},
|
|
{"S//REL TO USA, FVEY", "S//REL TO USA, FVEY", "S", true},
|
|
// OCR writes the separator every way; one marking, one canonical form.
|
|
{"S/NF", "S//NF", "S", true},
|
|
{"U/FOUO", "U//FOUO", "U", true},
|
|
{"TS/SI//REL", "TS//SI//REL", "TS", true},
|
|
{"TS//SI/NF", "TS//SI//NF", "TS", true},
|
|
{"TS//STLW//SI//OC/NF", "TS//STLW//SI//OC//NF", "TS", true},
|
|
{"TS//SI// ECI SOL", "TS//SI//ECI SOL", "TS", true},
|
|
// FOUO alone is a control on unclassified material.
|
|
{"FOUO", "U//FOUO", "U", true},
|
|
// Spelled out.
|
|
{"TOP SECRET//COMINT", "TS//COMINT", "TS", true},
|
|
// A level with a part no rule covers: recognised as a marking, but
|
|
// not counted - the Explore page lists these separately.
|
|
{"S//SI) (DNI collection", "", "S", false},
|
|
{"S//WIBBLE", "S//WIBBLE", "S", false},
|
|
}
|
|
for _, c := range cases {
|
|
got := parsePortion(c.in)
|
|
if got.Level != c.level {
|
|
t.Errorf("parsePortion(%q).Level = %q, want %q", c.in, got.Level, c.level)
|
|
}
|
|
if got.OK != c.ok {
|
|
t.Errorf("parsePortion(%q).OK = %v, want %v", c.in, got.OK, c.ok)
|
|
}
|
|
if c.ok && got.Canonical != c.canonical {
|
|
t.Errorf("parsePortion(%q).Canonical = %q, want %q", c.in, got.Canonical, c.canonical)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The things that merely look like markings: list enumerators, FOIA
|
|
// exemption citations, asides. Each of these appears hundreds of times.
|
|
func TestParsePortionRejectsNonMarkings(t *testing.T) {
|
|
for _, s := range []string{"1", "2", "a", "b", "iii", "d", "Cont'd.", "continued...",
|
|
"Dollars in Thousands", "b)(1), (b)(3", "A", "", " "} {
|
|
if got := parsePortion(s); got.Level != "" {
|
|
t.Errorf("parsePortion(%q) read a level %q - it is not a marking", s, got.Level)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestExtractPortions(t *testing.T) {
|
|
text := "" +
|
|
"TOP SECRET//COMINT\n" +
|
|
"\n" +
|
|
"(U) Some Tips on Portion Marking\n" +
|
|
"\n" +
|
|
"(U//FOUO) The lead-in sentence MUST ALWAYS be portion-marked.\n" +
|
|
"* (S//SI) a bullet carrying a complete thought\n" +
|
|
"* an unmarked bullet, covered by its lead-in\n" +
|
|
"## (TS//SI//NF) A marked heading\n" +
|
|
"1. (C) a numbered portion\n" +
|
|
"(1) not a marking, a list number\n" +
|
|
"text mentioning (U//FOUO) mid-line is not a portion mark\n"
|
|
got := make([]string, 0, 5)
|
|
for _, m := range extractPortions(text) {
|
|
got = append(got, m.Canonical)
|
|
}
|
|
want := []string{"U", "U//FOUO", "S//SI", "TS//SI//NF", "C"}
|
|
if !reflect.DeepEqual(got, want) {
|
|
t.Errorf("extractPortions = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
// A document quoting a marking is not marked by it. Fenced blocks and block
|
|
// quotes are where a transcript puts an example, and they are the curator's
|
|
// correction surface: wrap the example, and it stops being counted.
|
|
func TestExtractPortionsSkipsQuotedExamples(t *testing.T) {
|
|
text := "" +
|
|
"(U) Wrong way:\n" +
|
|
"\n" +
|
|
"> (U) All good men and women should come to the aid of their country through:\n" +
|
|
"> * (U//FOUO) volunteering for a tdy to the Middle East\n" +
|
|
"\n" +
|
|
"```\n" +
|
|
"(TS//SI//NF) an example inside a fenced block\n" +
|
|
"```\n" +
|
|
"(S) a real portion after the block\n"
|
|
var got []string
|
|
for _, m := range extractPortions(text) {
|
|
got = append(got, m.Canonical)
|
|
}
|
|
want := []string{"U", "S"}
|
|
if !reflect.DeepEqual(got, want) {
|
|
t.Errorf("extractPortions = %v, want %v (quoted and fenced examples must not count)", got, want)
|
|
}
|
|
}
|
|
|
|
// Markings are written in capitals. Court filings enumerate subsections
|
|
// "(a)", "(b)", "(c)" at the start of a line, and "(c)" is not CONFIDENTIAL.
|
|
func TestParsePortionNeedsCapitals(t *testing.T) {
|
|
for _, s := range []string{"c", "s", "u", "ts", "s//si", "Ts", "secret"} {
|
|
if got := parsePortion(s); got.Level != "" {
|
|
t.Errorf("parsePortion(%q) read level %q from a lower-case label", s, got.Level)
|
|
}
|
|
}
|
|
// The controls after a capital level are read whatever their case.
|
|
if got := parsePortion("S//si"); got.Canonical != "S//SI" || !got.OK {
|
|
t.Errorf("parsePortion(S//si) = %+v, want S//SI", got)
|
|
}
|
|
}
|