From 8c4dcae475e4378063a8d91d33da092b3557c81d Mon Sep 17 00:00:00 2001 From: libroot Date: Thu, 19 Nov 2026 04:40:59 +0200 Subject: [PATCH] first commit --- LICENSE | 661 ++++++++ README.md | 177 ++ agencies.go | 60 + api.go | 335 ++++ articles.go | 211 +++ assemblies.go | 273 +++ browse.go | 2273 +++++++++++++++++++++++++ changelog.go | 102 ++ codewords.go | 298 ++++ corpus.go | 162 ++ data.go | 1470 +++++++++++++++++ explore.go | 377 +++++ explore_ids.go | 114 ++ glossary.go | 135 ++ go.mod | 3 + history.go | 194 +++ main.go | 773 +++++++++ mdparser.go | 591 +++++++ outlets.go | 132 ++ pages.go | 393 +++++ pins.go | 145 ++ portions.go | 230 +++ prefs.go | 262 +++ search.go | 2644 ++++++++++++++++++++++++++++++ seo.go | 233 +++ static/main.js | 415 +++++ static/style.css | 2353 ++++++++++++++++++++++++++ templates/404.html | 14 + templates/about.html | 544 ++++++ templates/article.html | 25 + templates/articles.html | 39 + templates/assemblies.html | 32 + templates/assembly.html | 60 + templates/base.html | 35 + templates/browse.html | 155 ++ templates/category.html | 59 + templates/changelog.html | 25 + templates/classpatterns.html | 50 + templates/codewords.html | 38 + templates/doc.html | 272 +++ templates/explore.html | 50 + templates/explore_table.html | 35 + templates/glossary.html | 36 + templates/ids.html | 63 + templates/index.html | 150 ++ templates/list.html | 95 ++ templates/outlets.html | 54 + templates/pages.html | 40 + templates/partials.html | 412 +++++ templates/pinned.html | 46 + templates/portions.html | 82 + templates/preferences.html | 97 ++ templates/search.html | 280 ++++ templates/version.html | 59 + tests/api_test.go | 122 ++ tests/assemblies_test.go | 211 +++ tests/bench_test.go | 312 ++++ tests/bugs_test.go | 78 + tests/consistency_test.go | 538 ++++++ tests/corpus_test.go | 199 +++ tests/data_test.go | 123 ++ tests/fixture_test.go | 295 ++++ tests/flags_test.go | 167 ++ tests/foia_test.go | 272 +++ tests/go.mod | 5 + tests/headings_test.go | 53 + tests/http_test.go | 231 +++ tests/ids_test.go | 99 ++ tests/image_sizes_test.go | 53 + tests/invariants_test.go | 277 ++++ tests/loader_test.go | 166 ++ tests/main_test.go | 74 + tests/mdparser_test.go | 145 ++ tests/outlets_test.go | 54 + tests/portions_test.go | 119 ++ tests/prefs_matrix_test.go | 153 ++ tests/prefs_test.go | 117 ++ tests/probes_test.go | 126 ++ tests/published_together_test.go | 43 + tests/query_test.go | 193 +++ tests/related_test.go | 77 + tests/reporting_only_test.go | 49 + tests/routes_test.go | 705 ++++++++ tests/run.sh | 22 + tests/search_test.go | 1000 +++++++++++ tests/sidtoday_scope_test.go | 58 + tests/tmplcheck_test.go | 464 ++++++ tests/versions_pages_test.go | 59 + tests/versions_search_test.go | 101 ++ 89 files changed, 24623 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 agencies.go create mode 100644 api.go create mode 100644 articles.go create mode 100644 assemblies.go create mode 100644 browse.go create mode 100644 changelog.go create mode 100644 codewords.go create mode 100644 corpus.go create mode 100644 data.go create mode 100644 explore.go create mode 100644 explore_ids.go create mode 100644 glossary.go create mode 100644 go.mod create mode 100644 history.go create mode 100644 main.go create mode 100644 mdparser.go create mode 100644 outlets.go create mode 100644 pages.go create mode 100644 pins.go create mode 100644 portions.go create mode 100644 prefs.go create mode 100644 search.go create mode 100644 seo.go create mode 100644 static/main.js create mode 100644 static/style.css create mode 100644 templates/404.html create mode 100644 templates/about.html create mode 100644 templates/article.html create mode 100644 templates/articles.html create mode 100644 templates/assemblies.html create mode 100644 templates/assembly.html create mode 100644 templates/base.html create mode 100644 templates/browse.html create mode 100644 templates/category.html create mode 100644 templates/changelog.html create mode 100644 templates/classpatterns.html create mode 100644 templates/codewords.html create mode 100644 templates/doc.html create mode 100644 templates/explore.html create mode 100644 templates/explore_table.html create mode 100644 templates/glossary.html create mode 100644 templates/ids.html create mode 100644 templates/index.html create mode 100644 templates/list.html create mode 100644 templates/outlets.html create mode 100644 templates/pages.html create mode 100644 templates/partials.html create mode 100644 templates/pinned.html create mode 100644 templates/portions.html create mode 100644 templates/preferences.html create mode 100644 templates/search.html create mode 100644 templates/version.html create mode 100644 tests/api_test.go create mode 100644 tests/assemblies_test.go create mode 100644 tests/bench_test.go create mode 100644 tests/bugs_test.go create mode 100644 tests/consistency_test.go create mode 100644 tests/corpus_test.go create mode 100644 tests/data_test.go create mode 100644 tests/fixture_test.go create mode 100644 tests/flags_test.go create mode 100644 tests/foia_test.go create mode 100644 tests/go.mod create mode 100644 tests/headings_test.go create mode 100644 tests/http_test.go create mode 100644 tests/ids_test.go create mode 100644 tests/image_sizes_test.go create mode 100644 tests/invariants_test.go create mode 100644 tests/loader_test.go create mode 100644 tests/main_test.go create mode 100644 tests/mdparser_test.go create mode 100644 tests/outlets_test.go create mode 100644 tests/portions_test.go create mode 100644 tests/prefs_matrix_test.go create mode 100644 tests/prefs_test.go create mode 100644 tests/probes_test.go create mode 100644 tests/published_together_test.go create mode 100644 tests/query_test.go create mode 100644 tests/related_test.go create mode 100644 tests/reporting_only_test.go create mode 100644 tests/routes_test.go create mode 100755 tests/run.sh create mode 100644 tests/search_test.go create mode 100644 tests/sidtoday_scope_test.go create mode 100644 tests/tmplcheck_test.go create mode 100644 tests/versions_pages_test.go create mode 100644 tests/versions_search_test.go diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..be3f7b2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f392a81 --- /dev/null +++ b/README.md @@ -0,0 +1,177 @@ +# Snowchive server + +A self-contained web server for browsing and searching an archive of published +Snowden documents in Snowchive website. The server includes faceted browsing, a +boolean/fielded search engine with stemming and accent folding, a SIDtoday +serial reader, a read-only JSON API, sitemaps, and per-visitor preferences, pins +and history that live only in the visitor's cookies. The server uses just Go's +standard library. + +No JavaScript is required to use the website. A file in `static/main.js` adds +progressive enhancements only. No database and no external services. + +See it in action: https://snowchive.org + +If you are looking for Snowchive data or metadata: + +Data (PDFs, page images): + +https://snowchive.org/about#downloading-all-data + +Metadata (with extracted texts): + +https://git.disroot.org/libroot/snowchive-meta + +## Run + +Full Snowchive data is required (containing all the PDFs and metadata) to run +the server. If you don't yet have the data, download it first: +https://snowchive.org/about#downloading-all-data + +```sh +go run . -root=/path/to/snowchive-data/ +``` + +## Building + +If you want to build a binary to deploy one executable for production use: + +```sh +go build -ldflags "-X main.buildCommit=$(git rev-parse --short HEAD)" -o snowchive . +./snowchive -root=/path/to/snowchive-data/ -base=https://snowchive.example.org +``` + +## Flags + +| flag | purpose | +|---|---| +| `-root` | Snowchive data directory holding `data/` (the corpus, one directory per document) and `project/` (`glossary.json`, `assemblies.json`, `changelog.json`, `related.json`) (default: the working directory, `$SNOWCHIVE_ROOT`) | +| `-listen` | TCP address (default `127.0.0.1:8080`, `$SNOWCHIVE_LISTEN`). | +| `-base` | canonical `scheme://host` for absolute URLs in robots.txt and the sitemap; set it on anything internet-facing. Without it those URLs follow each request's host. | +| `-no-api` | disable the JSON API (`$SNOWCHIVE_NO_API=1`) | +| `-accel-files PREFIX` | behind nginx only: answer document files with an `X-Accel-Redirect` into this internal location (e.g. `/_files/`, an `alias` of the data directory), so nginx sends the bytes and this server only decides whether the file is served (`$SNOWCHIVE_ACCEL_FILES`) | +| `-accel-expires DURATION` | let the proxy cache a page served to a reader without cookies for this long (e.g. `10m`, sent as `X-Accel-Expires`); browsers keep the 10-second `max-age`. Clear the proxy cache after a restart with new data (`$SNOWCHIVE_ACCEL_EXPIRES`) | +| `-dev` | serve `templates/` and `static/` from the working directory, re-parse per request, verbose errors. Never in production. | +| `-check` | load the corpus, print the warnings and the tally, then exit non-zero if anything was reported - run it after a curation pass or in CI. | +| `-rebuild-related` | compute `related.json` entries for documents that lack one, or whose list links to a document no longer in the corpus, drop the entries of documents that are gone, then exit. It only fills gaps, so after a scoring change delete `related.json` first to rebuild every list. | + +## What it costs to run + +Measured on a 2-core VM against the 3,106-document corpus: ~7 seconds to load +(warm page cache), ~260 MB of live heap, and ~750 MB of address space under +sustained concurrent load. + +## Directory layout + +``` +data/ + / one directory per document + metadata.json document metadata + .pdf document PDF file + .md extracted text transcript from PDF + thumb.jpg, thumb.gif document thumbnail images + images/page-N.jpg PDF page scans + versions// another published copy that differs (e.g. less + redacted, extra pages): its own PDF, + .md and images/ (optional) + FOIA// the FOIA document collection + SIDtoday-archive// the bulk SIDtoday document collection +``` + +The files about the archive as a whole are in the project directory +(`-project`): + +``` +project/ + glossary.json format documented in glossary.go + assemblies.json see "Assemblies" below + changelog.json snowchive changelog, format in changelog.go + related.json documents relating to each other, generated by + -rebuild-related +``` + +The corpus is read exactly once at startup and everything derived from it is +computed there. All of it lives in one `Corpus` value (`corpus.go`), which is +built whole and pre-rendered. The server writes nothing while it runs. Only +`-rebuild-related` writes a file (`related.json`, in the project directory). + +## About metadata + +`metadata.json` carries the title, description, tags, the `document` +block (creator, date_created, programs, classification, dissemination, +banner, genre, series, countries, people) and the `publication` list +(publisher, title, link, reporters, contributors, date_published, plus +`doc_source` and `doc_url` for the copy that entry published, and `note` +and `version` when that copy differs from the primary one). Every field +except `title` is optional. There is no document-level `doc_source`/`doc_url`, +because where a copy came from is a property of the publication that published +it. + +A publication entry is an outlet that published the document. A partner in a +joint investigation that published the story but no documents is the one +exception (they have entry `"reporting_only": true`). + +A publication that released a different copy of the same document is in +`version` directory. The directory `versions//` is in the document +directory, holding that copy's PDF, `.md` and `images/`. The document page shows +the publication's `note` and links the copy, which has its own page at +`/doc//versions/`. Versions never count as documents. Search covers +the lines of a version's transcript that the main transcript lacks; results +label those matches with the copy they came from. A version no publication names +is listed as another copy. + +The `versions` object records, per version directory, the SHA-256 of that copy's +PDF and the pages that differ from the main document, by the number in the page +image's filename: +`"versions": {"": {"pdf_sha256": "...", "changed_pages": [7, 12]}}`. + +`aka_id` lists other names the document was published under, as an array of +objects: `{"id": "...", "doc_source": "...", "doc_url": "..."}`. An aka is a +pointer and stores no file, which is what separates it from a `versions/` entry. + +More info about the metadata: https://git.disroot.org/libroot/snowchive-meta + +## Assemblies + +`assemblies.json` is a hand-maintained list of sets of pages drawn from several +documents and versions, for material that was published in pieces, such as a +slide deck. Each gets a page at `/explore/assemblies/` showing all its +slides in one grid and lightbox, listed at `/explore/assemblies` + +```json +[{ + "id": "prism-overview", + "title": "PRISM/US-984XN briefing", + "note": "Shown at the top: how the set was put together and what is unproven.", + "slides": [ + {"doc": "", "pages": "1-7"}, + {"doc": "", "version": "", "pages": "8", "note": "Shown under the slide."}, + {"doc": "", "pages": "9,11", + "alt": [{"doc": "", "version": "", "pages": "1-2"}]} + ] +}] +``` +An assembly is the archive's own arrangement, it's not an original document and +stores no files, and is neither counted nor searched. + +## Tests + +./tests/run.sh + +TODO. The tests are currently written largely by Claude. + +## Other/contact + +The maintainers of this project do not endorse any public Snowchive instance +other than the official snowchive.org instance (unless you are a reputable +organization that wants to host a public instance). + +Please submit issues/feedback on the Snowchive server's repository: + +https://git.disroot.org/libroot/snowchive-server + +or to Codeberg (mirror): https://codeberg.org/libroot/snowchive-server + +Contact the maintainers via email: + +https://libroot.org/about diff --git a/agencies.go b/agencies.go new file mode 100644 index 0000000..a20d594 --- /dev/null +++ b/agencies.go @@ -0,0 +1,60 @@ +// agencies.go - display names for the acronyms used as document creators + +package main + +import "strings" + +var agencyNames = map[string]string{ + "CSE": "Communications Security Establishment (Canada)", + "DFS": "Directorate for Signals Intelligence (Japan)", + "DNI": "Director of National Intelligence (US)", + "DOD": "Department of Defense (US)", + "DOJ": "Department of Justice (US)", + "DOS": "Department of State (US)", + "DSD": "Defence Signals Directorate (Australia; now ASD)", + "FISC": "Foreign Intelligence Surveillance Court (US)", + "FVEY": "Five Eyes (US, UK, Canada, Australia, New Zealand)", + "GCHQ": "Government Communications Headquarters (UK)", + "GCSB": "Government Communications Security Bureau (New Zealand)", + "MI5": "Security Service (UK)", + "NATO": "North Atlantic Treaty Organization", + "NCTC": "National Counterterrorism Center (US)", + "NSA": "National Security Agency (US)", + "ODNI": "Office of the Director of National Intelligence (US)", + "SIS": "Secret Intelligence Service (UK)", + "SWAC": "Southwest Analysis Center", + "USG": "United States Government", +} + +func agencyName(creator string) string { + var out strings.Builder + known := false + start := 0 + flush := func(end int) { + part := creator[start:end] + key := strings.ToUpper(strings.TrimSpace(part)) + if name, ok := agencyNames[key]; ok { + known = true + out.WriteString(name) + } else { + out.WriteString(strings.TrimSpace(part)) + } + } + for i := 0; i < len(creator); i++ { + switch creator[i] { + case '/': + flush(i) + out.WriteString(" / ") + start = i + 1 + case ',': + flush(i) + out.WriteString(", ") + start = i + 1 + } + } + flush(len(creator)) + if !known { + return "" + } + return out.String() +} diff --git a/api.go b/api.go new file mode 100644 index 0000000..4afc0e0 --- /dev/null +++ b/api.go @@ -0,0 +1,335 @@ +// api.go - read-only JSON API mirroring the UI's features. +// +// Endpoints (all GET): +// +// /api/docs list documents; params: coll (repeatable: +// news, sidtoday-archive, foia), type +// (repeatable), sort (any list-page sort: title, +// agency, date, pubdate, classification, ...; see +// docSorts), order (asc|desc), p, per_page (max +// 500, or "all" for exports) +// /api/doc/{id} full document: metadata, full text, images, +// related IDs, published-together IDs, +// version_copies (other published copies: PDF, +// page images, transcript, publication indices) +// /api/search params: q (simple) OR fqv/fqf/fqc clause +// triples (same field codes and value syntax as +// the UI: word, "text", =exact, *wildcard*), +// plus coll and type (both repeatable, as on +// /api/docs; no coll means every collection), +// date_doc_start/_end, date_pub_start/_end +// (partial END dates are inclusive: 2013 means +// through 2013-12-31), sort, order, and the same +// p / per_page pagination as /api/docs (default +// 50, max 500, "all" for every match without +// snippets see apiPageSize). results come +// as one flat list, mirroring the search page: +// all collections rank equally. a result whose match +// is in another published copy of the document +// lists that copy in matched_versions (slug, +// label, url, snippet) +// /api/facets facet maps (whole corpus, like the UI; the +// one news-scoped map is news_types) plus +// collection counts +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +const ( + apiPerPageDefault = 50 + apiMaxPerPage = 500 +) + +func writeJSON(w http.ResponseWriter, status int, v interface{}) { + b, err := json.Marshal(v) + if err != nil { + log.Printf("API: marshal error: %v", err) + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(`{"error":"internal error"}`)) + return + } + writeJSONBytes(w, status, b) +} + +func writeJSONBytes(w http.ResponseWriter, status int, b []byte) { + setJSONHeaders(w) + w.WriteHeader(status) + w.Write(b) +} + +func setJSONHeaders(w http.ResponseWriter) { + cc := w.Header().Get("Cache-Control") + if cc == "" { + cc = "public, max-age=60" + w.Header().Set("Cache-Control", cc) + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + if !strings.Contains(cc, "private") { + w.Header().Set("Access-Control-Allow-Origin", "*") + } +} + +func apiError(w http.ResponseWriter, status int, msg string) { + writeJSON(w, status, map[string]string{"error": msg}) +} + +type apiDocSummary struct { + Document + URL string `json:"url"` + PDF string `json:"pdf"` +} + +func docSummary(d Document) apiDocSummary { + return apiDocSummary{ + Document: d, + URL: docPath(d.ID), + PDF: filePath(d.ID, d.PDFFile), + } +} + +type apiDocDetail struct { + apiDocSummary + FullText string `json:"full_text,omitempty"` + Images []string `json:"images,omitempty"` + Related []string `json:"related,omitempty"` + PublishedTogether []string `json:"published_together,omitempty"` + VersionCopies []apiVersion `json:"version_copies,omitempty"` +} + +type apiSearchResult struct { + apiDocSummary + Score float64 `json:"score"` + Snippet string `json:"snippet,omitempty"` + MatchedVersions []apiMatchedVersion `json:"matched_versions,omitempty"` +} + +func (co *Corpus) handleAPIDocs(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + + typeFilter, _ := selectedTypes(q) + + // coll=, repeatable, as /api/search and the pages take it. + pool := filterByCollection(co.everyDoc, collectionFilter(q)) + docs := filterByType(append([]*Document(nil), pool...), typeFilter) + + sortDocs(docs, q.Get("sort"), q.Get("order")) + + perPage := apiPageSize(q) + page, _ := strconv.Atoi(q.Get("p")) + start, end, page, totalPages := paginate(len(docs), page, perPage) + + out := make([]apiDocSummary, 0, end-start) + for _, d := range docs[start:end] { + out = append(out, docSummary(*d)) + } + + writeJSON(w, http.StatusOK, map[string]interface{}{ + "total": len(docs), + "page": page, + "per_page": apiRespPerPage(perPage, len(docs)), + "total_pages": totalPages, + "docs": out, + }) +} + +func apiPageSize(q url.Values) int { + if q.Get("per_page") == "all" { + return 0 + } + if v, err := strconv.Atoi(q.Get("per_page")); err == nil && v > 0 { + if v > apiMaxPerPage { + return apiMaxPerPage + } + return v + } + return apiPerPageDefault +} + +func apiRespPerPage(perPage, total int) int { + if perPage > 0 { + return perPage + } + if total == 0 { + return 1 + } + return total +} + +func (co *Corpus) handleAPIDoc(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + doc := co.docIndex[id] + if doc == nil { + apiError(w, http.StatusNotFound, "document not found") + return + } + + var pubIDs []string + for _, d := range co.publishedTogether(doc) { + pubIDs = append(pubIDs, d.ID) + } + + writeJSON(w, http.StatusOK, apiDocDetail{ + apiDocSummary: docSummary(*doc), + FullText: doc.FullText, + Images: doc.Images, + Related: co.archive.RelatedDocs[id], + PublishedTogether: pubIDs, + VersionCopies: apiVersions(doc), + }) +} + +func (co *Corpus) handleAPISearch(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + + sp := readSearchParams(q) + if len(sp.Queries) == 0 { + apiError(w, http.StatusBadRequest, + "missing query: pass ?q=term or fqv/fqf/fqc clause parameters") + return + } + if sp.OverClauses > 0 { + apiError(w, http.StatusBadRequest, fmt.Sprintf( + "too many clauses: %d (the limit is %d)", sp.OverClauses, maxSearchClauses)) + return + } + if sp.OverWords > 0 { + apiError(w, http.StatusBadRequest, fmt.Sprintf( + "too many words: %d (the limit is %d)", sp.OverWords, maxSearchWords)) + return + } + scored, _ := co.executeSearch(sp) + snippetTerm, _ := snippetAnchor(sp.Queries, sp.Fields, sp.Conjs) + + perPage := apiPageSize(q) + page, _ := strconv.Atoi(q.Get("p")) + start, end, page, totalPages := paginate(len(scored), page, perPage) + withSnippets := snippetTerm != "" && perPage > 0 + + results := make([]apiSearchResult, 0, end-start) + for _, s := range scored[start:end] { + res := apiSearchResult{apiDocSummary: docSummary(*s.Doc), Score: s.Score} + if withSnippets { + if body := s.Doc.bodySnippets(snippetTerm, 1); len(body) > 0 { + res.Snippet = body[0] + } + for _, h := range versionHits(s.Doc, snippetTerm, 1) { + res.MatchedVersions = append(res.MatchedVersions, apiMatchedVersion{ + Slug: h.Version.Slug, + Label: h.Version.Label, + URL: versionPath(s.Doc.ID, h.Version.Slug), + Snippet: h.Snippets[0], + }) + } + } + results = append(results, res) + } + + body := map[string]interface{}{ + "total": len(scored), + "page": page, + "per_page": apiRespPerPage(perPage, len(scored)), + "total_pages": totalPages, + "results": results, + } + if snippetTerm != "" && !withSnippets { + body["snippets_omitted"] = true + } + writeJSON(w, http.StatusOK, body) +} + +func (co *Corpus) buildFacetsJSON() { + b, err := json.Marshal(map[string]interface{}{ + "note": "facet counts cover the whole corpus; news_types covers the news collection only", + "collections": co.archive.CollectionCounts, + "sidtoday_total": co.archive.SIDTodayCount, + "types": co.archive.Types, + "news_types": co.archive.NewsTypes, + "agencies": co.archive.Facets["agency"], + "programs": co.archive.Facets["program"], + "journalists": co.archive.Facets["journalist"], + "publishers": co.archive.Facets["publisher"], + "countries": co.archive.Facets["country"], + "people": co.archive.Facets["people"], + "classification": co.archive.Facets["classification"], + "dissemination": co.archive.Facets["dissemination"], + "banners": co.archive.Facets["banner"], + "genres": co.archive.Facets["genre"], + "series": co.archive.Facets["series"], + "tags": co.archive.Facets["tag"], + "pub_years": co.archive.Facets["published"], + "doc_years": co.archive.Facets["docdate"], + "by_collection": co.archive.FacetsByColl, + }) + if err != nil { + log.Fatalf("facets: marshal failed: %v", err) + } + co.facetsJSON = b + sum := sha256.Sum256(b) + co.facetsETag = `"` + hex.EncodeToString(sum[:8]) + `"` + log.Printf("Facets JSON pre-built: %d bytes", len(co.facetsJSON)) +} + +func (co *Corpus) handleAPIFacets(w http.ResponseWriter, r *http.Request) { + setJSONHeaders(w) + w.Header().Set("ETag", co.facetsETag) + http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(co.facetsJSON)) +} + +// "version" as in document versions (metadata.json can have "versions" object) +type apiVersion struct { + Slug string `json:"slug"` + URL string `json:"url"` + PDF string `json:"pdf"` + PDFSha256 string `json:"pdf_sha256,omitempty"` + Images []string `json:"images,omitempty"` + FullText string `json:"full_text,omitempty"` + Publications []int `json:"publications,omitempty"` + ChangedPages []int `json:"changed_pages,omitempty"` +} + +func apiVersions(doc *Document) []apiVersion { + var out []apiVersion + for _, v := range doc.Versions { + out = append(out, apiVersion{ + Slug: v.Slug, + URL: versionPath(doc.ID, v.Slug), + PDF: versionFilePath(doc.ID, v.Slug, v.PDFFile), + PDFSha256: v.SHA, + Images: v.Images, + FullText: v.FullText, + Publications: v.PubIndex, + ChangedPages: changedPageNumbers(v.ChangedPages), + }) + } + return out +} + +type apiMatchedVersion struct { + Slug string `json:"slug"` + Label string `json:"label"` + URL string `json:"url"` + Snippet string `json:"snippet"` +} + +func changedPageNumbers(pages []ChangedPage) []int { + var out []int + for _, p := range pages { + out = append(out, p.Page) + } + return out +} diff --git a/articles.go b/articles.go new file mode 100644 index 0000000..aead1c8 --- /dev/null +++ b/articles.go @@ -0,0 +1,211 @@ +// articles.go - news articles as first-class objects. +// +// An article's identity is its publication LINK. Giving a publication entry +// a title is what promotes its article to a story. + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "net/http" + "net/url" + "slices" + "sort" + "strings" +) + +type Article struct { + Key string // short stable hash of the link (the /article/{key} path) + Title string + Link string + Host string // display fallback when Title is empty + Publisher string + Date string // earliest date_published seen + Journalists []string // union across docs, first-seen order + DocIDs []string + Book bool // published by a book publisher, not a news outlet +} + +// bookPublishers are the publication entries that are books, not news +// articles (Greenwald's No Place to Hide, Gellman's Dark Mirror, Poitras's +// Astro Noise). +var bookPublishers = map[string]bool{ + "Metropolitan Books": true, + "Penguin Press": true, + "Yale University Press": true, +} + +// articleKeyFor derives the stable URL key for a link +func articleKeyFor(link string) string { + h := sha256.Sum256([]byte(link)) + return hex.EncodeToString(h[:])[:12] +} + +func hostOf(link string) string { + u, err := url.Parse(link) + if err != nil || u.Host == "" { + return link + } + return strings.TrimPrefix(u.Host, "www.") +} + +// buildArticles indexes every publication entry that has a link +func (co *Corpus) buildArticles() { + co.articleByLink = make(map[string]*Article) + co.articleIndex = make(map[string]*Article) + co.articles = nil + + for _, d := range co.archive.Docs { + for _, p := range d.Publication { + if p.Link == "" { + continue + } + a := co.articleByLink[p.Link] + if a == nil { + a = &Article{ + Key: articleKeyFor(p.Link), + Link: p.Link, + Host: hostOf(p.Link), + } + co.articleByLink[p.Link] = a + co.articleIndex[a.Key] = a + } + if a.Title == "" && p.Title != "" { + a.Title = p.Title + } + if a.Publisher == "" && p.Publisher != "" { + a.Publisher = p.Publisher + a.Book = bookPublishers[p.Publisher] + } + if p.DatePublished != "" && (a.Date == "" || p.DatePublished < a.Date) { + a.Date = p.DatePublished + } + for _, j := range p.Journalists { + if j == "" { + continue + } + if !slices.ContainsFunc(a.Journalists, func(x string) bool { return sameValue(x, j) }) { + a.Journalists = append(a.Journalists, j) + } + } + dup := false + for _, id := range a.DocIDs { + if id == d.ID { + dup = true + break + } + } + if !dup { + a.DocIDs = append(a.DocIDs, d.ID) + } + } + } + + for _, a := range co.articleByLink { + co.articles = append(co.articles, *a) + } + // Newest first; undated articles sort last; ties by title then link so + // the order is deterministic. + sort.Slice(co.articles, func(i, j int) bool { + di, dj := co.articles[i].Date, co.articles[j].Date + if di == "" && dj != "" { + return false + } + if di != "" && dj == "" { + return true + } + if di != dj { + return di > dj + } + ti := strings.ToLower(displayTitle(co.articles[i])) + tj := strings.ToLower(displayTitle(co.articles[j])) + if ti != tj { + return ti < tj + } + return co.articles[i].Link < co.articles[j].Link + }) + + co.buildArticleGroups() +} + +// inCollection reports whether one of the article's documents is in coll. +func (a *Article) inCollection(co *Corpus, coll string) bool { + for _, id := range a.DocIDs { + if d := co.docIndex[id]; d != nil && d.Collection == coll { + return true + } + } + return false +} + +func displayTitle(a Article) string { + if a.Title != "" { + return a.Title + } + return a.Host +} + +type articleYearGroup struct { + Year string + Articles []Article +} + +func (co *Corpus) buildArticleGroups() { + co.articleGroups, co.articlesTitled, co.articlesBooks = nil, 0, 0 + for _, a := range co.articles { + if a.Title == "" { + continue + } + co.articlesTitled++ + if a.Book { + co.articlesBooks++ + } + year := "Undated" + if len(a.Date) >= 4 { + year = a.Date[:4] + } + if len(co.articleGroups) == 0 || co.articleGroups[len(co.articleGroups)-1].Year != year { + co.articleGroups = append(co.articleGroups, articleYearGroup{Year: year}) + } + co.articleGroups[len(co.articleGroups)-1].Articles = append(co.articleGroups[len(co.articleGroups)-1].Articles, a) + } +} + +func (co *Corpus) handleArticles(w http.ResponseWriter, r *http.Request) { + render(w, r, &articlesPage{ + Groups: co.articleGroups, + GroupsHTML: co.preHTML("articles"), + Total: co.articlesTitled - co.articlesBooks, + Books: co.articlesBooks, + }) +} + +func (co *Corpus) handleArticleView(w http.ResponseWriter, r *http.Request) { + a := co.articleIndex[r.PathValue("key")] + if a == nil { + notFound(w, r) + return + } + prefs := readPrefs(r) + scope := pageScope(nil, prefs, false) + docs := make([]*Document, 0, len(a.DocIDs)) + for _, id := range a.DocIDs { + if d := co.docIndex[id]; d != nil { + docs = append(docs, d) + scope = scope.including(d.Collection) + } + } + data := &articlePage{ + Base: pageWith(prefs), + Article: *a, + Total: len(docs), + BrowseQS: scope.suffix(), + Cards: newCardView(prefs, docs), + } + // A partner that only reported the story has no publisher page. + if scope.hasValue(co, "publisher", a.Publisher) { + data.PublisherURL = scope.link(categoryValueURL("publisher", a.Publisher)) + } + render(w, r, data) +} diff --git a/assemblies.go b/assemblies.go new file mode 100644 index 0000000..3e12058 --- /dev/null +++ b/assemblies.go @@ -0,0 +1,273 @@ +// assemblies.go - hand-built sets of pages drawn from several documents, +// loaded from data/assemblies.json. + +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" + "os" + "strconv" + "strings" +) + +type assemblyRef struct { + Doc string `json:"doc"` + Version string `json:"version,omitempty"` + Pages string `json:"pages"` +} + +type assemblySlideSpec struct { + assemblyRef + Note string `json:"note,omitempty"` + Alt []assemblyRef `json:"alt,omitempty"` +} + +type assemblySpec struct { + ID string `json:"id"` + Title string `json:"title"` + Note string `json:"note,omitempty"` + Slides []assemblySlideSpec `json:"slides"` +} + +type AssemblyCopy struct { + Image string + Size imageSize + DocID string + DocTitle string + Version string + Page int + Source string // the outlets that published this copy + Path string // the document or version page it came from +} + +type AssemblySlide struct { + AssemblyCopy + Pos int + Note string + Alts []AssemblyCopy +} + +type Assembly struct { + ID string + Title string + Note string + Slides []AssemblySlide + DocList []string // documents the slides are shown from, in slide order + Copies int // copies the slides are shown from: a document or one of its versions +} + +// parsePageList expands "1-3,7" into page numbers, keeping the order given. +func parsePageList(s string) ([]int, error) { + var out []int + for _, part := range strings.Split(s, ",") { + part = strings.TrimSpace(part) + first, last, isRange := strings.Cut(part, "-") + if !isRange { + last = first + } + a, err := strconv.Atoi(strings.TrimSpace(first)) + if err != nil { + return nil, fmt.Errorf("%q is not a page or a page range", part) + } + b, err := strconv.Atoi(strings.TrimSpace(last)) + if err != nil { + return nil, fmt.Errorf("%q is not a page or a page range", part) + } + if a < 1 || b < a { + return nil, fmt.Errorf("%q is not a page or a page range", part) + } + for n := a; n <= b; n++ { + out = append(out, n) + } + } + return out, nil +} + +// copySource names the outlets that published a copy, oldest entry first as +// the metadata lists them. Partners that published no documents are left out. +func copySource(d *Document, slug string) string { + var names []string + seen := map[string]bool{} + for _, p := range d.Publication { + key := valueKey(p.Publisher) + if p.ReportingOnly || p.Version != slug || key == "" || seen[key] { + continue + } + seen[key] = true + names = append(names, p.Publisher) + } + return strings.Join(names, ", ") +} + +func (co *Corpus) resolveAssemblyRef(label string, r assemblyRef) ([]*AssemblyCopy, bool) { + pages, err := parsePageList(r.Pages) + if err != nil { + warnf("%s: %s: pages: %v - skipped", label, r.Doc, err) + return nil, false + } + out := make([]*AssemblyCopy, len(pages)) + d := co.docIndex[r.Doc] + if d == nil { + warnf("%s: no document %q - skipped", label, r.Doc) + return out, true + } + images, sizes, base, path := d.Images, d.ImageSizes, imageBase(d.ID, ""), docPath(d.ID) + if r.Version != "" { + v := d.VersionFor(r.Version) + if v == nil { + warnf("%s: %s has no version %q - skipped", label, r.Doc, r.Version) + return out, true + } + images, sizes, base, path = v.Images, v.ImageSizes, imageBase(d.ID, v.Slug), versionPath(d.ID, v.Slug) + } + byNumber := make(map[int]int, len(images)) + for i, name := range images { + if n, err := pageNumber(name); err == nil { + byNumber[n] = i + } + } + source := copySource(d, r.Version) + for i, n := range pages { + k, ok := byNumber[n] + if !ok { + warnf("%s: %s %s has no page image %d - skipped", label, r.Doc, r.Version, n) + continue + } + out[i] = &AssemblyCopy{ + Image: base + url.PathEscape(images[k]), Size: sizes[k], DocID: d.ID, DocTitle: d.Title, + Version: r.Version, Page: n, Source: source, Path: path, + } + } + return out, true +} + +func (co *Corpus) buildAssembly(spec assemblySpec) *Assembly { + label := "assemblies.json: " + spec.ID + a := &Assembly{ID: spec.ID, Title: strings.TrimSpace(spec.Title), Note: strings.TrimSpace(spec.Note)} + seenImage := map[string]bool{} + seenDoc := map[string]bool{} + seenCopy := map[[2]string]bool{} + for _, s := range spec.Slides { + copies, ok := co.resolveAssemblyRef(label, s.assemblyRef) + if !ok { + continue + } + alts := make([][]*AssemblyCopy, 0, len(s.Alt)) + for _, ref := range s.Alt { + c, ok := co.resolveAssemblyRef(label, ref) + if !ok { + continue + } + if len(c) != len(copies) { + warnf("%s: alt %s lists %d pages for a run of %d - skipped", label, ref.Doc, len(c), len(copies)) + continue + } + alts = append(alts, c) + } + for i, c := range copies { + if c == nil { + continue + } + if seenImage[c.Image] { + warnf("%s: %s page %d is listed twice", label, c.DocID, c.Page) + } + seenImage[c.Image] = true + if !seenDoc[c.DocID] { + seenDoc[c.DocID] = true + a.DocList = append(a.DocList, c.DocID) + } + slide := AssemblySlide{AssemblyCopy: *c, Pos: len(a.Slides) + 1, Note: s.Note} + seenCopy[[2]string{c.DocID, c.Version}] = true + for _, alt := range alts { + if alt[i] != nil { + slide.Alts = append(slide.Alts, *alt[i]) + } + } + a.Slides = append(a.Slides, slide) + } + } + a.Copies = len(seenCopy) + return a +} + +func (co *Corpus) loadAssemblies() { + co.assemblies = nil + co.assemblyIndex = map[string]*Assembly{} + co.assembliesByDoc = map[string][]*Assembly{} + + p := projectPath("assemblies.json") + b, err := os.ReadFile(p) + if err != nil { + log.Printf("Note: %s not found - no assemblies.", p) + return + } + var specs []assemblySpec + if err := json.Unmarshal(b, &specs); err != nil { + warnf("%s is malformed: %v - assemblies disabled.", p, err) + return + } + for _, spec := range specs { + spec.ID = strings.TrimSpace(spec.ID) + if spec.ID == "" || !docIDSafe(spec.ID) { + warnf("assemblies.json: id %q is empty or unsafe in a URL - skipped", spec.ID) + continue + } + if co.assemblyIndex[spec.ID] != nil { + warnf("assemblies.json: id %q appears twice - keeping the first", spec.ID) + continue + } + a := co.buildAssembly(spec) + if len(a.Slides) == 0 { + warnf("assemblies.json: %s has no slides that can be shown - skipped", spec.ID) + continue + } + co.assemblies = append(co.assemblies, a) + co.assemblyIndex[a.ID] = a + linked := map[string]bool{} + link := func(id string) { + if !linked[id] { + linked[id] = true + co.assembliesByDoc[id] = append(co.assembliesByDoc[id], a) + } + } + for _, s := range a.Slides { + link(s.DocID) + for _, alt := range s.Alts { + link(alt.DocID) + } + } + } + log.Printf("Assemblies loaded: %d", len(co.assemblies)) +} + +// assemblyPath is an assembly's page; assemblies live under /explore. +const assembliesPath = "/explore/assemblies" + +func assemblyPath(id string) string { return assembliesPath + "/" + url.PathEscape(id) } + +func (co *Corpus) handleAssemblies(w http.ResponseWriter, r *http.Request) { + prefs := readPrefs(r) + render(w, r, &assembliesPage{ + Base: pageWith(prefs), + Assemblies: co.assemblies, + TextOnly: prefs.TextOnly, + }) +} + +func (co *Corpus) handleAssemblyView(w http.ResponseWriter, r *http.Request) { + a := co.assemblyIndex[r.PathValue("id")] + if a == nil { + notFound(w, r) + return + } + prefs := readPrefs(r) + render(w, r, &assemblyPage{ + Base: pageWith(prefs), + Assembly: a, + TextOnly: prefs.TextOnly, + }) +} diff --git a/browse.go b/browse.go new file mode 100644 index 0000000..528a8dd --- /dev/null +++ b/browse.go @@ -0,0 +1,2273 @@ +package main + +import ( + "bytes" + "cmp" + "fmt" + "html/template" + "math/rand" + "net/http" + "net/url" + "regexp" + "slices" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +type facetEntry struct { + Name string + Count int + URL string +} + +// facetLinks renders a metadata field's values as links to their category +// pages, wherever a card, a table row, a search result or a document page +// lists them. It takes the template's dict: +// +// Vals the values ("-" stands for none) +// Cat the category's slug +// Sep between the links: "dot", "space", or a comma when not given +// AsList one
  • per link instead +// Class the links' class +// Active the value of the page the links are on, when it is this +// category's: its own link is marked active +// QS the collection scope the links carry ("?coll=..."), when the page +// they open would otherwise leave out the document they come from +// +// The output is what html/template made of the same markup: every value +// goes through htmlEscape. +func facetLinks(p map[string]interface{}) (template.HTML, error) { + vals, ok := p["Vals"].([]string) + if !ok && p["Vals"] != nil { + return "", fmt.Errorf("facetLinks: Vals is %T, not a list of values", p["Vals"]) + } + if len(vals) == 0 { + return "-", nil + } + cat, _ := p["Cat"].(string) + sep, _ := p["Sep"].(string) + class, _ := p["Class"].(string) + active, _ := p["Active"].(string) + qs, _ := p["QS"].(string) + asList, _ := p["AsList"].(bool) + + var b strings.Builder + if asList { + b.WriteString(`
      `) + } + for i, v := range vals { + switch { + case asList: + b.WriteString("
    • ") + case i == 0: + case sep == "dot": + b.WriteString(` · `) + case sep == "space": + b.WriteString(" ") + default: + b.WriteString(", ") + } + on := active != "" && sameValue(v, active) + b.WriteString(`") + b.WriteString(htmlEscape(v)) + b.WriteString("") + if asList { + b.WriteString("
    • ") + } + } + if asList { + b.WriteString("
    ") + } + return template.HTML(b.String()), nil +} + +func htmlEscape(s string) string { + if !strings.ContainsAny(s, "\x00\"&'+<>") && isASCII(s) { + return s + } + var b strings.Builder + b.Grow(len(s) + 16) + for i := 0; i < len(s); { + r, w := utf8.DecodeRuneInString(s[i:]) + switch { + case r == 0: + b.WriteString("\uFFFD") + case r == '"': + b.WriteString(""") + case r == '&': + b.WriteString("&") + case r == '\'': + b.WriteString("'") + case r == '+': + b.WriteString("+") + case r == '<': + b.WriteString("<") + case r == '>': + b.WriteString(">") + case r >= 0xfdd0 && r <= 0xfdef, r >= 0xfff0 && r <= 0xffff: + fmt.Fprintf(&b, "&#x%x;", r) + default: + b.WriteString(s[i : i+w]) + } + i += w + } + return b.String() +} + +// rankCounts returns the keys of a count map in rank order (most counted +// first, ties alphabetically ignoring accents and case, then by the raw +// spelling) +func rankCounts(m map[string]int, n int) []string { + type keyed struct { + name, fold string + count int + } + all := make([]keyed, 0, len(m)) + for k, v := range m { + all = append(all, keyed{k, normFoldString(k), v}) + } + sort.Slice(all, func(i, j int) bool { + a, b := &all[i], &all[j] + if a.count != b.count { + return a.count > b.count + } + if a.fold != b.fold { + return a.fold < b.fold + } + return a.name < b.name + }) + if n >= 0 && len(all) > n { + all = all[:n] + } + out := make([]string, len(all)) + for i := range all { + out[i] = all[i].name + } + return out +} + +func topFacets(m map[string]int, n int) []facetEntry { + names := rankCounts(m, n) + out := make([]facetEntry, len(names)) + for i, k := range names { + out[i] = facetEntry{Name: k, Count: m[k]} + } + return out +} + +func barPct(count, denom, floor int) int { + if denom <= 0 || count <= 0 { + return 0 + } + pct := count * 100 / denom + if pct < floor { + pct = floor + } + return pct +} + +func sortedYears(m map[string]int) (years []string, maxCount int) { + years = make([]string, 0, len(m)) + for y, c := range m { + years = append(years, y) + if c > maxCount { + maxCount = c + } + } + sort.Strings(years) + return years, maxCount +} + +func yearBars(m map[string]int, urlFor func(y string) string) []timelineBar { + years, maxCount := sortedYears(m) + if maxCount == 0 { + return nil + } + bars := make([]timelineBar, 0, len(years)) + for _, y := range years { + bars = append(bars, timelineBar{ + Year: y, + Count: m[y], + Pct: barPct(m[y], maxCount, 1), + URL: urlFor(y), + }) + } + return bars +} + +type homeFigures struct { + TopAgencies, TopPrograms, TopCountries, TopPeople, + TopJournalists, TopPublishers, TopTags []facetEntry + DocHist, PubHist []timelineBar +} + +func (co *Corpus) buildHomeStats() { + co.homeStats = co.homeFigures(false) + co.homeStatsSIDtoday = co.homeFigures(true) + co.buildRandomPools() +} + +func (co *Corpus) homeFigures(sidtoday bool) homeFigures { + scope := scopeOf(map[string]bool{collectionNews: true, collectionArchive: sidtoday}) + top := func(slug string, n int) []facetEntry { + entries := topFacets(co.homeFacet(slug, sidtoday), n) + for i := range entries { + entries[i].URL = scope.link(categoryValueURL(slug, entries[i].Name)) + } + return entries + } + f := homeFigures{ + TopAgencies: top("agency", 10), + TopPrograms: top("program", 20), + TopCountries: top("country", 15), + TopPeople: top("people", 15), + TopJournalists: top("journalist", 12), + TopPublishers: top("publisher", 10), + TopTags: top("tag", 25), + } + f.DocHist, f.PubHist = co.homeHistograms(scope) + return f +} + +func (co *Corpus) homeFacet(slug string, sidtoday bool) map[string]int { + out := make(map[string]int) + for value, c := range co.archive.FacetsByColl[slug] { + n := c.News + if sidtoday { + n += c.Archive + } + if n > 0 { + out[value] = n + } + } + return out +} + +func (co *Corpus) homeHistograms(scope collScope) (docHist, pubHist []timelineBar) { + docYears := map[string]int{} + pubYears := map[string]int{} + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + if !scope.shows(d) { + continue + } + if y := docYear(d); y != "" { + docYears[y]++ + } + if y := d.FirstPubYear(); y != "" { + pubYears[y]++ + } + } + link := func(slug string) func(string) string { + return func(y string) string { return scope.link(categoryValueURL(slug, y)) } + } + return yearBars(docYears, link("docdate")), yearBars(pubYears, link("published")) +} + +func (co *Corpus) addFacetNav(b *Base, prefs Prefs) { + b.ShowCatBoxes = true + b.CatCounts = co.categoryValueCounts[prefs.IncludeFOIA] +} + +func (co *Corpus) handleIndex(w http.ResponseWriter, r *http.Request) { + prefs := readPrefs(r) + var recentDocs []*Document + if prefs.TrackRecent { + for _, id := range readIDListCookie(r, recentDocsCookie, recentDocsMax) { + if d := co.docIndex[id]; d != nil { + recentDocs = append(recentDocs, d) + } + } + } + if len(recentDocs) > 0 { + noStore(w) + } + + withSIDtoday := r.URL.Query().Get("sidtoday") == "1" + stats := co.homeStats + if withSIDtoday { + stats = co.homeStatsSIDtoday + } + render(w, r, &indexPage{ + Base: pageWith(prefs), + homeFigures: stats, + TrackRecent: prefs.TrackRecent, + RecentDocs: recentDocs, + SnowdenTotal: co.archive.CollectionCounts[collectionNews] + co.archive.CollectionCounts[collectionArchive], + ArchiveTotal: co.archive.CollectionCounts[collectionArchive], + FOIATotal: co.archive.CollectionCounts[collectionFOIA], + IncludeFOIA: prefs.IncludeFOIA, + ReportedTotal: co.archive.CollectionCounts[collectionNews], + WithSIDtoday: withSIDtoday, + }) +} + +// randomWeights says how often /random lands in each collection +var randomWeights = []struct { + coll string + weight int +}{ + {collectionNews, 65}, + {collectionArchive, 25}, + {collectionFOIA, 10}, +} + +func (co *Corpus) buildRandomPools() { + co.randomPools = map[string][]int{} + for i, d := range co.archive.Docs { + co.randomPools[d.Collection] = append(co.randomPools[d.Collection], i) + } +} + +func (co *Corpus) buildCollectionDocs() { + co.everyDoc = make([]*Document, len(co.archive.Docs)) + co.collectionDocs = make(map[string][]*Document, len(validCollections)) + co.sidtodayDocs = nil + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + co.everyDoc[i] = d + co.collectionDocs[d.Collection] = append(co.collectionDocs[d.Collection], d) + if d.Type == "SIDTODAY" { + co.sidtodayDocs = append(co.sidtodayDocs, d) + } + } +} + +func (co *Corpus) randomDoc(includeFOIA bool) *Document { + pool := func(coll string) []int { + if coll == collectionFOIA && !includeFOIA { + return nil + } + return co.randomPools[coll] + } + total := 0 + for _, rw := range randomWeights { + if len(pool(rw.coll)) > 0 { + total += rw.weight + } + } + if total == 0 { + return nil + } + n := rand.Intn(total) + for _, rw := range randomWeights { + p := pool(rw.coll) + if len(p) == 0 { + continue + } + if n < rw.weight { + return &co.archive.Docs[p[rand.Intn(len(p))]] + } + n -= rw.weight + } + return nil +} + +func (co *Corpus) handleRandom(w http.ResponseWriter, r *http.Request) { + d := co.randomDoc(readPrefs(r).IncludeFOIA) + if d == nil { + notFound(w, r) + return + } + http.Redirect(w, r, docPath(d.ID), http.StatusFound) +} + +func handleAbout(w http.ResponseWriter, r *http.Request) { + render(w, r, &aboutPage{}) +} + +func paginate(total, page, perPage int) (start, end, clampedPage, totalPages int) { + if perPage <= 0 { + return 0, total, 1, 1 + } + totalPages = (total + perPage - 1) / perPage + + if page < 1 { + page = 1 + } + maxPage := totalPages + if maxPage < 1 { + maxPage = 1 + } + if page > maxPage { + page = maxPage + } + start = (page - 1) * perPage + if start < 0 || start > total { + start = total + } + end = start + perPage + if end < start || end > total { + end = total + } + return start, end, page, totalPages +} + +// buildListView assembles everything the shared doc-cards / doc-table +// layouts need: view, sort, pagination, the link bases their partials +// build on, and the card display prefs. +// +// The page embeds what is returned and adds its own fields, including the +// cards' ActiveCategory and ActiveValue when it has a filter value for them +// to highlight. +func (co *Corpus) buildListView(q url.Values, prefs Prefs, docs []*Document, basePath string, params url.Values, defaultSort string, explicitView bool) listView { + return co.buildListViewBoosted(q, prefs, docs, basePath, params, defaultSort, explicitView, "") +} + +func (co *Corpus) buildListViewBoosted(q url.Values, prefs Prefs, docs []*Document, basePath string, params url.Values, defaultSort string, explicitView bool, boost string) listView { + view := q.Get("view") + sortBy := sortToken(q.Get("sort")) + order := sortToken(q.Get("order")) + if docSortByKey[sortBy] == nil { + sortBy = "" + } + + effSort := cmp.Or(sortBy, defaultSort) + sortDocs(docs, effSort, order) + if sortBy == "" && boost != "" { + boostTitleMatches(docs, boost) + } + + page, _ := strconv.Atoi(q.Get("p")) + finalDocs := docs + totalPages := 1 + if view == "table" { + page = 1 + } else { + var start, end int + start, end, page, totalPages = paginate(len(docs), page, prefs.cardsPerPage()) + finalDocs = docs[start:end] + } + + baseQ := cloneValues(params) + sortQ := cloneValues(params) + if view != "" && (explicitView || view == "table") { + baseQ.Set("view", view) + } + + lv := listView{ + Docs: finalDocs, + Total: len(docs), + Page: page, + TotalPages: totalPages, + View: view, + Sort: sortBy, + Order: order, + DefaultSort: defaultSort, + BaseURL: queryBase(basePath, baseQ), + SortURL: queryBase(basePath, sortQ), + TableHeads: sortHeads(queryBase(basePath, baseQ), docTableCols, effSort, order == "desc"), + Cards: newCardView(prefs, finalDocs), + } + if view == "table" { + lv.RowsHTML = co.buildDocRows(finalDocs, false) + } + return lv +} + +// selectedTypes reads the ?type= params into a set, and reports whether the +// request actually filters (no params means everything, not nothing). +func selectedTypes(v url.Values) (map[string]bool, bool) { + sel := make(map[string]bool, len(v["type"])) + for _, t := range v["type"] { + if t = strings.ToUpper(strings.TrimSpace(t)); t != "" { + sel[t] = true + } + } + return sel, len(sel) > 0 +} + +func typeSelected(d *Document, sel map[string]bool) bool { + return len(sel) == 0 || sel[d.Type] +} + +func typeCounts(docs []*Document) map[string]int { + counts := make(map[string]int) + for i := range docs { + counts[docs[i].Type]++ + } + return counts +} + +func filterByType(docs []*Document, sel map[string]bool) []*Document { + if len(sel) == 0 { + return docs + } + kept := docs[:0] + for _, d := range docs { + if typeSelected(d, sel) { + kept = append(kept, d) + } + } + return kept +} + +func allTypesSelected(counts map[string]int) map[string]bool { + sel := make(map[string]bool, len(counts)) + for t := range counts { + sel[t] = true + } + return sel +} + +var validCollections = map[string]bool{ + collectionNews: true, + collectionArchive: true, + collectionFOIA: true, +} + +var collectionOrder = []string{collectionNews, collectionArchive, collectionFOIA} + +func collectionParams(sel map[string]bool) []string { + var out []string + for _, c := range collectionOrder { + if sel[c] { + out = append(out, c) + } + } + return out +} + +// defaultCollections is what a page shows when its URL names no +// collection: FOIA releases are opt-in (Prefs.IncludeFOIA). +var defaultCollections = map[string]bool{collectionNews: true, collectionArchive: true} + +// collScope is a page's collection scope: the documents it shows and what +// its own links carry. Every link a page builds from its documents goes +// through link, so the number on a link is the number its click returns. +type collScope struct { + filter map[string]bool // the documents shown; nil: every collection + shown map[string]bool // the same as a set, never nil (the form's boxes) + defaulted bool // the URL named no collection: the preference decided + colls []string // the coll= the page's links carry; nil: none + extra string // more scope for the links (a publisher page's year) +} + +// scopeOf is an explicit scope: sel, with links that name it, so a count +// holds whatever the next reader's FOIA preference. +func scopeOf(sel map[string]bool) collScope { + s := collScope{shown: sel, colls: collectionParams(sel)} + if len(s.colls) < len(validCollections) { + s.filter = sel + } + return s +} + +// wholeArchive scopes links to all three collections: a count taken over +// the whole archive must not change with the reader's FOIA preference. +var wholeArchive = scopeOf(validCollections) + +// pageScope reads a page's scope: the coll= its URL names or, when it +// names none, the reader's preference, FOIA releases only when opted in. +func pageScope(v url.Values, p Prefs, pinDefault bool) collScope { + if _, given := v["coll"]; given { + if f := collectionFilter(v); f != nil { + return scopeOf(f) + } + return wholeArchive + } + s := scopeOf(defaultCollections) + if p.IncludeFOIA { + s = wholeArchive + } + s.defaulted = true + if !pinDefault { + s.colls = nil + } + return s +} + +func (s collScope) shows(d *Document) bool { return inCollections(d, s.filter) } + +func (s collScope) hasValue(co *Corpus, slug, value string) bool { + c := co.archive.FacetsByColl[slug][co.categoryNames[slug][valueKey(value)]] + if c == nil { + return false + } + in := func(coll string, n int) bool { return n > 0 && (s.filter == nil || s.filter[coll]) } + return in(collectionNews, c.News) || in(collectionArchive, c.Archive) || in(collectionFOIA, c.FOIA) +} + +func (s collScope) docs(docs []*Document) []*Document { return filterByCollection(docs, s.filter) } + +func (s collScope) explicit() bool { return s.filter != nil && !s.defaulted } + +func (s collScope) including(coll string) collScope { + if s.filter == nil || s.filter[coll] { + return s + } + return wholeArchive +} + +func (s collScope) and(qs string) collScope { + if s.extra != "" { + qs = s.extra + "&" + qs + } + s.extra = qs + return s +} + +func (s collScope) qs() string { + q := "" + if len(s.colls) > 0 { + q = url.Values{"coll": s.colls}.Encode() + } + switch { + case q == "": + return s.extra + case s.extra != "": + return q + "&" + s.extra + } + return q +} + +func (s collScope) link(u string) string { return withQuery(u, s.qs()) } + +func (s collScope) suffix() string { + if q := s.qs(); q != "" { + return "?" + q + } + return "" +} + +func withQuery(link, qs string) string { + switch { + case qs == "" || link == "": + return link + case strings.Contains(link, "?"): + return link + "&" + qs + } + return link + "?" + qs +} + +func cloneValues(v url.Values) url.Values { + out := make(url.Values, len(v)) + for k, vs := range v { + out[k] = append([]string(nil), vs...) + } + return out +} + +func (co *Corpus) handleBrowseRoot(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + collection := q.Get("collection") + if !validCollections[collection] { + collection = collectionNews + } + + if collection == collectionArchive { + co.renderArchiveBrowse(w, r, q) + return + } + + prefs := readPrefs(r) + + selTypes, typeFiltered := selectedTypes(q) + typeFiltered = typeFiltered && collection == collectionNews + selectedTypeSet := map[string]bool{} + if collection == collectionNews { + selectedTypeSet = selTypes + if !typeFiltered { + selectedTypeSet = allTypesSelected(co.archive.NewsTypes) + } + } + + docs := append([]*Document(nil), co.collectionDocs[collection]...) + if collection == collectionNews { + docs = filterByType(docs, selectedTypeSet) + } + + params := url.Values{} + if collection != collectionNews { + params.Set("collection", collection) + } + if typeFiltered { + params["type"] = q["type"] + } + + data := &browsePage{ + Base: pageWith(prefs), + listView: co.buildListView(q, prefs, docs, "/browse", params, "title", false), + Collection: collection, + CollectionCounts: co.archive.CollectionCounts, + SIDTodayCount: co.archive.SIDTodayCount, + SelectedTypes: selectedTypeSet, + AllTypes: co.archive.NewsTypes, + Params: q, + } + co.addFacetNav(&data.Base, prefs) + render(w, r, data) +} + +// --- SIDtoday archive: chronological view --- +type archiveMonth struct { + Name string + Docs []*Document +} + +type archiveYear struct { + Year string + Count int + Months []archiveMonth +} + +var monthNames = [...]string{"", "January", "February", "March", "April", + "May", "June", "July", "August", "September", "October", "November", + "December"} + +func (co *Corpus) buildArchiveTimeline() { + co.archiveTimeline = nil + docs := append([]*Document(nil), co.sidtodayDocs...) + + sort.SliceStable(docs, func(i, j int) bool { + a, b := docs[i].DocMeta.DateCreated, docs[j].DocMeta.DateCreated + if (a == "") != (b == "") { + return b == "" + } + if a != b { + return a < b + } + ta, tb := strings.ToLower(docs[i].Title), strings.ToLower(docs[j].Title) + if ta == tb { + return false + } + return ta < tb + }) + + for _, d := range docs { + y, m := "Undated", "" + date := d.DocMeta.DateCreated + if len(date) >= 4 { + y = date[:4] + } + if len(date) >= 7 { + if n, err := strconv.Atoi(date[5:7]); err == nil && n >= 1 && n <= 12 { + m = monthNames[n] + } + } + + if len(co.archiveTimeline) == 0 || co.archiveTimeline[len(co.archiveTimeline)-1].Year != y { + co.archiveTimeline = append(co.archiveTimeline, archiveYear{Year: y}) + } + yr := &co.archiveTimeline[len(co.archiveTimeline)-1] + yr.Count++ + if len(yr.Months) == 0 || yr.Months[len(yr.Months)-1].Name != m { + yr.Months = append(yr.Months, archiveMonth{Name: m}) + } + mo := &yr.Months[len(yr.Months)-1] + mo.Docs = append(mo.Docs, d) + } +} + +func (co *Corpus) renderArchiveBrowse(w http.ResponseWriter, r *http.Request, q url.Values) { + prefs := readPrefs(r) + view := q.Get("view") + + data := &browsePage{ + Base: pageWith(prefs), + Collection: collectionArchive, + CollectionCounts: co.archive.CollectionCounts, + SIDTodayCount: co.archive.SIDTodayCount, + SeriesList: co.seriesBySize[prefs.IncludeFOIA], + } + co.addFacetNav(&data.Base, prefs) + + if view == "card" || view == "table" { + docs := append([]*Document(nil), co.sidtodayDocs...) + params := url.Values{"collection": {collectionArchive}} + data.listView = co.buildListView(q, prefs, docs, "/browse", params, "date", true) + data.MarkReported, data.Cards.MarkReported = true, true + if data.RowsHTML != "" { + data.RowsHTML = co.buildDocRows(data.Docs, true) + } + render(w, r, data) + return + } + + data.Total = co.archive.SIDTodayCount + data.View = view + data.ArchiveYears = co.archiveTimeline + data.TimelineHTML = co.preHTML("archive-timeline") + render(w, r, data) +} + +func docPath(id string) string { return "/doc/" + url.PathEscape(id) } + +func filePath(id, name string) string { + return "/files/" + url.PathEscape(id) + "/" + url.PathEscape(name) +} + +func versionPath(id, slug string) string { + return docPath(id) + "/versions/" + url.PathEscape(slug) +} + +func versionFilePath(id, slug, name string) string { + return "/files/" + url.PathEscape(id) + "/versions/" + url.PathEscape(slug) + "/" + url.PathEscape(name) +} + +func imageBase(id, slug string) string { + if slug == "" { + return "/files/" + url.PathEscape(id) + "/images/" + } + return "/files/" + url.PathEscape(id) + "/versions/" + url.PathEscape(slug) + "/images/" +} + +func (co *Corpus) publishedTogether(doc *Document) []*Document { + var out []*Document + seen := map[string]bool{doc.ID: true} + for _, p := range doc.Publication { + if p.Link == "" { + continue + } + a := co.articleByLink[p.Link] + if a == nil { + continue + } + for _, id := range a.DocIDs { + if seen[id] { + continue + } + seen[id] = true + if d := co.docIndex[id]; d != nil { + out = append(out, d) + } + } + } + return out +} + +// ogImageAbs is the document's first page image, or its thumbnail when it +// has no page images. +func ogImageAbs(r *http.Request, doc *Document) string { + switch { + case len(doc.Images) > 0: + return requestBase(r) + imageBase(doc.ID, "") + url.PathEscape(doc.Images[0]) + case doc.HasThumb: + return requestBase(r) + filePath(doc.ID, "thumb.jpg") + } + return "" +} + +// queryBase is path plus q, ready for one more parameter to be appended. +func queryBase(path string, q url.Values) string { + if enc := q.Encode(); enc != "" { + return path + "?" + enc + "&" + } + return path + "?" +} + +func (co *Corpus) handleDocPages(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + doc := co.docIndex[id] + if doc == nil { + notFound(w, r) + return + } + prefs := readPrefs(r) + render(w, r, &pagesPage{ + Base: pageWith(prefs), + viewer: viewer{Images: doc.Images, Sizes: doc.ImageSizes, ImageBase: imageBase(doc.ID, "")}, + Doc: doc, + TextOnly: prefs.TextOnly, + }) +} + +func (co *Corpus) handleDocView(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + + doc := co.docIndex[id] + if doc == nil { + notFound(w, r) + return + } + + together := co.publishedTogether(doc) + shownTogether := together + var moreArticles []*Article + if len(together) > pubTogetherCap { + shownTogether = together[:pubTogetherCap] + moreArticles = co.docArticles(doc) + } + + prefs := readPrefs(r) + scope := pageScope(nil, prefs, false).including(doc.Collection) + var relatedDocs []*Document + for _, relID := range co.archive.RelatedDocs[id] { + if related := co.docIndex[relID]; related != nil && scope.shows(related) { + relatedDocs = append(relatedDocs, related) + } + } + + if prefs.TrackRecent && !crossSiteFetch(r) { + pushRecentDoc(w, r, id) + } + + var serialPrevDoc, serialNextDoc *Document + if doc.Type == "SIDTODAY" { + if id, ok := co.serialPrev[doc.ID]; ok { + serialPrevDoc = co.docIndex[id] + } + if id, ok := co.serialNext[doc.ID]; ok { + serialNextDoc = co.docIndex[id] + } + } + + embedBase := requestBase(r) + absURL := embedBase + docPath(doc.ID) + pdfURL := embedBase + filePath(doc.ID, doc.PDFFile) + + render(w, r, &docPage{ + Base: pageWith(prefs), + viewer: viewer{ + Images: doc.Images, + Sizes: doc.ImageSizes, + ImageBase: imageBase(doc.ID, ""), + Content: doc.HTMLContent, + }, + Doc: doc, + BrowseQS: scope.suffix(), + OGImageURL: ogImageAbs(r, doc), + SeriesGroup: co.seriesIndex[valueKey(doc.DocMeta.Series.Name)], + Assemblies: co.assembliesByDoc[doc.ID], + DocAbsURL: absURL, + Embeds: buildDocEmbeds(doc, absURL, pdfURL), + SerialPrev: serialPrevDoc, + SerialNext: serialNextDoc, + PublishedTogether: together, + PublishedShown: len(shownTogether), + PublishedMore: moreArticles, + RelatedDocs: relatedDocs, + PublishedCards: newCardView(prefs, shownTogether), + RelatedCards: newCardView(prefs, relatedDocs), + Pinned: isPinned(r, id), + }) +} + +func sortToken(s string) string { + for i := 0; i < len(s); i++ { + if s[i] < 'a' || s[i] > 'z' { + return "" + } + } + return s +} + +// firstLetter returns the uppercase A-Z bucket for a facet value, folding +// accents first ("Çağlar" -> "C"), with "#" for anything non-alphabetic. +func firstLetter(s string) string { + s = strings.ToUpper(strings.TrimSpace(foldAccents(s))) + if s != "" && s[0] >= 'A' && s[0] <= 'Z' { + return string(s[0]) + } + return "#" +} + +// TODO split this huge ass file to separate files, browse_cateogry etc? + +// category describes one browsable dimension: its URL slug, the two page +// headings, and the search field code its values map to +type category struct { + Slug string + Title string // list page e.g. "Browse by Programs" + Label string // value page e.g. "Program: XKEYSCORE" + Field string // search field code (fqf), "" for date categories + DateParam string // "date_doc" / "date_pub" for the date categories, else "" + Blurb string +} + +func (c *category) isDate() bool { return c.DateParam != "" } + +// facet = category's corpus counts (value -> documents) +func (c *category) Facet(co *Corpus) map[string]int { return co.archive.Facets[c.Slug] } + +var categories = []category{ + {"program", "Programs", "Program", "PR", "", + "Codenames and programs recorded for each document."}, + {"journalist", "Journalists", "Journalist", "RE", "", + "Journalists bylined on the stories that published these documents. Contributors credited without a byline are searchable but not listed here."}, + {"agency", "Agencies", "Agency", "CR", "", + "The agency that WROTE each document."}, + {"publisher", "Publishers", "Publisher", "PU", "", + "Who published each document. Mostly a news outlet, but sometimes a book. A partner outlet that reported the story without publishing the documents keeps its byline and its article, but is not a publisher here."}, + {"country", "Countries", "Country", "CO", "", + "Countries mentioned in each document. A country may be mentioned directly (e.g., France) or inferred from a related reference, such as a city (Paris -> France) or a domain name (.fr -> France)."}, + {"people", "People", "Person", "PE", "", + "People who appear in these documents, named in the text or recognised in an image. An appearance implies nothing about the person beyond that appearance."}, + {"classification", "Classification Levels", "Classification", "CL", "", + "Each document's classification level, as component values. If a document contains multiple classifications, the highest is used here."}, + {"dissemination", "Dissemination Controls", "Dissemination control", "DI", "", + "Handling controls assessed for each document, as component values. If a document contains multiple disseminations, the most restrictive is used here."}, + {"banner", "Classification Banners", "Banner", "BA", "", + "The marking line each document prints."}, + {"genre", "Genres", "Genre", "GE", "", + "What kind of document it is: slides, report, wiki, manual, guide, newsletter."}, + {"series", "Document Series", "Series", "SE", "", + "Recurring columns and multi-part runs in SIDtoday documents. A series lists its parts in the order the documents print. A missing part shows as a gap."}, + {"published", "Publication Years", "Publication year", "", "date_pub", + "When a publisher first published each document. If a document is published multiple times, only the oldest one is used here."}, + {"docdate", "Document Dates", "Document year", "", "date_doc", + "When each document was written, from its own dateline. Undated documents don't appear here."}, + {"tag", "Tags", "Tag", "TA", "", + "Free-text subjects the curators added. Unlike the other axes these are not necessarily from the documents' own vocabulary, so they are the loosest thing to browse by (and the most forgiving)."}, +} + +// pubTogetherCap = on doc view, how many published together docs are shown, max +const pubTogetherCap = 10 + +func (co *Corpus) docArticles(doc *Document) []*Article { + var out []*Article + for _, p := range doc.Publication { + a := co.articleByLink[p.Link] + if a != nil && len(a.DocIDs) > 1 && !slices.Contains(out, a) { + out = append(out, a) + } + } + return out +} + +func categoryValueURL(slug, value string) string { + return "/browse/" + slug + "/" + url.PathEscape(value) +} + +func inCollections(d *Document, sel map[string]bool) bool { + return sel == nil || sel[d.Collection] +} + +func filterByCollection(docs []*Document, sel map[string]bool) []*Document { + if sel == nil { + return docs + } + out := make([]*Document, 0, len(docs)) + for _, d := range docs { + if inCollections(d, sel) { + out = append(out, d) + } + } + return out +} + +func categoryBySlug(slug string) *category { + for i := range categories { + if categories[i].Slug == slug { + return &categories[i] + } + } + return nil +} + +const azMinEntries = 50 + +func (co *Corpus) prerenderCategoryRows() { + t := tmplSet["category.html"] + if t == nil { + return + } + exec := func(block string, data interface{}) (template.HTML, bool) { + var buf bytes.Buffer + if err := t.ExecuteTemplate(&buf, block, data); err != nil { + warnf("prerender category %s: %v - rendering rows per request", block, err) + return "", false + } + return template.HTML(buf.String()), true + } + co.categoryRowHTML = make(map[string][]template.HTML, len(co.categoryTables)) + co.categoryRowSize = make(map[string]int, len(co.categoryTables)) + co.azRowHTML = map[string]template.HTML{} + for ci := range categories { + for _, key := range []string{categories[ci].Slug, noFOIAKey(categories[ci].Slug)} { + entries := co.categoryTables[key].Entries + rows := make([]template.HTML, len(entries)) + for i, e := range entries { + h, ok := exec("category-row", e) + if !ok { + return + } + rows[i] = h + co.categoryRowSize[key] += len(h) + if _, seen := co.azRowHTML[e.FirstLetter]; !seen { + if h, ok := exec("category-az-row", e.FirstLetter); ok { + co.azRowHTML[e.FirstLetter] = h + } + } + } + co.categoryRowSize[key] += 26 * 64 + co.categoryRowHTML[key] = rows + } + } +} + +func (co *Corpus) prerenderDocRows() { + t := tmplSet["browse.html"] + if t == nil { + return + } + rows := make(map[string]template.HTML, len(co.archive.Docs)) + marked := map[string]template.HTML{} + var buf bytes.Buffer + render := func(row docTableRow) (template.HTML, bool) { + buf.Reset() + if err := t.ExecuteTemplate(&buf, "doc-table-row", row); err != nil { + warnf("prerender doc-table-row: %v - rendering table rows per request", err) + return "", false + } + return template.HTML(buf.String()), true + } + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + row, ok := render(docTableRow{Document: d}) + if !ok { + return + } + rows[d.ID] = row + if d.ReportedIssue() { + if marked[d.ID], ok = render(docTableRow{d, true}); !ok { + return + } + } + } + co.docRowHTML, co.docRowMarkedHTML = rows, marked +} + +// buildDocRows joins the documents' pre-rendered table rows; markReported +// takes the rows that badge an individually reported SIDtoday issue. +func (co *Corpus) buildDocRows(docs []*Document, markReported bool) template.HTML { + rows := make([]template.HTML, len(docs)) + n := 0 + for i, d := range docs { + row, ok := co.docRowMarkedHTML[d.ID] + if !markReported || !ok { + row, ok = co.docRowHTML[d.ID] + } + if !ok { + return "" + } + rows[i] = row + n += len(row) + } + var b strings.Builder + b.Grow(n) + for _, row := range rows { + b.WriteString(string(row)) + } + return template.HTML(b.String()) +} + +func azID(letter string) string { + if letter == "#" { + return "other" + } + return letter +} + +func (co *Corpus) buildCategoryRows(key string, t *categoryTable, order []int32, showAZ bool) template.HTML { + rows := co.categoryRowHTML[key] + if len(rows) != len(t.Entries) { + return "" + } + var b strings.Builder + b.Grow(co.categoryRowSize[key]) + prevL := "" + for _, n := range order { + if l := t.Entries[n].FirstLetter; showAZ && l != prevL { + b.WriteString(string(co.azRowHTML[l])) + prevL = l + } + b.WriteString(string(rows[n])) + } + return template.HTML(b.String()) +} + +func (co *Corpus) handleCategoryList(w http.ResponseWriter, r *http.Request) { + cat := r.PathValue("category") + q := r.URL.Query() + sortBy := sortToken(q.Get("sort")) + order := sortToken(q.Get("order")) + + c := categoryBySlug(cat) + if c == nil { + notFound(w, r) + return + } + + prefs := readPrefs(r) + collCols := cat != "series" + listSort := "name" + switch { + case sortBy == "count": + listSort = sortBy + case !collCols: + case sortBy == "reported", sortBy == "sidtoday": + listSort = sortBy + case sortBy == "foia": + if prefs.IncludeFOIA { + listSort = sortBy + } + } + desc := order == "desc" + if listSort != "name" { + desc = order != "asc" + } + key := readerListKey(cat, prefs) + table := co.categoryTables[key] + rowOrder := table.order(listSort, desc) + + showAZ := listSort == "name" && len(rowOrder) >= azMinEntries + var letters []string + if showAZ { + letters = table.letters[desc] + } + + collSort := listSort == "reported" || listSort == "sidtoday" || listSort == "foia" + + nameCol := "NAME" + if c.isDate() { + nameCol = "YEAR" + } + + cols := []sortCol{ + {Key: "name", Label: nameCol}, + {Key: "count", Label: "TOTAL", Class: "num-col", DescFirst: true}, + } + if collCols { + cols = append(cols, + sortCol{Key: "reported", Label: "REPORTED", Class: "num-col coll-col", DescFirst: true}, + sortCol{Key: "sidtoday", Label: "SIDTODAY", Class: "num-col coll-col", DescFirst: true}) + } + if collCols && prefs.IncludeFOIA { + cols = append(cols, sortCol{Key: "foia", Label: "FOIA", Class: "num-col coll-col", DescFirst: true}) + } + + data := &categoryPage{ + Base: pageWith(prefs), + Title: c.Title, + Blurb: c.Blurb, + Heads: sortHeads("?", cols, listSort, desc), + RowsHTML: co.buildCategoryRows(key, table, rowOrder, showAZ), + CollCols: collCols, + ShowCollCols: collCols && (prefs.ShowCollCols || collSort), + ShowAZ: showAZ, + Letters: letters, + } + data.ActiveCategory = cat + if data.RowsHTML == "" { + data.Entries = table.ordered(rowOrder) + } + co.addFacetNav(&data.Base, prefs) + + switch cat { + case "classification", "dissemination": + data.SeeAlso = []seeAlsoLink{{"/browse/banner", "full classification banners (verbatim marking lines)"}} + case "banner": + data.SeeAlso = []seeAlsoLink{ + {"/browse/classification", "classification components"}, + {"/browse/dissemination", "dissemination controls"}, + } + } + + render(w, r, data) +} + +// categoryEntry = one row of a /browse/{category} list page +type categoryEntry struct { + Name string + URL template.URL // /browse/{category}/{name}, path-escaped + Note string // the agency expansion on the agency list, else "" + Count int // total, all collections + News int // type REPORTED (categoryTypeCounts), any collection + Archive int // type SIDTODAY, incl. individually reported issues + FOIA int // type FOIA + NonFOIA int // in the Reported and SIDtoday collections: the total when FOIA releases are left out + NoFOIA bool // a row of the FOIA-less table: no FOIA column + FirstLetter string + fold string // accent-folded lowercase name: the name sort key +} + +// categoryTable = one category's list page +type categoryTable struct { + Entries []categoryEntry // by name, ascending + orders map[string][]int32 // categoryListKey -> indices into Entries, in that order + letters map[bool][]string // the two name orderings' A-Z letters as they come, by desc +} + +func (t *categoryTable) order(sortBy string, desc bool) []int32 { + return t.orders[categoryListKey(sortBy, desc)] +} + +func (t *categoryTable) ordered(order []int32) []categoryEntry { + out := make([]categoryEntry, len(order)) + for i, n := range order { + out[i] = t.Entries[n] + } + return out +} + +// noFOIAKey is where a category's FOIA-less table is kept in categoryLists +// and categoryRowHTML. +func noFOIAKey(slug string) string { return slug + "|nofoia" } + +// readerListKey is the table a reader's category list comes from: the +// FOIA-less one unless they include FOIA releases. +func readerListKey(slug string, p Prefs) string { + if p.IncludeFOIA { + return slug + } + return noFOIAKey(slug) +} + +// newCategoryTable sorts entries by name and works out the orderings the +// sorts ask for, ascending and descending. +func newCategoryTable(entries []categoryEntry, sorts ...string) *categoryTable { + sortCategoryEntries(entries, "name", false) + t := &categoryTable{Entries: entries, orders: map[string][]int32{}, letters: map[bool][]string{}} + for _, sortBy := range sorts { + for _, desc := range []bool{false, true} { + order := make([]int32, len(entries)) + for i := range order { + order[i] = int32(i) + } + sort.Slice(order, func(i, j int) bool { + return categoryLess(&entries[order[i]], &entries[order[j]], sortBy, desc) + }) + t.orders[categoryListKey(sortBy, desc)] = order + if sortBy != "name" { + continue + } + seen := map[string]bool{} + for _, n := range order { + if l := entries[n].FirstLetter; !seen[l] { + seen[l] = true + t.letters[desc] = append(t.letters[desc], l) + } + } + } + } + return t +} + +func categoryListKey(sortBy string, desc bool) string { + if desc { + return sortBy + "/desc" + } + return sortBy + "/asc" +} + +func (co *Corpus) buildCategoryLists() { + co.categoryTables = make(map[string]*categoryTable, 2*len(categories)) + co.categoryValueCounts = map[bool]map[string]int{false: {}, true: {}} + for ci := range categories { + c := &categories[ci] + byType := co.categoryTypeCounts[c.Slug] + byColl := co.archive.FacetsByColl[c.Slug] + facet := c.Facet(co) + base := make([]categoryEntry, 0, len(facet)) + for name, count := range facet { + e := categoryEntry{ + Name: name, + URL: template.URL(categoryValueURL(c.Slug, name)), + Count: count, + FirstLetter: firstLetter(name), + fold: normFoldString(name), + } + if c.Slug == "agency" { + e.Note = agencyName(name) + } + if tc := byType[name]; tc != nil { + e.News, e.Archive, e.FOIA = tc.News, tc.Archive, tc.FOIA + } + if cc := byColl[name]; cc != nil { + e.NonFOIA = cc.News + cc.Archive + } + base = append(base, e) + } + noFOIA := make([]categoryEntry, 0, len(base)) + for i := range base { + e := base[i] + if e.NonFOIA > 0 { + e.Count = e.NonFOIA + e.NoFOIA = true + noFOIA = append(noFOIA, e) + } + } + co.categoryTables[c.Slug] = newCategoryTable(base, "name", "count", "reported", "sidtoday", "foia") + co.categoryTables[noFOIAKey(c.Slug)] = newCategoryTable(noFOIA, "name", "count", "reported", "sidtoday") + co.categoryValueCounts[true][c.Slug] = len(base) + co.categoryValueCounts[false][c.Slug] = len(noFOIA) + } + co.seriesBySize = map[bool][]categoryEntry{} + for _, foia := range []bool{false, true} { + t := co.categoryTables[readerListKey("series", Prefs{IncludeFOIA: foia})] + co.seriesBySize[foia] = t.ordered(t.order("count", true)) + } +} + +func sortCategoryEntries(entries []categoryEntry, sortBy string, desc bool) { + sort.Slice(entries, func(i, j int) bool { + return categoryLess(&entries[i], &entries[j], sortBy, desc) + }) +} + +// categoryLess orders two rows of a category table: by the sorted column in +// the direction asked, then the larger total first, then by name. +func categoryLess(a, b *categoryEntry, sortBy string, desc bool) bool { + col := func(e *categoryEntry) int { + switch sortBy { + case "count": + return e.Count + case "reported": + return e.News + case "sidtoday": + return e.Archive + case "foia": + return e.FOIA + } + return 0 + } + byName := sortBy == "name" + if !byName { + if x, y := col(a), col(b); x != y { + if desc { + return x > y + } + return x < y + } + if a.Count != b.Count { + return a.Count > b.Count + } + } + if a.fold != b.fold { + if byName && desc { + return a.fold > b.fold + } + return a.fold < b.fold + } + return a.Name < b.Name +} + +type seeAlsoLink struct { + URL string + Text string +} + +type timelineBar struct { + Year string + Count int + Pct int + URL string +} + +func listSortKey(vals []string) string { + best := "" + for _, v := range vals { + lv := strings.ToLower(strings.TrimSpace(v)) + if lv != "" && (best == "" || lv < best) { + best = lv + } + } + return best +} + +var classAlias = map[string]string{ + "ts": "top secret", "s": "secret", "c": "confidential", "u": "unclassified", + "strap1": "strap 1", "strap2": "strap 2", "talent keyhole": "tk", +} + +func foldClass(v string) string { + lv := strings.ToLower(strings.TrimSpace(v)) + if a, ok := classAlias[lv]; ok { + return a + } + return lv +} + +// classSortKey ranks a marking set per the curators' order: level first +// (TOP SECRET > SECRET > CONFIDENTIAL > UNCLASSIFIED), a level WITH SI +// above the bare level, and the restrictive compartments raising rank +// cumulatively - STRAP 2 (weight 4) > STRAP 1 (2), TK and MR (1 each) - +// so TOP SECRET SI TK outranks TOP SECRET SI. +func classSortKey(vals []string) string { + if len(vals) == 0 { + return "" + } + level, si, raise := 0, 0, 0 + for _, v := range vals { + switch foldClass(v) { + case "top secret": + if level < 4 { + level = 4 + } + case "secret": + if level < 3 { + level = 3 + } + case "confidential": + if level < 2 { + level = 2 + } + case "unclassified": + if level < 1 { + level = 1 + } + case "si", "comint": + si = 1 + case "strap 2": + raise += 4 + case "strap 1": + raise += 2 + case "tk", "mr": + raise++ + } + } + if level == 0 && si == 0 && raise == 0 { + return "" + } + r := strconv.Itoa(raise) + if len(r) < 2 { + r = "0" + r + } + return strconv.Itoa(level) + strconv.Itoa(si) + r +} + +// TODO this file really needs the split. + +// searchClause is one row of the search form: the field code, the value as +// typed into it and how the row joins the ones before it. +type searchClause struct{ Conj, Field, Value string } + +// searchURL builds an executed /search link +func searchURL(clauses []searchClause, ranges ...[2]string) string { + q := url.Values{} + q.Set("do_search", "1") + q.Set("action", "search") + for _, c := range clauses { + conj := c.Conj + if conj == "" { + conj = "and" + } + q.Add("fqc", conj) + q.Add("fqv", c.Value) + q.Add("fqf", c.Field) + } + for _, r := range ranges { + q.Set(r[0]+"_start", r[1]) + q.Set(r[0]+"_end", r[1]) + } + return "/search?" + q.Encode() +} + +func dateRangeSearchURL(fqf, fqv string, ranges ...[2]string) string { + return searchURL([]searchClause{{Field: fqf, Value: fqv}}, ranges...) +} + +func appendClause(base, field, value string) string { + return base + "fqf=" + url.QueryEscape(field) + + "&fqv=" + url.QueryEscape(value) + "&fqc=and" +} + +// entityTimelineBy is the per-year histogram behind both value-page +// timelines. docs are the page's documents, already narrowed to its +// collection scope, so every bar link carries that scope too. +// yearsOf returns a document's distinct years on the bar axis. +func entityTimelineBy(docs []*Document, c *category, val string, scope collScope, dateParam string, yearsOf func(*Document) []string) []timelineBar { + counts := map[string]int{} + for _, d := range docs { + for _, y := range yearsOf(d) { + counts[y]++ + } + } + if len(counts) < 2 { + return nil + } + pageParam := "" + if c.isDate() && validDatePrefix(val) { + pageParam = c.DateParam + } + return yearBars(counts, func(y string) string { + switch { + case c.Field != "": + return scope.link(dateRangeSearchURL(c.Field, exactClause(val), [2]string{dateParam, y})) + case pageParam == "date_doc" && dateParam == "date_pub": + return scope.link(dateRangeSearchURL("TE", "", [2]string{pageParam, val}, [2]string{dateParam, y})) + } + return "" + }) +} + +func entityTimeline(docs []*Document, c *category, val string, scope collScope) []timelineBar { + return entityTimelineBy(docs, c, val, scope, "date_doc", func(d *Document) []string { + if y := docYear(d); y != "" { + return []string{y} + } + return nil + }) +} + +func entityPubTimeline(docs []*Document, c *category, val string, scope collScope) []timelineBar { + switch c.Slug { + case "journalist", "publisher": + return entityTimelineBy(docs, c, val, scope, "date_pub", func(d *Document) []string { + return pubYearsMatching(d, c.Field, val) + }) + } + return entityTimelineBy(docs, c, val, scope, "date_pub", pubYearsOf) +} + +type monthNav struct { + Months []monthNavEntry + Year string + YearURL string + YearActive bool + Monthless int +} + +func (co *Corpus) buildMonthNav(c *category, val string, allMatching []*Document, scope collScope) *monthNav { + cat := c.Slug + if !c.isDate() || !validDatePrefix(val) { + return nil + } + year := val[:4] + yearDocs := allMatching + if len(val) == 7 { + yearDocs = scope.docs(co.categoryMatches(cat, year)) + } + counts := map[string]int{} + for _, d := range yearDocs { + ds := d.DocMeta.DateCreated + if cat == "published" { + ds = oldestPubDate(d) + } + if len(ds) >= 7 { + counts[ds[5:7]]++ + } + } + var months []monthNavEntry + shown := 0 + for m := 1; m <= 12; m++ { + mm := strconv.Itoa(m) + if m < 10 { + mm = "0" + mm + } + c := counts[mm] + if c == 0 { + continue + } + shown += c + months = append(months, monthNavEntry{ + Label: monthNames[m][:3], + URL: scope.link(categoryValueURL(cat, year+"-"+mm)), + Count: c, + Active: len(val) == 7 && val[5:7] == mm, + }) + } + if len(months) == 0 { + return nil + } + return &monthNav{ + Months: months, + Year: year, + YearURL: scope.link(categoryValueURL(cat, year)), + YearActive: len(val) == 4, + Monthless: len(yearDocs) - shown, + } +} + +// bdGroup = one of a value page's breakdown charts (its banners, its +// classification levels), each value a bar. +type bdGroup struct { + Label string + Values []bdVal +} + +type bdVal struct { + Value string + Count int + URL string + Pct int +} + +func (co *Corpus) handleCategoryFilter(w http.ResponseWriter, r *http.Request) { + cat := r.PathValue("category") + val := r.PathValue("value") + c := categoryBySlug(cat) + if c == nil { + notFound(w, r) + return + } + if name, ok := co.categoryNames[cat][valueKey(val)]; ok && name != val { + u := categoryValueURL(cat, name) + if r.URL.RawQuery != "" { + u += "?" + r.URL.RawQuery + } + http.Redirect(w, r, u, http.StatusMovedPermanently) + return + } + prefs := readPrefs(r) + q := r.URL.Query() + + selectedTypeSet, typeFiltered := selectedTypes(q) + + scope := pageScope(q, prefs, false) + + everything := co.categoryMatches(cat, val) + allMatching := scope.docs(everything) + pubYear := "" + if y := q.Get("year"); cat == "publisher" && len(y) == 4 && allDigits(y) { + pubYear = y + allMatching = publishedBy(allMatching, val, y) + } + viewTypeCounts := typeCounts(allMatching) + if !typeFiltered { + selectedTypeSet = allTypesSelected(co.archive.Types) + } + results := append([]*Document(nil), allMatching...) + results = filterByType(results, selectedTypeSet) + + basePath := categoryValueURL(cat, val) + params := url.Values{} + if typeFiltered { + params["type"] = q["type"] + } + if scope.colls != nil { + params["coll"] = scope.colls + } + if pubYear != "" { + params.Set("year", pubYear) + } + defSort, boost := "title", val + if cat == "series" { + defSort = "series" + } + if cat == "series" || c.isDate() { + boost = "" + } + data := &listPage{ + Base: pageWith(prefs), + listView: co.buildListViewBoosted(q, prefs, results, basePath, params, defSort, false, boost), + } + + searchLink := "" + if c.Field != "" { + searchLink = searchURL([]searchClause{ + {Field: "TE"}, + {Field: c.Field, Value: exactClause(val)}, + }) + } else if c.isDate() && validDatePrefix(val) { + searchLink = dateRangeSearchURL("TE", "", [2]string{c.DateParam, val}) + } + + docScope := scope + if pubYear != "" { + docScope = scope.and(url.Values{"date_pub_start": {pubYear}, "date_pub_end": {pubYear}}.Encode()) + } + searchLink = docScope.link(searchLink) + + if mn := co.buildMonthNav(c, val, allMatching, scope); mn != nil { + data.MonthNav = mn.Months + data.MonthNavYear = mn.Year + data.MonthNavYearURL = mn.YearURL + data.MonthNavYearActive = mn.YearActive + if mn.Monthless > 0 { + data.MonthlessCount = mn.Monthless + } + } + + var gloss *GlossaryEntry + if cat == "program" { + gloss = co.glossaryLookup(val) + } + + data.GlossaryEntry = gloss + if pubYear != "" { + data.YearScope = pubYear + data.YearScopeAllURL = scope.link(basePath) + } + refs := categoryRefinements(allMatching, c, val, docScope, pubYear) + data.Refinements = refs + + var breakdowns []bdGroup + for _, g := range refs { + if g.Label != "Banner" && g.Label != "Classification" { + continue + } + top := 0 + for _, v := range g.Values { + if v.Count > top { + top = v.Count + } + } + bg := bdGroup{Label: g.Label} + for _, v := range g.Values { + bg.Values = append(bg.Values, bdVal{v.Name, v.Count, v.URL, barPct(v.Count, top, 2)}) + } + breakdowns = append(breakdowns, bg) + } + data.Breakdowns = breakdowns + data.Timeline = entityTimeline(allMatching, c, val, docScope) + if pubYear == "" { + // a year-scoped page has one publication year: no timeline + data.PubTimeline = entityPubTimeline(allMatching, c, val, scope) + } + co.addFacetNav(&data.Base, prefs) + data.Title = val + data.ActiveCategory = cat + data.Cards.ActiveCategory, data.Cards.ActiveValue = cat, val + data.SearchLink = searchLink + data.SelectedTypes = selectedTypeSet + data.AllTypes = viewTypeCounts + data.Params = q + data.CatLabel = c.Label + data.Blurb = c.Blurb + render(w, r, data) +} + +// categoryRefinements builds the "combine with another filter" facet groups +// for a category value page. For the docs matching this page's value, tally +// the other browsable dimensions and link each value to a TWO-clause search +// (this page's value AND that value). On a publisher page scoped to pubYear +// the links carry that publication year, and the tallies read the +// journalists the way that search will (refineValues). +func categoryRefinements(docs []*Document, c *category, val string, scope collScope, pubYear string) []searchFacet { + if c.Field == "" || len(docs) < 2 { + return nil + } + base := scope.link(searchURL([]searchClause{{Field: c.Field, Value: exactClause(val)}})) + "&" + + groups := []facetGroup{ + {"Banner", "BA"}, {"Classification", "CL"}, {"Dissemination", "DI"}, {"Genre", "GE"}, + {"Programs", "PR"}, {"Agencies", "CR"}, {"Countries", "CO"}, {"Journalists", "RE"}, + } + kept := groups[:0] + for _, g := range groups { + if g.Code != c.Field { + kept = append(kept, g) + } + } + return facetGroups(docs, kept, base, refineValues(pubYear, pubYear)) +} + +func (co *Corpus) buildCategoryDocs() { + co.categoryDocs = make(map[string]map[string][]int32, len(categories)) + co.categoryNames = make(map[string]map[string]string, len(categories)) + co.archive.Facets = make(map[string]map[string]int, len(categories)) + co.archive.FacetsByColl = make(map[string]map[string]*collCounts, len(categories)) + co.categoryTypeCounts = make(map[string]map[string]*collCounts, len(categories)) + for ci := range categories { + c := &categories[ci] + byValue := make(map[string][]int32) + name := make(map[string]string) // index key -> display value + add := func(key, value string, i int) { + if _, ok := name[key]; !ok { + name[key] = value + } + byValue[key] = append(byValue[key], int32(i)) + } + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + if c.isDate() { + date := d.DocMeta.DateCreated + if c.DateParam == "date_pub" { + date = oldestPubDate(d) + } + if len(date) >= 4 { + add(date[:4], date[:4], i) + } + if len(date) >= 7 { + add(date[:7], date[:7], i) + } + continue + } + for _, v := range facetValues(d, c.Field) { + add(valueKey(v), v, i) + } + } + counts := make(map[string]int) + byColl := make(map[string]*collCounts) + byType := make(map[string]*collCounts) + for key, docs := range byValue { + if c.isDate() && len(key) != 4 { + continue + } + cc, tc := &collCounts{}, &collCounts{} + for _, n := range docs { + cc.add(co.archive.Docs[n].Collection) + tc.addType(co.archive.Docs[n].Type) + } + counts[name[key]] = len(docs) + byColl[name[key]] = cc + byType[name[key]] = tc + } + co.categoryDocs[c.Slug] = byValue + co.categoryNames[c.Slug] = name + co.archive.Facets[c.Slug] = counts + co.archive.FacetsByColl[c.Slug] = byColl + co.categoryTypeCounts[c.Slug] = byType + } +} + +// categoryMatches returns the documents on a /browse/{category}/{value} +// page, in corpus order, as a fresh slice of pointers the caller owns. +func (co *Corpus) categoryMatches(cat, val string) []*Document { + c := categoryBySlug(cat) + if c == nil { + return nil + } + if c.isDate() && !validDatePrefix(val) { + return nil + } + idx := co.categoryDocs[cat][valueKey(val)] + out := make([]*Document, len(idx)) + for i, n := range idx { + out[i] = &co.archive.Docs[n] + } + return out +} + +func validDatePrefix(val string) bool { + switch len(val) { + case 4: + return allDigits(val) + case 7: + return allDigits(val[:4]) && val[4] == '-' && allDigits(val[5:]) + } + return false +} + +func validDate(val string) bool { + if len(val) == 10 { + return validDatePrefix(val[:7]) && val[7] == '-' && allDigits(val[8:]) + } + return validDatePrefix(val) +} + +func allDigits(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + if s[i] < '0' || s[i] > '9' { + return false + } + } + return true +} + +func publishedBy(docs []*Document, publisher, year string) []*Document { + var out []*Document + for _, d := range docs { + for _, p := range d.Publication { + if !p.ReportingOnly && sameValue(p.Publisher, publisher) && + strings.HasPrefix(p.DatePublished, year) { + out = append(out, d) + break + } + } + } + return out +} + +func publisherYearURL(publisher, year string) string { + return categoryValueURL("publisher", publisher) + "?year=" + url.QueryEscape(year) +} + +func oldestPubDate(d *Document) string { + oldest := "" + for _, p := range d.Publication { + if len(p.DatePublished) >= 4 && (oldest == "" || p.DatePublished < oldest) { + oldest = p.DatePublished + } + } + return oldest +} + +func (d *Document) FirstPubYear() string { + if date := oldestPubDate(d); len(date) >= 4 { + return date[:4] + } + return "" +} + +func docYear(d *Document) string { + if len(d.DocMeta.DateCreated) >= 4 { + return d.DocMeta.DateCreated[:4] + } + return "" +} + +// pubYearsOf lists the years the document was published in, any entry's, +// in the entries' order. +func pubYearsOf(d *Document) []string { + if d.fields != nil { + return d.fields.pubYears + } + var out []string + for _, p := range d.Publication { + if len(p.DatePublished) < 4 { + continue + } + if y := p.DatePublished[:4]; !slices.Contains(out, y) { + out = append(out, y) + } + } + return out +} + +func pubYearsMatching(d *Document, field, val string) []string { + var out []string + for i := range d.Publication { + date := d.Publication[i].DatePublished + if len(date) < 4 || slices.Contains(out, date[:4]) { + continue + } + for _, v := range entryValues(d, i, field) { + if sameValue(v, val) { + out = append(out, date[:4]) + break + } + } + } + return out +} + +type monthNavEntry struct { + Label string + URL string + Count int + Active bool +} + +// docSort is one way to order a list of documents. The sort bar, the +// doc-table's column headers, the list pages, search and /api/docs all read +// this table, so a sort means the same thing everywhere. +type docSort struct { + Key string + Label string + Value func(d *Document) string + EmptiesLast bool + DescFirst bool + InBar bool // offered in the sort bar +} + +// titleSortKey sorts a title by its first letter or digit +func titleSortKey(d *Document) string { + return strings.TrimLeftFunc(strings.ToLower(d.Title), func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) +} + +var docSorts = []docSort{ + {"title", "Title", titleSortKey, false, false, true}, + {"agency", "Agency", func(d *Document) string { return strings.ToLower(d.DocMeta.Creator) }, false, false, true}, + {"date", "Doc Date", func(d *Document) string { return d.DocMeta.DateCreated }, true, false, true}, + {"pubdate", "Pub Date", oldestPubDate, true, false, true}, + {"classification", "Classification", func(d *Document) string { return classSortKey(d.DocMeta.Classification) }, true, true, false}, + {"dissemination", "Dissemination", func(d *Document) string { return listSortKey(d.DocMeta.Dissemination) }, true, false, false}, + {"program", "Programs", func(d *Document) string { return listSortKey(d.DocMeta.Programs) }, true, false, false}, + {"country", "Countries", func(d *Document) string { return listSortKey(d.DocMeta.Countries) }, true, false, false}, + {"people", "People", func(d *Document) string { return listSortKey(d.DocMeta.People) }, true, false, false}, + {"journalist", "Journalists", func(d *Document) string { return listSortKey(d.JournalistsDeduped()) }, true, false, false}, + {"type", "Type", func(d *Document) string { return strings.ToLower(d.Type) }, false, false, false}, + {"series", "Series order", func(d *Document) string { + if p := d.DocMeta.Series.Part; p > 0 { + return fmt.Sprintf("%06d", p) + } + return "" + }, true, false, false}, +} + +var docSortByKey = func() map[string]*docSort { + m := make(map[string]*docSort, len(docSorts)) + for i := range docSorts { + m[docSorts[i].Key] = &docSorts[i] + } + return m +}() + +var docTableCols = func() []sortCol { + col := func(key string) sortCol { + s := docSortByKey[key] + return sortCol{Key: s.Key, Label: s.Label, DescFirst: s.DescFirst} + } + return []sortCol{ + col("title"), col("agency"), col("date"), col("pubdate"), + col("classification"), col("dissemination"), col("program"), + col("country"), col("people"), col("journalist"), + {Label: "Published"}, + col("type"), + } +}() + +type sortCol struct { + Key, Label, Class string + DescFirst, Fixed bool +} + +type sortHead struct { + sortCol + URL string + Active, Desc bool +} + +func sortHeads(base string, cols []sortCol, cur string, desc bool) []sortHead { + heads := make([]sortHead, len(cols)) + for i, c := range cols { + h := sortHead{sortCol: c} + if c.Key != "" { + next := c.DescFirst + if h.Active = c.Key == cur; h.Active { + h.Desc = desc + if !c.Fixed { + next = !desc + } + } + h.URL = base + "sort=" + c.Key + if !c.Fixed { + h.URL += "&order=" + orderParam(next) + } + } + heads[i] = h + } + return heads +} + +func orderParam(desc bool) string { + if desc { + return "desc" + } + return "asc" +} + +// sortCmp orders two sort keys: in the direction asked, an empty key after +// the others either way where the sort puts its empties last. +func sortCmp(a, b string, desc, emptiesLast bool) int { + if emptiesLast && (a == "") != (b == "") { + if b == "" { + return -1 + } + return 1 + } + if desc { + return cmp.Compare(b, a) + } + return cmp.Compare(a, b) +} + +// boostTitleMatches moves the documents whose title has word as a whole word, +// in any case, to the front, keeping each group's order e.g. on +// /browse/tag/Tor the documents about Tor come before the ones that only carry +// the tag. +func boostTitleMatches(docs []*Document, word string) { + re, err := regexp.Compile(`(?i)\b` + regexp.QuoteMeta(word) + `\b`) + if err != nil { + return + } + var hit, rest []*Document + for _, d := range docs { + if re.MatchString(d.Title) { + hit = append(hit, d) + } else { + rest = append(rest, d) + } + } + copy(docs, append(hit, rest...)) +} + +func sortDocs(docs []*Document, sortBy, order string) { + desc := order == "desc" + if docSortByKey[sortBy] == nil { + sortBy = "title" + } + + copy(docs, sortByField(docs, func(d *Document) *Document { return d }, sortBy, desc)) +} + +func sortScored(scored []ScoredDoc, sortBy, order string) []ScoredDoc { + if docSortByKey[sortBy] == nil { + return scored + } + return sortByField(scored, func(s ScoredDoc) *Document { return s.Doc }, sortBy, order == "desc") +} + +func sortByField[T any](items []T, doc func(T) *Document, sortBy string, desc bool) []T { + ds := docSortByKey[sortBy] + keys := make([]string, len(items)) + for i := range items { + keys[i] = ds.Value(doc(items[i])) + } + idx := make([]int, len(items)) + for i := range idx { + idx[i] = i + } + slices.SortStableFunc(idx, func(a, b int) int { + return sortCmp(keys[a], keys[b], desc, ds.EmptiesLast) + }) + out := make([]T, len(items)) + for i, o := range idx { + out[i] = items[o] + } + return out +} + +func (co *Corpus) docVersion(r *http.Request) (*Document, *DocVersion) { + doc := co.docIndex[r.PathValue("id")] + if doc == nil { + return nil, nil + } + return doc, doc.VersionFor(r.PathValue("slug")) +} + +func (co *Corpus) handleVersionView(w http.ResponseWriter, r *http.Request) { + doc, v := co.docVersion(r) + if v == nil { + notFound(w, r) + return + } + pubs := doc.Publication[:0:0] + for _, i := range v.PubIndex { + pubs = append(pubs, doc.Publication[i]) + } + prefs := readPrefs(r) + render(w, r, &versionPage{ + Base: pageWith(prefs), + viewer: viewer{ + Images: v.Images, + Sizes: v.ImageSizes, + ImageBase: imageBase(doc.ID, v.Slug), + Content: v.HTMLContent, + Version: v, + }, + Doc: doc, + Publications: pubs, + PDFURL: versionFilePath(doc.ID, v.Slug, v.PDFFile), + }) +} + +func (co *Corpus) handleVersionPages(w http.ResponseWriter, r *http.Request) { + doc, v := co.docVersion(r) + if v == nil { + notFound(w, r) + return + } + prefs := readPrefs(r) + render(w, r, &pagesPage{ + Base: pageWith(prefs), + viewer: viewer{Images: v.Images, Sizes: v.ImageSizes, ImageBase: imageBase(doc.ID, v.Slug), Version: v}, + Doc: doc, + TextOnly: prefs.TextOnly, + }) +} diff --git a/changelog.go b/changelog.go new file mode 100644 index 0000000..9dd4f45 --- /dev/null +++ b/changelog.go @@ -0,0 +1,102 @@ +// changelog.go - the archive changelog (/changelog). +// +// The changelog is a HAND-MAINTAINED file: data/changelog.json. Shape: +// +// [ +// { +// "date": "2026-06-11", // YYYY-MM-DD (or full RFC3339 if you care) +// "type": "added", // added | removed | updated | note +// "id": "some-doc-id", // which document (omit for type=note) +// "title": "Document title", // shown in the headline (omit for note) +// "components": ["pdf"], // optional, for updated: what changed +// "note": "free text" // the body for type=note, optional otherwise +// } +// ] +package main + +import ( + "encoding/json" + "log" + "net/http" + "os" + "sort" + "strings" + "time" +) + +type ChangeEntry struct { + Date string `json:"date"` + Type string `json:"type"` + ID string `json:"id,omitempty"` + Title string `json:"title,omitempty"` + Components []string `json:"components,omitempty"` + Note string `json:"note,omitempty"` +} + +func (co *Corpus) loadChangelog() { + p := projectPath("changelog.json") + b, err := os.ReadFile(p) + if err != nil { + log.Printf("Note: %s not found - the changelog will be empty (see the format at the top of changelog.go).", p) + return + } + if err := json.Unmarshal(b, &co.changelog); err != nil { + warnf("%s is malformed: %v - changelog disabled.", p, err) + co.changelog = nil + return + } + sort.SliceStable(co.changelog, func(i, j int) bool { + return co.changelog[i].Date > co.changelog[j].Date + }) + for _, e := range co.changelog { + if !validChangelogDate(e.Date) { + warnf("changelog entry %q has date %q - must be YYYY-MM-DD or RFC3339", e.Headline(), e.Date) + } + if e.Type != "removed" && e.ID != "" && co.docIndex[e.ID] == nil { + warnf("changelog entry %q names document %q, which is not in the corpus", e.Headline(), e.ID) + } + } + log.Printf("Changelog loaded: %d entries", len(co.changelog)) +} + +func validChangelogDate(d string) bool { + if _, err := time.Parse("2006-01-02", d); err == nil { + return true + } + _, err := time.Parse(time.RFC3339, d) + return err == nil +} + +func (e ChangeEntry) Headline() string { + switch e.Type { + case "added": + return "Added: " + e.Title + case "removed": + return "Removed: " + e.Title + case "updated": + s := "Updated: " + e.Title + if len(e.Components) > 0 { + s += " (" + strings.Join(e.Components, ", ") + ")" + } + return s + default: // note + if e.Note != "" { + return e.Note + } + return "Archive note" + } +} + +func (e ChangeEntry) DocURL() string { + if e.ID != "" && e.Type != "removed" { + return docPath(e.ID) + } + return "/changelog" +} + +func (co *Corpus) handleChangelog(w http.ResponseWriter, r *http.Request) { + render(w, r, &changelogPage{ + Entries: co.changelog, + Total: len(co.changelog), + }) +} diff --git a/codewords.go b/codewords.go new file mode 100644 index 0000000..1c19f7d --- /dev/null +++ b/codewords.go @@ -0,0 +1,298 @@ +package main + +import ( + "html/template" + "net/http" + "net/url" + "sort" + "strings" +) + +// The codeword concordance: an automatically extracted index of ALL-CAPS +// tokens across the corpus - program names, compartments, org designators. + +var codewordStoplist = map[string]bool{ + "REDACTED": true, + "DYNAMIC": true, + "PAGE": true, + "HIGHEST": true, + "POSSIBLE": true, + "DERIVED": true, + "DECLASSIFY": true, + "JAN": true, + "ADDED": true, + "CLASSIFICATION": true, +} + +type codewordEntry struct { + Token string + Count int // total occurrences across the corpus + Docs int // number of documents containing it + First string // earliest document year containing it ("" if only undated docs) + Last string // latest such year +} + +func isCodewordToken(w string) bool { + if len(w) < 3 || w[0] < 'A' || w[0] > 'Z' { + return false + } + letters := 0 + var seen [26]bool + distinct := 0 + for i := 0; i < len(w); i++ { + c := w[i] + switch { + case c >= 'A' && c <= 'Z': + letters++ + if !seen[c-'A'] { + seen[c-'A'] = true + distinct++ + } + case c >= '0' && c <= '9': + // digits allowed + default: + return false + } + } + return letters >= 3 && distinct >= 2 +} + +func scanRuns(text string, mention func(w string)) { + start := -1 + for j := 0; j <= len(text); j++ { + alnum := false + if j < len(text) { + ch := text[j] + alnum = (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') + } + if alnum { + if start < 0 { + start = j + } + continue + } + if start >= 0 { + mention(text[start:j]) + start = -1 + } + } +} + +func (co *Corpus) buildCodewords() { + co.codewords, co.codewordTable = nil, nil + caps := map[string]int{} + other := map[string]int{} + docCaps := make([][]string, len(co.archive.Docs)) + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + here := map[string]bool{} + count := func(w string) { + switch { + case codewordStoplist[w]: + // boilerplate or archive markup (codewordStoplist) + case isCodewordToken(w): + caps[w]++ + here[w] = true + default: + other[strings.ToLower(w)]++ + } + } + scanRuns(d.Title, count) + scanRuns(d.FullText, count) + for w := range here { + docCaps[i] = append(docCaps[i], w) + } + } + for w, cnt := range caps { + if cnt < 2 || cnt <= other[strings.ToLower(w)] { + continue + } + co.codewords = append(co.codewords, codewordEntry{Token: w, Count: cnt}) + } + kept := make(map[string]*codewordEntry, len(co.codewords)) + for i := range co.codewords { + kept[co.codewords[i].Token] = &co.codewords[i] + } + + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + year := docYear(d) + inDoc := map[string]bool{} + exact := func(w string) { + if e := kept[w]; e != nil { + inDoc[e.Token] = true + } + } + folded := func(w string) { + if e := kept[strings.ToUpper(w)]; e != nil { + inDoc[e.Token] = true + } + } + for _, w := range docCaps[i] { + exact(w) + } + scanRuns(d.Description, exact) + scanRuns(d.LongDescription, exact) + for _, p := range d.DocMeta.Programs { + scanRuns(p, folded) + } + for _, t := range d.Tags { + scanRuns(t, folded) + } + for w := range inDoc { + e := kept[w] + e.Docs++ + if year == "" { + continue + } + if e.First == "" || year < e.First { + e.First = year + } + if year > e.Last { + e.Last = year + } + } + } + sort.Slice(co.codewords, func(i, j int) bool { + if co.codewords[i].Count != co.codewords[j].Count { + return co.codewords[i].Count > co.codewords[j].Count + } + return co.codewords[i].Token < co.codewords[j].Token + }) + + lowerProg := make(map[string]string, len(co.archive.Facets["program"])) + for p := range co.archive.Facets["program"] { + lowerProg[valueKey(p)] = p + } + shown := co.codewords + if len(shown) > codewordRows { + shown = shown[:codewordRows] + } + co.codewordTable = make([]codewordRow, 0, len(shown)) + for _, e := range shown { + row := codewordRow{codewordEntry: e, Gloss: co.glossaryLookup(e.Token), SearchURL: tokenSearchURL(e.Token)} + if p, ok := lowerProg[valueKey(e.Token)]; ok { + row.ProgURL = wholeArchive.link(categoryValueURL("program", p)) + } + if e.First != "" { + row.FirstURL = yearTokenURL(e.Token, e.First) + } + if e.Last != "" { + row.LastURL = yearTokenURL(e.Token, e.Last) + } + if row.Gloss != nil { + row.GlossURL = template.URL("/glossary#term-" + url.PathEscape(row.Gloss.Term)) + } + co.codewordTable = append(co.codewordTable, row) + } + co.buildCodewordViews() +} + +func yearTokenURL(token, year string) template.URL { + return template.URL(wholeArchive.link(dateRangeSearchURL("TE", exactClause(token), [2]string{"date_doc", year}))) +} + +func tokenSearchURL(token string) template.URL { + return template.URL(wholeArchive.link("/search?q=" + url.QueryEscape(exactClause(token)))) +} + +type codewordRow struct { + codewordEntry + Gloss *GlossaryEntry + ProgURL string + SearchURL template.URL + FirstURL, LastURL template.URL + GlossURL template.URL +} + +// codewordRows caps the page +const codewordRows = 500 + +var codewordCols = []sortCol{ + {Key: "token", Label: "Token"}, + {Key: "count", Label: "Occurrences", DescFirst: true}, + {Key: "docs", Label: "Documents", DescFirst: true}, + {Key: "first", Label: "First seen"}, + {Key: "last", Label: "Last seen", DescFirst: true}, + {Key: "gloss", Label: "Glossary"}, +} + +func (co *Corpus) handleCodewords(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + sortBy := sortToken(q.Get("sort")) + order := sortToken(q.Get("order")) + view, ok := co.codewordViews[sortBy+"/"+order] + if !ok { + view, ok = co.codewordViews[sortBy+"/"+codewordDefaultOrder(sortBy)] + } + if !ok { + view = co.codewordViews["count/desc"] + } + render(w, r, &codewordsPage{ + Rows: view.Rows, + RowsHTML: co.preHTML("codewords:" + view.Key), + Total: len(co.codewords), + Shown: len(view.Rows), + Heads: view.Heads, + }) +} + +type codewordView struct { + Key string + Rows []codewordRow + Heads []sortHead +} + +func codewordDefaultOrder(sortBy string) string { + for _, col := range codewordCols { + if col.Key == sortBy { + return orderParam(col.DescFirst) + } + } + return "" +} + +func (co *Corpus) buildCodewordViews() { + co.codewordViews = make(map[string]codewordView, 2*len(codewordCols)) + for _, sc := range codewordCols { + for _, order := range []string{"asc", "desc"} { + rows := append([]codewordRow(nil), co.codewordTable...) + sortCodewordRows(rows, sc.Key, order == "desc") + heads := sortHeads("?", codewordCols, sc.Key, order == "desc") + co.codewordViews[sc.Key+"/"+order] = codewordView{sc.Key + "/" + order, rows, heads} + } + } +} + +func sortCodewordRows(rows []codewordRow, sortBy string, desc bool) { + sort.SliceStable(rows, func(i, j int) bool { + var less, eq bool + switch sortBy { + case "token": + less, eq = rows[i].Token < rows[j].Token, rows[i].Token == rows[j].Token + case "docs": + less, eq = rows[i].Docs < rows[j].Docs, rows[i].Docs == rows[j].Docs + case "first", "last": + a, b := rows[i].First, rows[j].First + if sortBy == "last" { + a, b = rows[i].Last, rows[j].Last + } + if (a == "") != (b == "") { + return b == "" + } + less, eq = a < b, a == b + case "gloss": + a, b := rows[i].Gloss != nil, rows[j].Gloss != nil + less, eq = !a && b, a == b // missing before defined, ascending + default: + less, eq = rows[i].Count < rows[j].Count, rows[i].Count == rows[j].Count + } + if eq { + return rows[i].Token < rows[j].Token + } + if desc { + return !less + } + return less + }) +} diff --git a/corpus.go b/corpus.go new file mode 100644 index 0000000..59783f5 --- /dev/null +++ b/corpus.go @@ -0,0 +1,162 @@ +package main + +import ( + "html/template" + "net/http" + "sync/atomic" +) + +// Corpus is everything a load derives from the data and project directories: the +// documents, every index and table built from them, and the page bodies +// rendered ahead of time. One is built per load and never changed after it +// goes live, so handlers read it without locks. +type Corpus struct { + // api.go + facetsJSON []byte + facetsETag string + + // articles.go + articles []Article + articleIndex map[string]*Article // key -> article + articleByLink map[string]*Article // link -> article + articleGroups []articleYearGroup + articlesTitled int + articlesBooks int // of articlesTitled + + // assemblies.go + assemblies []*Assembly + assemblyIndex map[string]*Assembly + assembliesByDoc map[string][]*Assembly + + // browse.go + // homeStats holds the corpus-derived widgets on the home page - the year + // histograms and top-N facet lists. + // + // By default every figure on the home page describes the REPORTED + // collection; homeStatsSIDtoday (/?sidtoday=1) adds the SIDtoday archive. FOIA + // material can postdate the June 2013 disclosures (a 2014 "year written" + // bar on a Snowden archive reads as impossible at a glance), and the + // SIDtoday archive is +2,100 issues of an internal newsletter that swamps + // the lists and the leak-era shape the charts exist to show. + // Every bar and list entry links to its /browse/{category}/{value} page + // scoped to that collection, so the number shown is the number of + // documents the page lists. + homeStats, homeStatsSIDtoday homeFigures + + randomPools map[string][]int + collectionDocs map[string][]*Document + sidtodayDocs []*Document + + // everyDoc is every document as pointers, built once. + everyDoc []*Document + + archiveTimeline []archiveYear + categoryRowHTML map[string][]template.HTML // table key -> each entry's row, as Entries has them + azRowHTML map[string]template.HTML + categoryRowSize map[string]int + docRowHTML map[string]template.HTML + docRowMarkedHTML map[string]template.HTML // rows with the "reported" badge, for ReportedIssue documents + + // categoryTables holds every category's table under its slug. + categoryTables map[string]*categoryTable + + // seriesBySize is the series index of the SIDtoday tab. + seriesBySize map[bool][]categoryEntry + + // categoryValueCounts is how many values each category's list holds. + categoryValueCounts map[bool]map[string]int + + // categoryDocs is the posting list behind every /browse/{category}/{value} + // page: slug -> folded value -> the documents holding it, in corpus order. + categoryDocs map[string]map[string][]int32 + + // categoryNames maps each category's index keys to the spelling the lists + // and counts show: the first one in corpus order. + categoryNames map[string]map[string]string + + // categoryTypeCounts splits each category value's count by document type + // (REPORTED / SIDTODAY / FOIA): the list pages' breakdown columns. They + // count type, not collection. + categoryTypeCounts map[string]map[string]*collCounts + + // changelog.go + changelog []ChangeEntry + + // codewords.go + codewords []codewordEntry + codewordTable []codewordRow + codewordViews map[string]codewordView + + // data.go + archive Archive + serialPrev map[string]string + serialNext map[string]string + seriesIndex map[string]*seriesGroup + + // explore.go + exploreTables map[string][]exploreTable + classPatterns struct { + Strips []clsYearRow + Sparse []clsYearRow + Cols []facetEntry // crossCols, each linked to its page + HeatRows []heatRow + } + + // explore_ids.go + idRows []idRow + idChecksums string + + // glossary.go + glossary []GlossaryEntry + glossaryIndex map[string]*GlossaryEntry + glossaryGroups []glossaryGroup + + // main.go + prerendered map[string]template.HTML + + // outlets.go + outletTables map[string]outletTable + + // portions.go + portionStats portionSummary + + // search.go + tfidf TFIDFIndex + + // docIndex is populated by loadData() in data.go after archive.Docs is fully + // built. Provides O(1) doc lookup by ID. + docIndex map[string]*Document + + // seo.go + sitemapPaths []string + canonicalSitemap []byte + + // sitemap.go + sitemapParts [][]byte +} + +// live is the corpus requests are served from. It is replaced whole, never +// modified: a load builds a new Corpus and stores it here when it is done. +var live atomic.Pointer[Corpus] + +func init() { live.Store(&Corpus{}) } + +// corpus returns the live corpus. Handlers do not call it. bind hands each +// request the corpus that was live when it arrived, and everything the +// request touches is reached through that one value. +func corpus() *Corpus { return live.Load() } + +// loadData reads the corpus under dataDir into a new Corpus, pre-renders its +// big page bodies (from tmplSet, so parse the templates first). +func loadData() { + co := &Corpus{} + co.load() + co.prerenderPages() + live.Store(co) +} + +// bind adapts a handler to the mux. the request is served from the corpus +// that is live when it arrives, and from that one throughout. +func bind(h func(*Corpus, http.ResponseWriter, *http.Request)) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { h(live.Load(), w, r) } +} diff --git a/data.go b/data.go new file mode 100644 index 0000000..697fad3 --- /dev/null +++ b/data.go @@ -0,0 +1,1470 @@ +package main + +import ( + "encoding/json" + "fmt" + "html/template" + "image" + _ "image/gif" + _ "image/jpeg" + _ "image/png" + "log" + "math" + "os" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + "sync" +) + +// Publication is one entry of a document's publication list. +type Publication struct { + Title string `json:"title,omitempty"` + Publisher string `json:"publisher"` + Journalists []string `json:"reporters"` + Contributors []string `json:"contributors"` + DatePublished string `json:"date_published"` + Link string `json:"link"` + Note string `json:"note,omitempty"` + Version string `json:"version,omitempty"` + DocSource string `json:"doc_source,omitempty"` + DocURL string `json:"doc_url,omitempty"` + ReportingOnly bool `json:"reporting_only,omitempty"` +} + +type Document struct { + ID string `json:"id"` + AkaID []AkaEntry `json:"aka_id,omitempty"` + Title string `json:"title"` + Type string `json:"type"` + Collection string `json:"collection"` + Description string `json:"description"` + LongDescription string `json:"long_description"` + Tags []string `json:"tags"` + Pages int `json:"pages,omitempty"` + PDFSha256 string `json:"pdf_sha256,omitempty"` + ArchiveComment string `json:"archive_comment,omitempty"` + VersionMeta map[string]VersionInfo `json:"versions,omitempty"` + DocMeta struct { + Creator string `json:"creator"` + DateCreated string `json:"date_created"` + Programs []string `json:"programs"` + Classification []string `json:"classification"` + Dissemination []string `json:"dissemination"` + Banner string `json:"banner"` + Genre string `json:"genre"` + Series struct { + Name string `json:"name"` + Part int `json:"part"` + } `json:"series"` + Countries []string `json:"countries"` + People []string `json:"people"` + } `json:"document"` + Publication []Publication `json:"publication"` + + Dir string `json:"-"` + FullText string `json:"-"` + SearchText string `json:"-"` + StemmedText string `json:"-"` + StemmedTitle string `json:"-"` + StemmedDesc string `json:"-"` + StemmedFull string `json:"-"` // OCR body only, for "Content" field searches + FoldedFull string `json:"-"` // accent-folded + lowercased OCR body, for fast quoted "Content" matching + bodyFoldLen int // how much of FoldedFull is the fold of FullText: the versions' lines follow it + foldedTitle string // normFoldString(Title) + foldedDesc string // normFoldString(Description + valueSep + LongDescription) + // terms is the document's entry in the TF-IDF index: every stemmed token + // it holds, in any field. fields is its metadata as the matchers and the + // tallies read it (buildDocFields). Both are nil until loadData sets them. + terms map[string]float64 + fields *docFields + + Images []string `json:"-"` + ImageSizes []imageSize `json:"-"` + PDFFile string `json:"-"` + PDFSize int64 `json:"-"` + TextFile string `json:"-"` // the .md OCR/transcript's file name, "" without one + Versions []DocVersion `json:"-"` // differing published copies from versions// (loadVersions) + PortionLevels []string `json:"-"` + PortionMarkings []string `json:"-"` + PortionControls []string `json:"-"` + HasThumb bool `json:"-"` + HasGifThumb bool `json:"-"` + HTMLContent template.HTML `json:"-"` +} + +func (d *Document) foldedBody() string { + if d.bodyFoldLen > len(d.FoldedFull) || (d.bodyFoldLen == 0 && d.FullText != "") { + return normFoldString(d.FullText) // a document setBodySearchText never saw + } + return d.FoldedFull[:d.bodyFoldLen] +} + +// ReportedIssue reports whether the document is a SIDtoday issue that was +// individually reported, and so is in the Reported collection as well. +func (d *Document) ReportedIssue() bool { + return d.Type == "SIDTODAY" && d.Collection == collectionNews +} + +func (d *Document) JournalistsDeduped() []string { return facetValues(d, "RE") } + +func (d *Document) Publishers() []string { return facetValues(d, "PU") } + +func journalistsOf(d *Document) []string { + var all []string + for _, p := range d.Publication { + all = append(all, p.Journalists...) + } + return normalizeList(all) +} + +func publishersOf(d *Document) []string { + var all []string + for _, p := range d.Publication { + if !p.ReportingOnly { + all = append(all, p.Publisher) + } + } + return normalizeList(all) +} + +func (d *Document) titleFold() string { + if d.foldedTitle == "" && d.Title != "" { + return normFoldString(d.Title) + } + return d.foldedTitle +} + +func (d *Document) descFold() string { + if d.foldedDesc == "" { + return normFoldString(d.Description + valueSep + d.LongDescription) + } + return d.foldedDesc +} + +// hasTerms reports whether the document holds every one of the stemmed +// tokens, in any field. +func (d *Document) hasTerms(tokens []string) bool { + if d.terms == nil { + return true + } + for _, t := range tokens { + if _, ok := d.terms[t]; !ok { + return false + } + } + return true +} + +// AkaEntry is another entry this document was published under. +type AkaEntry struct { + ID string `json:"id"` // usually a alternative PDF filename + DocSource string `json:"doc_source,omitempty"` + DocURL string `json:"doc_url,omitempty"` +} + +type DocVersion struct { + Slug string + PDFFile string + PDFSize int64 + TextFile string + FullText string + HTMLContent template.HTML + Images []string + ImageSizes []imageSize + PubIndex []int + SHA string + NovelText string + Label string + DocID string + ChangedPages []ChangedPage +} + +type VersionInfo struct { + ChangedPages []int `json:"changed_pages,omitempty"` + PDFSha256 string `json:"pdf_sha256,omitempty"` +} + +type ChangedPage struct { + Page int + Pos int +} + +func (v *DocVersion) PageChanged(pos int) bool { + if pos < 1 { + return false + } + for _, c := range v.ChangedPages { + if c.Pos == pos { + return true + } + } + return false +} + +func (d *Document) VersionFor(slug string) *DocVersion { + if slug == "" { + return nil + } + for i := range d.Versions { + if d.Versions[i].Slug == slug { + return &d.Versions[i] + } + } + return nil +} + +// servedFile reports whether rel, a slash-separated path inside the +// document's directory, is one of the files the loader indexed: the +// metadata.json, the PDF, the transcript, the thumbnails and the page images, +// of the document or of one of its versions. /files/ serves these and nothing else a directory +// has. +func (d *Document) servedFile(rel string) bool { + if rest, ok := strings.CutPrefix(rel, "versions/"); ok { + slug, name, _ := strings.Cut(rest, "/") + v := d.VersionFor(slug) + return v != nil && contentFile(name, v.PDFFile, v.TextFile, v.Images) + } + switch rel { + case "metadata.json": + return true + case "thumb.jpg": + return d.HasThumb + case "thumb.gif": + return d.HasGifThumb + } + return contentFile(rel, d.PDFFile, d.TextFile, d.Images) +} + +func contentFile(name, pdf, text string, images []string) bool { + if name == "" { + return false + } + if name == pdf || name == text { + return true + } + image, ok := strings.CutPrefix(name, "images/") + return ok && slices.Contains(images, image) +} + +func (d *Document) UnlinkedVersions() []DocVersion { + var out []DocVersion + for _, v := range d.Versions { + if len(v.PubIndex) == 0 { + out = append(out, v) + } + } + return out +} + +type collCounts struct { + News int `json:"reported"` + Archive int `json:"sidtoday_archive"` + FOIA int `json:"foia"` +} + +func (c *collCounts) add(coll string) { + switch coll { + case collectionNews: + c.News++ + case collectionArchive: + c.Archive++ + case collectionFOIA: + c.FOIA++ + } +} + +func (c *collCounts) addType(typ string) { + switch typ { + case "SIDTODAY": + c.Archive++ + case "FOIA": + c.FOIA++ + default: + c.News++ + } +} + +var genreVocabList = []string{"slides", "report", "wiki", "manual", "guide", "newsletter", "other"} + +var genreVocab = func() map[string]bool { + m := make(map[string]bool, len(genreVocabList)) + for _, g := range genreVocabList { + m[g] = true + } + return m +}() + +type Archive struct { + Docs []Document + Types map[string]int + NewsTypes map[string]int + CollectionCounts map[string]int + Facets map[string]map[string]int + FacetsByColl map[string]map[string]*collCounts + SIDTodayCount int + RelatedDocs map[string][]string +} + +const ( + collectionNews = "news" + collectionArchive = "sidtoday-archive" + collectionFOIA = "foia" +) + +// dataDir is the corpus, the data/ directory of the archive root (-root). +var dataDir = "data" + +// projectDir, the root's project/ directory, holds the files that describe +// the archive as a whole and are the same on every mirror: glossary.json, +// assemblies.json, changelog.json and related.json. +var projectDir = "project" + +func setRoot(root string) { + dataDir, projectDir = filepath.Join(root, "data"), filepath.Join(root, "project") +} + +var loadWarnings, loadErrors int + +func warnf(format string, args ...interface{}) { + loadWarnings++ + log.Printf("Warning: "+format, args...) +} + +func errorf(format string, args ...interface{}) { + loadErrors++ + log.Printf("Error: "+format, args...) +} + +func projectPath(name string) string { return filepath.Join(projectDir, name) } + +func relatedDocsPath() string { return projectPath("related.json") } + +const defaultDocType = "REPORTED" + +var pageImageExts = map[string]bool{ + ".jpg": true, ".jpeg": true, ".png": true, ".gif": true, ".webp": true, +} + +func (co *Corpus) buildSerialNav() { + co.serialPrev = map[string]string{} + co.serialNext = map[string]string{} + var prev string + for _, yr := range co.archiveTimeline { + for _, mo := range yr.Months { + for _, d := range mo.Docs { + if prev != "" { + co.serialPrev[d.ID] = prev + co.serialNext[prev] = d.ID + } + prev = d.ID + } + } + } +} + +var collectionDirs = map[string]string{ + "foia": collectionFOIA, + "sidtoday-archive": collectionArchive, +} + +type docLocation struct { + id string + dir string + collection string +} + +func docLocations(dataDir string) ([]docLocation, error) { + entries, err := os.ReadDir(dataDir) + if err != nil { + return nil, err + } + var out []docLocation + for _, e := range entries { + if !e.IsDir() { + continue + } + if coll, ok := collectionDirs[strings.ToLower(e.Name())]; ok { + subs, err := os.ReadDir(filepath.Join(dataDir, e.Name())) + if err != nil { + warnf("cannot read %s: %v", filepath.Join(dataDir, e.Name()), err) + continue + } + for _, s := range subs { + if s.IsDir() { + out = append(out, docLocation{ + id: s.Name(), + dir: filepath.Join(dataDir, e.Name(), s.Name()), + collection: coll, + }) + } + } + continue + } + out = append(out, docLocation{ + id: e.Name(), + dir: filepath.Join(dataDir, e.Name()), + }) + } + return out, nil +} + +func dirHasPDF(dir string) bool { + entries, err := os.ReadDir(dir) + if err != nil { + return false + } + for _, e := range entries { + if !e.IsDir() && isPDF(e.Name()) { + return true + } + } + return false +} + +func isPDF(name string) bool { return strings.EqualFold(filepath.Ext(name), ".pdf") } + +func isTranscript(name string) bool { return strings.EqualFold(filepath.Ext(name), ".md") } + +func firstN(vals []string, n int) []string { + if len(vals) <= n { + return vals + } + return append(append([]string{}, vals[:n]...), fmt.Sprintf("and %d more", len(vals)-n)) +} + +func plural(n int, one, many string) string { + if n == 1 { + return one + } + return many +} + +func normalizeList(vals []string) []string { + if len(vals) == 0 { + return vals + } + out := make([]string, 0, len(vals)) + seen := make(map[string]bool) + for _, v := range vals { + v = strings.TrimSpace(v) + if v == "" { + continue + } + k := valueKey(v) + if seen[k] { + continue + } + seen[k] = true + out = append(out, v) + } + return out +} + +type seriesGroup struct { + Name string + Docs []*Document +} + +func (co *Corpus) buildSeriesIndex() { + co.seriesIndex = make(map[string]*seriesGroup) + for i := range co.archive.Docs { + dd := &co.archive.Docs[i] + name := dd.DocMeta.Series.Name + if name == "" { + continue + } + if dd.DocMeta.Series.Part <= 0 { + warnf("%s: series %q without a part number - order matters; sorting it last", dd.ID, name) + } + key := valueKey(name) + g := co.seriesIndex[key] + if g == nil { + g = &seriesGroup{Name: name} + co.seriesIndex[key] = g + } else if g.Name != name { + warnf("%s: series name casing %q differs from first-seen %q - merged, but copy-paste the exact name", dd.ID, name, g.Name) + } + g.Docs = append(g.Docs, dd) + } + for _, g := range co.seriesIndex { + sort.SliceStable(g.Docs, func(i, j int) bool { + a, b := g.Docs[i].DocMeta.Series.Part, g.Docs[j].DocMeta.Series.Part + if (a <= 0) != (b <= 0) { + return b <= 0 // numbered parts before unnumbered ones + } + if a != b { + return a < b + } + return g.Docs[i].Title < g.Docs[j].Title + }) + seen := map[int]string{} + for _, m := range g.Docs { + p := m.DocMeta.Series.Part + if p <= 0 { + continue + } + if prev, dup := seen[p]; dup { + warnf("series %q: part %d claimed by both %s and %s", g.Name, p, prev, m.ID) + } + seen[p] = m.ID + } + } +} + +func wrapRedactions(html string) string { + return strings.ReplaceAll(html, "[REDACTED]", `[REDACTED]`) +} + +// load reads the corpus under dataDir into co and builds every index the +// handlers serve from. Written here once, before the corpus goes live +// (loadData), and read-only from then on. +func (co *Corpus) load() { + loadWarnings, loadErrors = 0, 0 + + co.loadGlossary() + + co.archive = Archive{ + Docs: []Document{}, + Types: make(map[string]int), + NewsTypes: make(map[string]int), + CollectionCounts: make(map[string]int), + RelatedDocs: make(map[string][]string), + } + + locations, err := docLocations(dataDir) + if err != nil { + log.Fatalf("Critical: Could not read data directory: %v", err) + } + + seen := make(map[string]string) + var skipped, skippedWithPDF []string + for _, loc := range locations { + metaPath := filepath.Join(loc.dir, "metadata.json") + b, err := os.ReadFile(metaPath) + if err != nil { + skipped = append(skipped, loc.id) + if dirHasPDF(loc.dir) { + skippedWithPDF = append(skippedWithPDF, loc.id) + } + continue + } + + var d Document + if err := json.Unmarshal(b, &d); err != nil { + errorf("invalid JSON in %s: %v - skipping document", metaPath, err) + continue + } + + d.Type = strings.ToUpper(strings.TrimSpace(d.Type)) + if d.Type == "" { + switch loc.collection { + case collectionArchive: + d.Type = "SIDTODAY" + case collectionFOIA: + d.Type = "FOIA" + default: + d.Type = defaultDocType + } + } + d.ID = loc.id + d.Dir = loc.dir + + if prev, dup := seen[d.ID]; dup { + errorf("duplicate document id %q in %s (already loaded from %s) - skipping", + d.ID, loc.dir, prev) + continue + } + + if !docIDSafe(d.ID) { + warnf("document id %q contains characters outside [A-Za-z0-9._~-] - links, sitemap entries or pin/history cookies may misbehave; consider renaming the directory", d.ID) + } + + d.Tags = normalizeList(d.Tags) + d.AkaID = normalizeAka(&d) + + files, derr := scanContentDir(d.Dir) + if derr != nil { + errorf("cannot read %s: %v - skipping document", d.Dir, derr) + continue + } + if files.PDFFile == "" { + errorf("%s: no PDF in %s - skipping document", d.ID, d.Dir) + continue + } + seen[d.ID] = loc.dir + d.PDFFile, d.PDFSize, d.TextFile = files.PDFFile, files.PDFSize, files.TextFile + d.HasThumb, d.HasGifThumb = files.HasThumb, files.HasGifThumb + if files.Text != "" { + d.FullText = files.Text + d.HTMLContent = template.HTML(wrapRedactions(Render(d.FullText))) + } + + d.Collection = strings.ToLower(strings.TrimSpace(d.Collection)) + if loc.collection != "" { + if d.Collection != "" && d.Collection != loc.collection { + warnf("%s declares collection %q but lives in the %s directory - using %q", + d.ID, d.Collection, loc.collection, loc.collection) + } + d.Collection = loc.collection + } else { + switch d.Collection { + case collectionNews, collectionArchive, collectionFOIA: + // explicit, keep + case "": + if d.Type == "FOIA" { + d.Collection = collectionFOIA + } else { + d.Collection = collectionNews + } + default: + warnf("%s has unknown collection %q - treating as %q", + d.ID, d.Collection, collectionNews) + d.Collection = collectionNews + } + } + if d.Collection == collectionArchive && d.Type != "SIDTODAY" { + warnf("%s is in the SIDtoday archive but typed %q - the SIDtoday tab lists the type SIDTODAY, so no browse tab shows it", d.ID, d.Type) + } + co.archive.CollectionCounts[d.Collection]++ + if d.Type == "SIDTODAY" { + co.archive.SIDTodayCount++ + } + + newsDoc := d.Collection == collectionNews + co.archive.Types[d.Type]++ + if newsDoc { + co.archive.NewsTypes[d.Type]++ + } + + d.Images = loadPageImages(d.Dir, d.ID) + + if d.Pages <= 0 { + d.Pages = len(d.Images) + } + + d.DocMeta.Programs = normalizeList(d.DocMeta.Programs) + d.DocMeta.Countries = normalizeList(d.DocMeta.Countries) + d.DocMeta.People = normalizeList(d.DocMeta.People) + d.DocMeta.Classification = normalizeList(d.DocMeta.Classification) + for _, c := range d.DocMeta.Classification { + if full, ok := classAlias[strings.ToLower(c)]; ok { + warnf("%s: classification %q is an abbreviation - write %q; searches and counts treat the two as different values", d.ID, c, strings.ToUpper(full)) + } + } + d.DocMeta.Dissemination = normalizeList(d.DocMeta.Dissemination) + for i := range d.Publication { + p := &d.Publication[i] + p.Publisher = strings.TrimSpace(p.Publisher) + p.Journalists = normalizeList(p.Journalists) + p.Contributors = normalizeList(p.Contributors) + p.Note = strings.TrimSpace(p.Note) + p.DocSource = strings.TrimSpace(p.DocSource) + p.DocURL = strings.TrimSpace(p.DocURL) + } + + if v := d.DocMeta.DateCreated; v != "" && !validDate(v) { + warnf("%s: date_created %q is not YYYY, YYYY-MM or YYYY-MM-DD - treating the document as undated", d.ID, v) + d.DocMeta.DateCreated = "" + } + for i := range d.Publication { + if v := d.Publication[i].DatePublished; v != "" && !validDate(v) { + warnf("%s: publication %d date_published %q is not YYYY, YYYY-MM or YYYY-MM-DD - treating it as undated", d.ID, i+1, v) + d.Publication[i].DatePublished = "" + } + } + + co.loadVersions(&d) + + d.DocMeta.Creator = strings.TrimSpace(d.DocMeta.Creator) + d.DocMeta.Series.Name = strings.TrimSpace(d.DocMeta.Series.Name) + d.DocMeta.Banner = strings.TrimSpace(d.DocMeta.Banner) + d.DocMeta.Genre = strings.ToLower(strings.TrimSpace(d.DocMeta.Genre)) + if d.DocMeta.Genre != "" && !genreVocab[d.DocMeta.Genre] { + warnf("%s: genre %q is outside the vocabulary (%s) - serving as-is", d.ID, d.DocMeta.Genre, strings.Join(genreVocabList, ", ")) + } + + for _, f := range searchFields { + if f.Values == nil { + continue + } + for _, v := range f.Values(&d) { + if strings.Contains(v, `"`) { + warnf("%s: %s %q contains a double quote - its search links cannot quote it; use ' or “ ” instead", d.ID, strings.ToLower(f.Label), v) + } + } + } + + // The all-fields forms are the title, the descriptions and the rest + // of the metadata, then the body: each part is tokenized once, and + // tokenize of the parts joined is the parts' tokens in a row. + meta := searchMeta(&d) + metaText := strings.Join([]string{d.Title, d.Description, d.LongDescription, meta}, valueSep) + titleToks := tokenize(d.Title) + descToks := tokenize(d.Description + " " + d.LongDescription) + metaToks := tokenize(meta) + stems := make([]string, 0, len(titleToks)+len(descToks)+len(metaToks)) + stems = append(append(append(stems, titleToks...), descToks...), metaToks...) + d.StemmedTitle = strings.Join(titleToks, " ") + d.StemmedDesc = strings.Join(descToks, " ") + d.foldedTitle = normFoldString(d.Title) + d.foldedDesc = normFoldString(d.Description + valueSep + d.LongDescription) + setBodySearchText(&d) + d.StemmedText = strings.TrimSpace(strings.Join(stems, " ") + " " + d.StemmedFull) + d.SearchText = normFoldString(metaText) + valueSep + d.FoldedFull + d.FoldedFull = d.SearchText[len(d.SearchText)-len(d.FoldedFull):] + d.StemmedFull = d.StemmedText[len(d.StemmedText)-len(d.StemmedFull):] + + co.archive.Docs = append(co.archive.Docs, d) + } + + if len(skipped) > 0 { + log.Printf("Note: %d directories have no metadata.json and were skipped: %s", + len(skipped), strings.Join(firstN(skipped, 8), ", ")) + if len(skippedWithPDF) > 0 { + warnf("%d skipped %s a PDF but no metadata.json - a document nobody described: %s", + len(skippedWithPDF), plural(len(skippedWithPDF), "directory holds", "directories hold"), + strings.Join(firstN(skippedWithPDF, 8), ", ")) + } + } + + sizesRead := co.readImageSizes() + log.Printf("Building TF-IDF index...") + co.tfidf = buildTFIDF(co.archive.Docs) + log.Printf("TF-IDF done") + sizesRead() + + co.docIndex = make(map[string]*Document, len(co.archive.Docs)) + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + co.docIndex[d.ID] = d + d.terms = co.tfidf.DocStats[d.ID] + } + + co.buildSeriesIndex() + co.loadAssemblies() + + log.Printf("Loading related docs index...") + co.archive.RelatedDocs = co.loadRelatedDocs(co.archive.Docs) + log.Printf("Related docs done") + + co.buildArticles() + co.loadChangelog() + + co.buildCollectionDocs() + co.buildCategoryDocs() + co.buildHomeStats() + co.buildCodewords() + co.buildCategoryLists() + co.buildExploreTables() + co.buildIDRows() + co.buildClassPatterns() + co.buildPortions() + co.buildDocFields() + co.buildOutletTables() + co.buildArchiveTimeline() + co.buildSerialNav() + co.buildFacetsJSON() + co.buildSitemapPaths() + co.buildCanonicalSitemap() + + log.Printf("Loaded: %d documents (%d error%s, %d warning%s)", + len(co.archive.Docs), loadErrors, plural(loadErrors, "", "s"), loadWarnings, plural(loadWarnings, "", "s")) +} + +// loadRelatedDocs reads related.json if it exists - run with --rebuild-related +// to regenerate the file. +func (co *Corpus) loadRelatedDocs(docs []Document) map[string][]string { + b, err := os.ReadFile(relatedDocsPath()) + if err != nil { + warnf("related.json not found - related docs unavailable. Run with --rebuild-related to generate it.") + return make(map[string][]string) + } + + var related map[string][]string + if err := json.Unmarshal(b, &related); err != nil { + warnf("related.json is malformed: %v - related docs unavailable.", err) + return make(map[string][]string) + } + + staleKeys, staleLinks, staleLists := 0, 0, 0 + for id, ids := range related { + if co.docIndex[id] == nil { + delete(related, id) + staleKeys++ + continue + } + kept := slices.DeleteFunc(ids, co.unknownDoc) + if n := len(ids) - len(kept); n > 0 { + staleLinks += n + staleLists++ + } + related[id] = kept + } + if staleKeys > 0 || staleLinks > 0 { + warnf("related.json: %d entries for documents no longer in the corpus, and %d links to such documents in %d lists - ignored; run --rebuild-related to recompute those lists", + staleKeys, staleLinks, staleLists) + } + + missing := 0 + for _, d := range docs { + if _, ok := related[d.ID]; !ok { + missing++ + } + } + if missing > 0 { + warnf("%d/%d documents missing from related.json - run --rebuild-related to update.", missing, len(docs)) + } else { + log.Printf("Related docs loaded: %d entries", len(related)) + } + + return related +} + +func (co *Corpus) unknownDoc(id string) bool { return co.docIndex[id] == nil } + +func saveRelatedDocs(related map[string][]string) error { + b, err := json.MarshalIndent(related, "", " ") + if err != nil { + return err + } + return os.WriteFile(relatedDocsPath(), b, 0644) +} + +func (co *Corpus) buildRelatedDocs(targets, all []Document) map[string][]string { + type scored struct { + id string + score float64 + collection string + } + + programDocs := make(map[string][]int) + tagDocs := make(map[string][]int) + titleDocs := make(map[string][]int) + + for i, d := range all { + for _, p := range d.DocMeta.Programs { + key := strings.ToLower(p) + programDocs[key] = append(programDocs[key], i) + } + for _, t := range d.Tags { + key := strings.ToLower(t) + tagDocs[key] = append(tagDocs[key], i) + } + for _, w := range uniqueStrings(tokenize(d.Title)) { + titleDocs[w] = append(titleDocs[w], i) + } + } + + tw := titleWeights{df: make(map[string]int, len(titleDocs)), n: len(all)} + for w, docs := range titleDocs { + tw.df[w] = len(docs) + } + maxTitleDF := len(all) / 10 + for w, docs := range titleDocs { + if len(docs) > maxTitleDF { + delete(titleDocs, w) + } + } + + result := make(map[string][]string, len(targets)) + + for i := range targets { + d := &targets[i] + + seen := make(map[int]bool) + for _, p := range d.DocMeta.Programs { + for _, j := range programDocs[strings.ToLower(p)] { + seen[j] = true + } + } + for _, t := range d.Tags { + for _, j := range tagDocs[strings.ToLower(t)] { + seen[j] = true + } + } + for _, w := range tokenize(d.Title) { + for _, j := range titleDocs[w] { + seen[j] = true + } + } + + var candidates []scored + for j := range seen { + if all[j].ID == d.ID { + continue + } + s := co.relatedScore(d, &all[j], tw) + if s >= minRelatedScore { + candidates = append(candidates, scored{all[j].ID, s, all[j].Collection}) + } + } + + sort.Slice(candidates, func(a, b int) bool { + if candidates[a].score != candidates[b].score { + return candidates[a].score > candidates[b].score + } + return candidates[a].id < candidates[b].id + }) + + const topN = 5 + maxBulk := topN + if d.Collection == collectionNews { + maxBulk = 2 + } + var ids []string + var skipped []string + bulk := 0 + for _, c := range candidates { + if len(ids) == topN { + break + } + if c.collection != collectionNews && bulk >= maxBulk { + skipped = append(skipped, c.id) + continue + } + if c.collection != collectionNews { + bulk++ + } + ids = append(ids, c.id) + } + for _, id := range skipped { + if len(ids) == topN { + break + } + ids = append(ids, id) + } + result[d.ID] = ids + } + + log.Printf("Related docs built: %d entries", len(result)) + return result +} + +const minRelatedScore = 1.5 + +type titleWeights struct { + df map[string]int + n int +} + +func (t titleWeights) weight(w string) float64 { + if t.n < 2 { + return 1 + } + return math.Log(float64(t.n+1)/float64(t.df[w]+1)) / math.Log(float64(t.n+1)) +} + +func uniqueStrings(in []string) []string { + seen := make(map[string]bool, len(in)) + out := in[:0:0] + for _, s := range in { + if key := strings.ToLower(s); !seen[key] { + seen[key] = true + out = append(out, s) + } + } + return out +} + +func sharedCount(a, b []string) int { + inA := make(map[string]bool, len(a)) + for _, s := range a { + inA[strings.ToLower(s)] = true + } + n := 0 + for _, s := range uniqueStrings(b) { + if inA[strings.ToLower(s)] { + n++ + } + } + return n +} + +func (co *Corpus) relatedScore(a, b *Document, tw titleWeights) float64 { + var score float64 + + aTitleWords := make(map[string]bool) + for _, w := range tokenize(a.Title) { + aTitleWords[w] = true + } + for _, w := range uniqueStrings(tokenize(b.Title)) { + if aTitleWords[w] { + score += 3.0 * tw.weight(w) + } + } + + score += 4.0 * float64(sharedCount(a.DocMeta.Programs, b.DocMeta.Programs)) + score += 2.0 * float64(sharedCount(a.Tags, b.Tags)) + + var ajournos, bjournos []string + for _, p := range a.Publication { + ajournos = append(ajournos, p.Journalists...) + } + for _, p := range b.Publication { + bjournos = append(bjournos, p.Journalists...) + } + score += 2.0 * float64(sharedCount(ajournos, bjournos)) + + if strings.EqualFold(a.DocMeta.Creator, b.DocMeta.Creator) && a.DocMeta.Creator != "" { + score += 1.0 + } + + score += 0.5 * float64(sharedCount(a.DocMeta.Countries, b.DocMeta.Countries)) + score += co.tfidf.similarity(a.ID, b.ID) * 2.0 + + return score +} + +func pageNumber(s string) (int, error) { + parts := strings.SplitN(s, "-", 2) + if len(parts) < 2 { + return 0, fmt.Errorf("unexpected image filename format: %q", s) + } + numPart := strings.SplitN(parts[1], ".", 2)[0] + n, err := strconv.Atoi(numPart) + if err != nil { + return 0, fmt.Errorf("could not parse page number from %q: %v", s, err) + } + return n, nil +} + +func parseImageNum(s string) int { + n, err := pageNumber(s) + if err != nil { + warnf("%v", err) + return 0 + } + return n +} + +func docIDSafe(id string) bool { + for i := 0; i < len(id); i++ { + c := id[i] + switch { + case c >= 'a' && c <= 'z', + c >= 'A' && c <= 'Z', + c >= '0' && c <= '9', + c == '-', c == '.', c == '_', c == '~': + default: + return false + } + } + return len(id) > 0 +} + +func normalizeAka(d *Document) []AkaEntry { + var out []AkaEntry + seen := make(map[string]bool) + for i, a := range d.AkaID { + a.ID = strings.TrimSpace(a.ID) + a.DocSource = strings.TrimSpace(a.DocSource) + a.DocURL = strings.TrimSpace(a.DocURL) + if a.ID == "" { + warnf("%s: aka_id[%d] has no id - ignored", d.ID, i) + continue + } + key := strings.ToLower(a.ID) + if seen[key] { + warnf("%s: aka_id %q is listed twice - keeping the first", d.ID, a.ID) + continue + } + seen[key] = true + out = append(out, a) + } + return out +} + +func searchMeta(d *Document) string { + var reporters, publishers, pubTitles, pubNotes, contributors []string + for _, p := range d.Publication { + if p.Title != "" { + pubTitles = append(pubTitles, p.Title) + } + if p.Note != "" { + pubNotes = append(pubNotes, p.Note) + } + if p.Publisher != "" { + publishers = append(publishers, p.Publisher) + } + contributors = append(contributors, p.Contributors...) + reporters = append(reporters, p.Journalists...) + } + return strings.Join([]string{ + akaIDs(d.AkaID), + d.DocMeta.Creator, + strings.Join(reporters, valueSep), + strings.Join(contributors, valueSep), + strings.Join(publishers, valueSep), + strings.Join(pubTitles, valueSep), + strings.Join(pubNotes, valueSep), + d.ArchiveComment, + strings.Join(d.DocMeta.People, valueSep), + strings.Join(d.DocMeta.Countries, valueSep), + strings.Join(d.DocMeta.Programs, valueSep), + strings.Join(d.DocMeta.Classification, valueSep), + strings.Join(d.DocMeta.Dissemination, valueSep), + d.DocMeta.Banner, + d.DocMeta.Genre, + d.DocMeta.Series.Name, + d.ID, + strings.Join(d.Tags, valueSep), + }, valueSep) +} + +func akaIDs(akas []AkaEntry) string { + ids := make([]string, len(akas)) + for i, a := range akas { + ids[i] = a.ID + } + return strings.Join(ids, valueSep) +} + +func loadPageImages(dir, label string) []string { + files, err := os.ReadDir(filepath.Join(dir, "images")) + if err != nil && !os.IsNotExist(err) { + warnf("%s: images directory unreadable: %v", label, err) + } + type pageImage struct { + name string + num int + } + var imgs []pageImage + for _, f := range files { + if !f.IsDir() && pageImageExts[strings.ToLower(filepath.Ext(f.Name()))] { + imgs = append(imgs, pageImage{f.Name(), parseImageNum(f.Name())}) + } + } + sort.SliceStable(imgs, func(i, j int) bool { return imgs[i].num < imgs[j].num }) + var out []string + for i, im := range imgs { + if i > 0 && im.num > 0 && im.num == imgs[i-1].num { + warnf("%s: images/%s and images/%s are both page %d - keep one", label, imgs[i-1].name, im.name, im.num) + } + out = append(out, im.name) + } + return out +} + +type imageSize struct{ W, H int } + +func (s imageSize) Attr() template.HTMLAttr { + if s.W <= 0 || s.H <= 0 { + return "" + } + return template.HTMLAttr(fmt.Sprintf(` width="%d" height="%d"`, s.W, s.H)) +} + +// readImageSizes reads every page image's pixel size from its header, so a +// page can reserve each image's height before it loads. It runs alongside the +// rest of the load; the returned func waits for it and reports the failures. +func (co *Corpus) readImageSizes() func() { + type job struct { + path string + size *imageSize + } + var jobs []job + add := func(dir string, names []string, sizes *[]imageSize) { + *sizes = make([]imageSize, len(names)) + for i, name := range names { + if !strings.EqualFold(filepath.Ext(name), ".webp") { + jobs = append(jobs, job{filepath.Join(dir, "images", name), &(*sizes)[i]}) + } + } + } + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + add(d.Dir, d.Images, &d.ImageSizes) + for j := range d.Versions { + v := &d.Versions[j] + add(filepath.Join(d.Dir, "versions", v.Slug), v.Images, &v.ImageSizes) + } + } + + ch := make(chan job) + var mu sync.Mutex + var wg sync.WaitGroup + var failed []string + for range 16 { + wg.Add(1) + go func() { + defer wg.Done() + for j := range ch { + f, err := os.Open(j.path) + if err == nil { + var c image.Config + c, _, err = image.DecodeConfig(f) + f.Close() + *j.size = imageSize{c.Width, c.Height} + } + if err != nil { + mu.Lock() + failed = append(failed, fmt.Sprintf("%s: %v", strings.TrimPrefix(j.path, dataDir+string(filepath.Separator)), err)) + mu.Unlock() + } + } + }() + } + go func() { + for _, j := range jobs { + ch <- j + } + close(ch) + }() + return func() { + wg.Wait() + sort.Strings(failed) + for _, f := range failed { + warnf("page image unreadable, served without its size: %s", f) + } + } +} + +func versionSlugSafe(s string) bool { + if s == "" || strings.Contains(s, "..") || strings.Contains(s, "~") || !docIDSafe(s) { + return false + } + c := s[0] + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' +} + +func (co *Corpus) loadVersions(d *Document) { + vdir := filepath.Join(d.Dir, "versions") + entries, err := os.ReadDir(vdir) + if err != nil && !os.IsNotExist(err) { + warnf("%s: versions directory unreadable: %v", d.ID, err) + } + for _, e := range entries { + name := e.Name() + if !e.IsDir() { + warnf("%s: versions/%s is not a folder - each version is a folder holding that copy's PDF, .md and images/", d.ID, name) + continue + } + if !versionSlugSafe(name) { + warnf("%s: versions/%s: version names may only contain letters, digits, \".\", \"_\" and \"-\", starting with a letter or digit - skipping", d.ID, name) + continue + } + v, err := loadVersion(filepath.Join(vdir, name), d.ID+"/versions/"+name) + if err != nil { + errorf("%s: versions/%s: %v - skipping version", d.ID, name, err) + continue + } + v.Slug = name + v.DocID = d.ID + d.Versions = append(d.Versions, v) + } + + for i := range d.Publication { + p := &d.Publication[i] + p.Version = strings.TrimSpace(p.Version) + if p.Version == "" { + continue + } + v := d.VersionFor(p.Version) + if v == nil { + warnf("%s: publication %d (%s) points at versions/%s, which is missing or failed to load - its copy isn't linked", d.ID, i+1, p.Publisher, p.Version) + continue + } + v.PubIndex = append(v.PubIndex, i) + } + for i := range d.Versions { + v := &d.Versions[i] + v.Label = versionLabel(d, v) + } + + for slug, info := range d.VersionMeta { + v := d.VersionFor(slug) + if v == nil { + warnf("%s: \"versions\" has an entry for versions/%s, which is missing or failed to load", d.ID, slug) + continue + } + v.ChangedPages = changedPages(d.ID, v, info.ChangedPages) + v.SHA = strings.ToLower(strings.TrimSpace(info.PDFSha256)) + } + + for _, v := range d.Versions { + if v.SHA == "" { + warnf("%s: versions/%s has no pdf_sha256 in metadata.json - its checksum isn't listed", d.ID, v.Slug) + } + } +} + +func changedPages(docID string, v *DocVersion, pages []int) []ChangedPage { + pos := make(map[int]int, len(v.Images)) + for i, name := range v.Images { + if n, err := pageNumber(name); err == nil { + if _, seen := pos[n]; !seen { + pos[n] = i + 1 + } + } + } + nums := append([]int(nil), pages...) + sort.Ints(nums) + var out []ChangedPage + for i, n := range nums { + if i > 0 && n == nums[i-1] { + continue + } + if n < 1 { + warnf("%s: versions/%s: changed page %d is not a page number - ignored", docID, v.Slug, n) + continue + } + p := pos[n] + if p == 0 { + warnf("%s: versions/%s: changed page %d has no page image with that number - listed without a link", docID, v.Slug, n) + } + out = append(out, ChangedPage{Page: n, Pos: p}) + } + return out +} + +type contentFiles struct { + Text string + TextFile string + PDFFile string + PDFSize int64 + HasThumb bool + HasGifThumb bool +} + +func scanContentDir(dir string) (contentFiles, error) { + var c contentFiles + entries, err := os.ReadDir(dir) + if err != nil { + return c, err + } + textDone := false + for _, f := range entries { + if f.IsDir() { + continue + } + name := f.Name() + if !textDone && isTranscript(name) { + textDone = true + c.TextFile = name + b, rerr := os.ReadFile(filepath.Join(dir, name)) + if rerr != nil { + warnf("cannot read %s: %v", filepath.Join(dir, name), rerr) + } else { + c.Text = string(b) + } + } + if c.PDFFile == "" && isPDF(name) { + c.PDFFile = name + if info, ierr := f.Info(); ierr == nil { + c.PDFSize = info.Size() + } + } + if name == "thumb.jpg" { + c.HasThumb = true + } + if name == "thumb.gif" { + c.HasGifThumb = true + } + } + return c, nil +} + +func loadVersion(dir, label string) (DocVersion, error) { + var v DocVersion + files, err := scanContentDir(dir) + if err != nil { + return v, err + } + if files.PDFFile == "" { + return v, fmt.Errorf("no PDF") + } + v.PDFFile, v.PDFSize, v.TextFile, v.FullText = files.PDFFile, files.PDFSize, files.TextFile, files.Text + if v.FullText != "" { + v.HTMLContent = template.HTML(wrapRedactions(Render(v.FullText))) + } + v.Images = loadPageImages(dir, label) + return v, nil +} + +// VersionSnippet is a search-result snippet taken from one of the document's +// versions, with the copy it came from. +type VersionSnippet struct { + Slug string + Label string + HTML template.HTML +} + +func versionLabel(d *Document, v *DocVersion) string { + var names []string + seen := make(map[string]bool) + date := "" + for _, i := range v.PubIndex { + p := d.Publication[i] + if key := valueKey(p.Publisher); key != "" && !seen[key] { + seen[key] = true + names = append(names, p.Publisher) + } + if p.DatePublished != "" && (date == "" || p.DatePublished < date) { + date = p.DatePublished + } + } + if len(names) == 0 { + return v.Slug + } + label := strings.Join(names, " / ") + if date != "" { + label += ", " + date + } + return label +} + +// lineKey is the form transcript lines are compared in: folded, lowercased, +// whitespace collapsed. Lines without a letter or digit ("---", "|") key to +// "" and are never new. +func lineKey(line string) string { + k := strings.Join(strings.Fields(normFoldString(line)), " ") + if strings.IndexFunc(k, isWordRune) < 0 { + return "" + } + return k +} + +func versionNovelText(d *Document) string { + if len(d.Versions) == 0 { + return "" + } + inMain := make(map[string]bool) + for _, line := range strings.Split(d.FullText, "\n") { + if k := lineKey(line); k != "" { + inMain[k] = true + } + } + added := make(map[string]bool) + var all []string + for i := range d.Versions { + v := &d.Versions[i] + var novel []string + for _, line := range strings.Split(v.FullText, "\n") { + k := lineKey(line) + if k == "" || inMain[k] { + continue + } + line = strings.TrimSpace(line) + novel = append(novel, line) + if !added[k] { + added[k] = true + all = append(all, line) + } + } + v.NovelText = strings.Join(novel, "\n") + } + return strings.Join(all, "\n") +} + +func setBodySearchText(d *Document) { + d.StemmedFull = strings.Join(tokenize(d.FullText), " ") + d.FoldedFull = normFoldString(d.FullText) + d.bodyFoldLen = len(d.FoldedFull) + if novel := versionNovelText(d); novel != "" { + d.StemmedFull = strings.TrimSpace(d.StemmedFull + " " + valueSep + " " + strings.Join(tokenize(novel), " ")) + d.FoldedFull += valueSep + normFoldString(novel) + } +} diff --git a/explore.go b/explore.go new file mode 100644 index 0000000..78ea5ec --- /dev/null +++ b/explore.go @@ -0,0 +1,377 @@ +package main + +import ( + "log" + "net/http" + "sort" + "strconv" + "strings" +) + +// The Explore section (/explore). + +type exploreCol struct { + Header string + Key string + Ratio bool +} + +type exploreMetric struct { + Slug string + Title string + Blurb string + Cols []exploreCol + Vals func(d *Document) ([]float64, bool) +} + +const exploreRows = 200 // rows shown per page ("top 200 of N") + +var exploreMetrics = []exploreMetric{ + { + Slug: "programs", + Title: "Most programs mentioned", + Blurb: "Documents referencing the most surveillance programs.", + Cols: []exploreCol{ + {Header: "Programs", Key: "programs"}, + {Header: "Pages", Key: "pages"}, + {Header: "Programs / page", Key: "ratio", Ratio: true}, + }, + Vals: func(d *Document) ([]float64, bool) { + n := len(d.DocMeta.Programs) + if n == 0 { + return nil, false + } + pages, ratio := float64(d.Pages), -1.0 + if d.Pages > 0 { + ratio = float64(n) / float64(d.Pages) + } + return []float64{float64(n), pages, ratio}, true + }, + }, + { + Slug: "people", + Title: "Most people mentioned", + Blurb: "Documents whose metadata names the most individuals.", + Cols: []exploreCol{{Header: "People", Key: "people"}}, + Vals: func(d *Document) ([]float64, bool) { + n := len(d.DocMeta.People) + return []float64{float64(n)}, n > 0 + }, + }, + { + Slug: "countries", + Title: "Most countries mentioned", + Blurb: "Documents spanning the most countries.", + Cols: []exploreCol{{Header: "Countries", Key: "countries"}}, + Vals: func(d *Document) ([]float64, bool) { + n := len(d.DocMeta.Countries) + return []float64{float64(n)}, n > 0 + }, + }, + { + Slug: "pages", + Title: "Longest documents", + Blurb: "Documents with the most pages.", + Cols: []exploreCol{{Header: "Pages", Key: "pages"}}, + Vals: func(d *Document) ([]float64, bool) { + return []float64{float64(d.Pages)}, d.Pages > 0 + }, + }, +} + +func exploreMetricBySlug(slug string) *exploreMetric { + for i := range exploreMetrics { + if exploreMetrics[i].Slug == slug { + return &exploreMetrics[i] + } + } + return nil +} + +func (co *Corpus) handleExploreIndex(w http.ResponseWriter, r *http.Request) { + render(w, r, &explorePage{ + Metrics: exploreMetrics, + HasAssemblies: len(co.assemblies) > 0, + }) +} + +type exploreRow struct { + Rank int + Doc Document + Vals []string +} + +type exploreTable struct { + Rows []exploreRow + Total int +} + +func exploreSet(slug string, foia bool) string { + if foia { + return slug + } + return slug + ":nofoia" +} + +func (co *Corpus) buildExploreTables() { + co.exploreTables = make(map[string][]exploreTable, 2*len(exploreMetrics)) + for mi := range exploreMetrics { + m := &exploreMetrics[mi] + if _, dup := co.exploreTables[m.Slug]; dup { + log.Fatalf("explore: duplicate metric slug %q", m.Slug) + } + for _, foia := range []bool{true, false} { + co.exploreTables[exploreSet(m.Slug, foia)] = co.buildExploreTable(m, foia) + } + } +} + +func (co *Corpus) buildExploreTable(m *exploreMetric, foia bool) []exploreTable { + type scored struct { + doc *Document + vals []float64 + } + var rows []scored + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + if !foia && d.Collection == collectionFOIA { + continue + } + if vals, ok := m.Vals(d); ok { + rows = append(rows, scored{d, vals}) + } + } + + tables := make([]exploreTable, len(m.Cols)) + for sortIdx := range m.Cols { + sorted := append([]scored(nil), rows...) + sort.SliceStable(sorted, func(i, j int) bool { + if a, b := sorted[i].vals[sortIdx], sorted[j].vals[sortIdx]; a != b { + return a > b + } + if a, b := sorted[i].vals[0], sorted[j].vals[0]; a != b { + return a > b + } + return sorted[i].doc.Title < sorted[j].doc.Title + }) + if len(sorted) > exploreRows { + sorted = sorted[:exploreRows] + } + out := make([]exploreRow, len(sorted)) + for i, rw := range sorted { + vals := make([]string, len(rw.vals)) + for k, v := range rw.vals { + switch { + case v < 0: + vals[k] = "-" + case m.Cols[k].Ratio: + vals[k] = trimRatio(v) + default: + vals[k] = strconv.Itoa(int(v)) + } + } + out[i] = exploreRow{Rank: i + 1, Doc: *rw.doc, Vals: vals} + } + tables[sortIdx] = exploreTable{Rows: out, Total: len(rows)} + } + return tables +} + +func exploreKey(set string, sortIdx int) string { + return "explore:" + set + ":" + strconv.Itoa(sortIdx) +} + +func (co *Corpus) handleExploreMetric(w http.ResponseWriter, r *http.Request) { + m := exploreMetricBySlug(r.PathValue("metric")) + if m == nil { + notFound(w, r) + return + } + + sortIdx := 0 + sortBy := r.URL.Query().Get("sort") + for i, c := range m.Cols { + if c.Key == sortBy { + sortIdx = i + break + } + } + set := exploreSet(m.Slug, readPrefs(r).IncludeFOIA) + t := co.exploreTables[set][sortIdx] + + cols := []sortCol{{Label: "#", Class: "num-col"}, {Label: "Document"}, {Label: "Date"}} + for _, c := range m.Cols { + cols = append(cols, sortCol{Key: c.Key, Label: c.Header, Class: "num-col", DescFirst: true, Fixed: true}) + } + + render(w, r, &exploreTablePage{ + Metric: m, + Rows: t.Rows, + RowsHTML: co.preHTML(exploreKey(set, sortIdx)), + Total: t.Total, + Shown: len(t.Rows), + Heads: sortHeads("?", cols, m.Cols[sortIdx].Key, true), + }) +} + +func trimRatio(v float64) string { + s := strconv.FormatFloat(v, 'f', 2, 64) + s = strings.TrimRight(s, "0") + return strings.TrimRight(s, ".") +} + +// --------------------------------------------------------------------------- +// /explore/classification + +type clsSeg struct { + Value string + Count int + Pct int + URL string + Class string // seg-0..seg-3, one per stripLevels entry +} + +type clsYearRow struct { + Year string + URL string // the year's page, over every collection as the counts are + Total int + Segs []clsSeg +} + +type heatCell struct { + Count int + URL string +} + +type heatRow struct { + Name string + URL string + Cells []heatCell +} + +var crossCols = []string{"TOP SECRET", "SECRET", "CONFIDENTIAL", "UNCLASSIFIED", "SI", "COMINT", "STRAP 1", "STRAP 2", "TK"} +var stripLevels = []string{"TOP SECRET", "SECRET", "CONFIDENTIAL", "UNCLASSIFIED"} + +const stripMinMarks = 5 + +func (co *Corpus) buildClassPatterns() { + top := crossCols + rank := make(map[string]int, len(top)) + for i, v := range top { + rank[valueKey(v)] = i + } + levelRank := make(map[string]int, len(stripLevels)) + for i, v := range stripLevels { + levelRank[valueKey(v)] = i + } + + yearCounts := map[string][]int{} + agCounts := map[string][]int{} + agName := map[string]string{} + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + year := docYear(d) + agency := d.DocMeta.Creator + if n, ok := agName[valueKey(agency)]; ok { + agency = n + } else { + agName[valueKey(agency)] = agency + } + for _, cv := range d.DocMeta.Classification { + if lvl, isLevel := levelRank[valueKey(cv)]; isLevel && year != "" { + yc := yearCounts[year] + if yc == nil { + yc = make([]int, len(stripLevels)) + yearCounts[year] = yc + } + yc[lvl]++ + } + idx, known := rank[valueKey(cv)] + if known && agency != "" { + ac := agCounts[agency] + if ac == nil { + ac = make([]int, len(top)) + agCounts[agency] = ac + } + ac[idx]++ + } + } + } + + years := make([]string, 0, len(yearCounts)) + for y := range yearCounts { + years = append(years, y) + } + sort.Strings(years) + strips := make([]clsYearRow, 0, len(years)) + var sparse []clsYearRow + for _, y := range years { + yc := yearCounts[y] + total := 0 + for _, n := range yc { + total += n + } + row := clsYearRow{Year: y, URL: wholeArchive.link(categoryValueURL("docdate", y)), Total: total} + if total < stripMinMarks { + sparse = append(sparse, row) + continue + } + for i, n := range yc { + if n == 0 { + continue + } + row.Segs = append(row.Segs, clsSeg{ + Value: stripLevels[i], + Count: n, + Pct: barPct(n, total, 1), + URL: wholeArchive.link(dateRangeSearchURL("CL", exactClause(stripLevels[i]), [2]string{"date_doc", y})), + Class: "seg-" + strconv.Itoa(i), + }) + } + strips = append(strips, row) + } + + mass := make(map[string]int, len(agCounts)) + for a, cs := range agCounts { + for _, n := range cs { + mass[a] += n + } + } + ags := rankCounts(mass, -1) + rows := make([]heatRow, 0, len(ags)) + for _, a := range ags { + hr := heatRow{Name: a, URL: wholeArchive.link(categoryValueURL("agency", a))} + for i, n := range agCounts[a] { + cell := heatCell{Count: n} + if n > 0 { + cell.URL = wholeArchive.link(searchURL([]searchClause{ + {Field: "CR", Value: exactClause(a)}, + {Field: "CL", Value: exactClause(top[i])}, + })) + } + hr.Cells = append(hr.Cells, cell) + } + rows = append(rows, hr) + } + + cols := make([]facetEntry, len(top)) + for i, v := range top { + cols[i] = facetEntry{Name: v, URL: wholeArchive.link(categoryValueURL("classification", v))} + } + + co.classPatterns.Strips = strips + co.classPatterns.Sparse = sparse + co.classPatterns.Cols = cols + co.classPatterns.HeatRows = rows +} + +func (co *Corpus) handleClassPatterns(w http.ResponseWriter, r *http.Request) { + render(w, r, &classPatternsPage{ + Cols: co.classPatterns.Cols, + Levels: stripLevels, + Strips: co.classPatterns.Strips, + Sparse: co.classPatterns.Sparse, + MinMarks: stripMinMarks, + HeatRows: co.classPatterns.HeatRows, + }) +} diff --git a/explore_ids.go b/explore_ids.go new file mode 100644 index 0000000..58a873f --- /dev/null +++ b/explore_ids.go @@ -0,0 +1,114 @@ +// explore_ids.go - /explore/ids: every document's identifiers in one table. +// +// /explore/ids.txt is the same checksums in sha256sum(1) format, so a mirror +// or a download verifies with "sha256sum -c" in the corpus root. +package main + +import ( + "fmt" + "net/http" + "path/filepath" + "sort" + "strings" +) + +type idVersionRow struct { + Slug string + PDFFile string + PDFURL string + Size string + Pages int + SHA string +} + +type idRow struct { + ID string + Collection string + Pages int + PDFFile string + PDFURL string + Size string + SHA string + Aka []AkaEntry + Versions []idVersionRow +} + +func humanSize(n int64) string { + switch { + case n <= 0: + return "" + case n < 1<<10: + return fmt.Sprintf("%d B", n) + case n < 1<<20: + return fmt.Sprintf("%.0f KB", float64(n)/(1<<10)) + case n < 1<<30: + return fmt.Sprintf("%.1f MB", float64(n)/(1<<20)) + } + return fmt.Sprintf("%.2f GB", float64(n)/(1<<30)) +} + +func (co *Corpus) buildIDRows() { + co.idRows = make([]idRow, 0, len(co.archive.Docs)) + var sums strings.Builder + + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + row := idRow{ + ID: d.ID, + Collection: d.Collection, + Pages: d.Pages, + PDFFile: d.PDFFile, + PDFURL: filePath(d.ID, d.PDFFile), + Size: humanSize(d.PDFSize), + SHA: strings.TrimSpace(d.PDFSha256), + Aka: d.AkaID, + } + for _, v := range d.Versions { + row.Versions = append(row.Versions, idVersionRow{ + Slug: v.Slug, + PDFFile: v.PDFFile, + PDFURL: versionFilePath(d.ID, v.Slug, v.PDFFile), + Size: humanSize(v.PDFSize), + Pages: len(v.Images), + SHA: v.SHA, + }) + } + co.idRows = append(co.idRows, row) + + rel, err := filepath.Rel(dataDir, d.Dir) + if err != nil { + rel = d.ID + } + if row.SHA != "" { + sums.WriteString(row.SHA + " " + filepath.ToSlash(filepath.Join(rel, d.PDFFile)) + "\n") + } + for _, v := range d.Versions { + if v.SHA != "" { + sums.WriteString(v.SHA + " " + filepath.ToSlash(filepath.Join(rel, "versions", v.Slug, v.PDFFile)) + "\n") + } + } + } + + sort.Slice(co.idRows, func(i, j int) bool { + a, b := strings.ToLower(co.idRows[i].ID), strings.ToLower(co.idRows[j].ID) + if a != b { + return a < b + } + return co.idRows[i].ID < co.idRows[j].ID + }) + co.idChecksums = sums.String() +} + +func (co *Corpus) handleExploreIDs(w http.ResponseWriter, r *http.Request) { + render(w, r, &idsPage{ + Rows: co.idRows, + RowsHTML: co.preHTML("ids"), + Shown: len(co.idRows), + }) +} + +func (co *Corpus) handleExploreIDsText(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Cache-Control", "public, max-age=3600") + w.Write([]byte(co.idChecksums)) +} diff --git a/glossary.go b/glossary.go new file mode 100644 index 0000000..2c89d63 --- /dev/null +++ b/glossary.go @@ -0,0 +1,135 @@ +// glossary.go - codename/term glossary loaded from data/glossary.json. + +package main + +import ( + "encoding/json" + "html/template" + "log" + "net/http" + "os" + "sort" + "strings" +) + +type GlossaryEntry struct { + Term string `json:"term"` + Aka []string `json:"aka,omitempty"` + Definition string `json:"definition"` + See []string `json:"see,omitempty"` + SeeLinks []seeLink `json:"-"` + + searchLink string +} + +type seeLink struct{ Name, Term string } + +func (e GlossaryEntry) DefinitionHTML() template.HTML { + return template.HTML(e.Definition) +} + +func (e GlossaryEntry) SearchURL() string { + if e.searchLink != "" { + return e.searchLink + } + return e.buildSearchURL() +} + +func (e GlossaryEntry) buildSearchURL() string { + clauses := []searchClause{{Conj: "and", Field: "TE", Value: searchClauseValue(e.Term)}} + for _, a := range e.Aka { + if strings.TrimSpace(a) != "" { + clauses = append(clauses, searchClause{Conj: "or", Field: "TE", Value: searchClauseValue(a)}) + } + } + return searchURL(clauses) +} + +func searchClauseValue(v string) string { + v = strings.TrimSpace(v) + if len(splitWords(v)) == 1 && len(splitWords(v)[0]) == len(v) { + return v + } + return `"` + v + `"` +} + +func (co *Corpus) glossaryLookup(term string) *GlossaryEntry { + return co.glossaryIndex[strings.ToLower(term)] +} + +func (co *Corpus) loadGlossary() { + p := projectPath("glossary.json") + b, err := os.ReadFile(p) + if err != nil { + log.Printf("Note: %s not found - glossary page will be empty.", p) + return + } + if err := json.Unmarshal(b, &co.glossary); err != nil { + warnf("%s is malformed: %v - glossary disabled.", p, err) + co.glossary = nil + return + } + co.glossaryIndex = map[string]*GlossaryEntry{} + sort.SliceStable(co.glossary, func(i, j int) bool { + return normFoldString(co.glossary[i].Term) < normFoldString(co.glossary[j].Term) + }) + for i := range co.glossary { + e := &co.glossary[i] + e.searchLink = e.buildSearchURL() + addIndex := func(key string) { + low := strings.ToLower(key) + if prev := co.glossaryIndex[low]; prev != nil && prev != e { + warnf("glossary term/alias %q appears under both %q and %q - the later entry wins", key, prev.Term, e.Term) + } + co.glossaryIndex[low] = e + } + addIndex(e.Term) + for _, a := range e.Aka { + addIndex(a) + } + } + var unknown []string + for i := range co.glossary { + e := &co.glossary[i] + e.SeeLinks = nil + for _, name := range e.See { + l := seeLink{Name: name} + if t := co.glossaryLookup(name); t != nil { + l.Term = t.Term + } else { + unknown = append(unknown, name) + } + e.SeeLinks = append(e.SeeLinks, l) + } + } + if len(unknown) > 0 { + warnf("glossary: %d see-also names match no term or alias, shown unlinked: %s", + len(unknown), strings.Join(firstN(unknown, 8), ", ")) + } + co.buildGlossaryGroups() + log.Printf("Glossary loaded: %d terms", len(co.glossary)) +} + +type glossaryGroup struct { + Letter string + Entries []GlossaryEntry +} + +func (co *Corpus) buildGlossaryGroups() { + co.glossaryGroups = nil + for _, e := range co.glossary { + letter := firstLetter(e.Term) + if len(co.glossaryGroups) == 0 || co.glossaryGroups[len(co.glossaryGroups)-1].Letter != letter { + co.glossaryGroups = append(co.glossaryGroups, glossaryGroup{Letter: letter}) + } + co.glossaryGroups[len(co.glossaryGroups)-1].Entries = append(co.glossaryGroups[len(co.glossaryGroups)-1].Entries, e) + } +} + +func (co *Corpus) handleGlossary(w http.ResponseWriter, r *http.Request) { + render(w, r, &glossaryPage{ + Groups: co.glossaryGroups, + GroupsHTML: co.preHTML("glossary"), + Total: len(co.glossary), + }) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8578a29 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module snowchive + +go 1.22 diff --git a/history.go b/history.go new file mode 100644 index 0000000..8b6c03a --- /dev/null +++ b/history.go @@ -0,0 +1,194 @@ +// history.go - opt-in, cookie-local browsing history: recently viewed +// documents and recent searches. Both are OFF by default (see Prefs). + +package main + +import ( + "net/http" + "net/url" + "strings" +) + +const ( + recentDocsCookie = "recent_docs" + recentSearchesCookie = "recent_searches" + recentDocsMax = 20 + recentSearchesMax = 15 +) + +func readIDListCookie(r *http.Request, name string, max int) []string { + c, err := r.Cookie(name) + if err != nil || c.Value == "" { + return nil + } + var out []string + for _, id := range strings.Split(c.Value, "|") { + if id != "" { + out = append(out, id) + if len(out) == max { + break + } + } + } + return out +} + +func pushFront(list []string, id string, max int) []string { + out := make([]string, 0, max) + out = append(out, id) + for _, x := range list { + if len(out) == max { + break + } + if x != id { + out = append(out, x) + } + } + return out +} + +func pushRecentDoc(w http.ResponseWriter, r *http.Request, id string) { + ids := pushFront(readIDListCookie(r, recentDocsCookie, recentDocsMax), id, recentDocsMax) + writeListCookie(w, recentDocsCookie, ids) +} + +type recentSearch struct { + Label string + URL string + QS string +} + +func recordRecentSearch(w http.ResponseWriter, r *http.Request, qs string) []string { + if len(qs) > searchQSMax { + return nil + } + entry := url.QueryEscape(qs) + + entries := readIDListCookie(r, recentSearchesCookie, recentSearchesMax) + entries = pushFront(entries, entry, recentSearchesMax) + return writeListCookie(w, recentSearchesCookie, entries) +} + +func (co *Corpus) readRecentSearches(r *http.Request) []recentSearch { + return co.parseSearchEntries(readIDListCookie(r, recentSearchesCookie, recentSearchesMax)) +} + +func (co *Corpus) parseSearchEntries(entries []string) []recentSearch { + var out []recentSearch + for _, e := range entries { + qs, err := url.QueryUnescape(e) + if err != nil { + continue + } + v, err := url.ParseQuery(qs) + if err != nil { + continue + } + if label, ok := co.searchLabel(v); ok { + out = append(out, recentSearch{Label: label, URL: "/search?" + qs, QS: qs}) + } + } + return out +} + +const searchQSMax = 500 + +func (co *Corpus) savableSearch(v url.Values) bool { + if len(v.Encode()) > searchQSMax { + return false + } + _, ok := co.searchLabel(v) + return ok +} + +func (co *Corpus) searchLabel(v url.Values) (label string, ok bool) { + var parts []string + fields := v["fqf"] + conjs := v["fqc"] + for i, term := range v["fqv"] { + if term == "" { + continue + } + label := term + if i < len(fields) && fields[i] != "" && fields[i] != "TE" { + code := fields[i] + if f, ok := fieldByCode[code]; ok { + label = term + " [" + f.Label + "]" + } else { + label = term + " [" + code + "]" + } + } + if i < len(conjs) && conjs[i] == "not" { + label = "-" + label + } + parts = append(parts, label) + } + var notes, filters []string + if colls := v["coll"]; len(colls) > 0 && len(colls) < len(validCollections) { + filters = append(filters, colls...) + } + if types := v["type"]; len(types) > 0 && (len(co.archive.Types) == 0 || len(types) < len(co.archive.Types)) { + filters = append(filters, types...) + } + if len(filters) > 0 { + notes = append(notes, "("+strings.Join(filters, ", ")+")") + } + if n := dateRangeNote("doc", v.Get("date_doc_start"), v.Get("date_doc_end")); n != "" { + notes = append(notes, n) + } + if n := dateRangeNote("pub", v.Get("date_pub_start"), v.Get("date_pub_end")); n != "" { + notes = append(notes, n) + } + switch { + case len(parts) == 0 && len(notes) == 0: + return "", false + case len(parts) == 0: + parts = []string{"all documents"} + } + return strings.Join(append(parts, notes...), " · "), true +} + +func dateRangeNote(kind, start, end string) string { + switch { + case start == "" && end == "": + return "" + case start == end || (start != "" && end == padDateEnd(start)): + // date:2013 arrives as start "2013", end "2013-12-31": one year + return "(" + kind + " " + start + ")" + case start == "": + return "(" + kind + " to " + end + ")" + case end == "": + return "(" + kind + " from " + start + ")" + default: + return "(" + kind + " " + start + " to " + end + ")" + } +} + +func handleHistoryClear(w http.ResponseWriter, r *http.Request) { + if !readPOSTForm(w, r) { + return + } + switch r.FormValue("which") { + case "docs": + delCookie(w, recentDocsCookie) + case "searches": + delCookie(w, recentSearchesCookie) + default: + delCookie(w, recentDocsCookie) + delCookie(w, recentSearchesCookie) + } + http.Redirect(w, r, safeBackPath(r.FormValue("back")), http.StatusSeeOther) +} + +// safeBackPath only allows local absolute paths as redirect targets. +func safeBackPath(p string) string { + if !strings.HasPrefix(p, "/") || strings.HasPrefix(p, "//") { + return "/" + } + for i := 0; i < len(p); i++ { + if p[i] == '\\' || p[i] < 0x20 { + return "/" + } + } + return p +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..cc97920 --- /dev/null +++ b/main.go @@ -0,0 +1,773 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "embed" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "html/template" + "io/fs" + "log" + "maps" + "net" + "net/http" + "net/url" + "os" + "os/signal" + "path" + "path/filepath" + "slices" + "strconv" + "strings" + "syscall" + "time" +) + +// devMode is set by the -dev flag. In dev mode templates and static files +// are read from ./templates and ./static on every request (so edits show +// without a rebuild), caching is disabled and error details are shown in +// responses. Never run production with -dev: the error detail leaks +// filesystem paths and template internals. +var devMode bool + +// The templates and static assets are compiled into the binary, so a +// deployment is one executable plus the data and project directories - nothing depends +// on the working directory the process happens to start in. -dev swaps in +// the on-disk copies (see initAssets). +// +//go:embed templates static +var embeddedAssets embed.FS + +var ( + templateFS fs.FS // templates/*.html + staticFS fs.FS // static/* +) + +// staticHashes holds a content hash per embedded asset, computed once at +// startup. It is both the assets' ETag (embed.FS files have a zero modtime, +// so net/http sends no Last-Modified and an unvalidated asset would be +// re-downloaded on every view) and their version stamp: base.html links +// them as ?v= (the asset func), and handleStatic caches exactly that +// URL forever. A changed file gets a new hash and so a new URL, whatever +// the build was stamped with. +var staticHashes map[string]string + +func buildStaticHashes() { + staticHashes = map[string]string{} + err := fs.WalkDir(staticFS, ".", func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + b, err := fs.ReadFile(staticFS, p) + if err != nil { + return err + } + sum := sha256.Sum256(b) + staticHashes[p] = hex.EncodeToString(sum[:8]) + return nil + }) + if err != nil { + log.Fatalf("embedded static: %v", err) + } +} + +func initAssets() { + if devMode { + templateFS = os.DirFS("templates") + staticFS = os.DirFS("static") + staticHashes = nil + return + } + var err error + if templateFS, err = fs.Sub(embeddedAssets, "templates"); err != nil { + log.Fatalf("embedded templates: %v", err) + } + if staticFS, err = fs.Sub(embeddedAssets, "static"); err != nil { + log.Fatalf("embedded static: %v", err) + } + buildStaticHashes() +} + +type sortOption struct{ Key, Label string } + +var sortOptions = func() []sortOption { + opts := []sortOption{{"relevance", "Relevance"}} + for _, s := range docSorts { + if s.InBar { + opts = append(opts, sortOption{s.Key, s.Label}) + } + } + return opts +}() + +func sortLabel(key string) string { + switch key { + case "relevance": + return "Relevance" + case "": + return "Title" + } + if s := docSortByKey[key]; s != nil { + return s.Label + } + return strings.ToUpper(key[:1]) + key[1:] +} + +var ( + // tmplSet holds one parsed template set PER PAGE (base + partials + + // page). + tmplSet map[string]*template.Template + + funcMap = template.FuncMap{ + "buildInfo": func() map[string]string { + return map[string]string{"Commit": buildCommit} + }, + // articleKey is the key of the article page for a publication's + // link. Every linked publication of a loaded document has one + // (buildArticles), so the key needs no lookup - and no corpus. + "articleKey": func(link string) string { + if link == "" { + return "" + } + return articleKeyFor(link) + }, + "asset": func(name string) string { + if h, ok := staticHashes[name]; ok { + return "/static/" + name + "?v=" + h + } + return "/static/" + name + }, + // wholeArchiveQS is the scope that names every collection, for the + // links of a FOIA release on a page rendered once for all readers. + "wholeArchiveQS": func() string { return wholeArchive.suffix() }, + "facetLinks": facetLinks, + "add": func(a, b int) int { return a + b }, + "eqFold": sameValue, + "sortOptions": func() []sortOption { return sortOptions }, + "sortLabel": sortLabel, + "agencyName": agencyName, + "dict": func(values ...interface{}) (map[string]interface{}, error) { + if len(values)%2 != 0 { + return nil, fmt.Errorf("dict: odd number of arguments (%d)", len(values)) + } + dict := make(map[string]interface{}, len(values)/2) + for i := 0; i < len(values); i += 2 { + key, ok := values[i].(string) + if !ok { + return nil, fmt.Errorf("dict: argument %d is not a string key (%T)", i, values[i]) + } + dict[key] = values[i+1] + } + return dict, nil + }, + // pathEscape percent-encodes a value for use as a single URL path + // segment (so values like "TOP SECRET//SI" don't split the path). + "pathEscape": func(s string) template.URL { + return template.URL(url.PathEscape(s)) + }, + "azID": azID, + "pages": func(current, total int) []int { + var out []int + last := 0 + for i := 1; i <= total; i++ { + if i == 1 || i == total || (i >= current-2 && i <= current+2) { + if last != 0 && i-last > 1 { + out = append(out, 0) + } + out = append(out, i) + last = i + } + } + return out + }, + } +) + +// Build identity, stamped at build time so mirrors are comparable at a +// glance. Shown in the footer next to the data load time: +// +// go build -ldflags "-X main.buildCommit=$(git rev-parse --short HEAD)" +var buildCommit = "dev" + +// accelPrefix is set by -accel-files: the internal nginx location that maps +// to the data directory. With it, handleFiles still decides whether a file is +// served, but answers with an X-Accel-Redirect and nginx sends the bytes +// (sendfile, ranges, slow readers) without them passing through this process. +// Only set it behind nginx: any other client gets an empty 200. +var accelPrefix string + +// accelExpires is set by -accel-expires: how long the proxy may keep a page +// this server marks public, sent as X-Accel-Expires (which nginx obeys ahead +// of Cache-Control and strips from the response). Browsers still get the +// short max-age. The corpus is read once at startup, so such a page cannot +// change before the next restart; clear the proxy's cache after one. +var accelExpires string + +// apiEnabled is cleared by -no-api: public instances that don't want bulk +// scraping can turn the JSON API off. +var apiEnabled = true + +func main() { + rebuildRelated := flag.Bool("rebuild-related", false, "Rebuild related docs index and exit") + flag.BoolVar(&devMode, "dev", false, "Dev mode: serve templates/ and static/ from the working directory and re-parse templates per request, no immutable static cache, verbose errors") + baseFlag := flag.String("base", os.Getenv("SNOWCHIVE_BASE"), + "Canonical base URL (e.g. https://archive.example.org) for the absolute URLs in robots.txt and the sitemap. "+ + "Without it those URLs are built from each request's Host header, which is fine only behind a proxy that pins Host. "+ + "Set this on anything internet-facing. Defaults to $SNOWCHIVE_BASE.") + listen := flag.String("listen", envOr("SNOWCHIVE_LISTEN", "127.0.0.1:8080"), + "TCP address to listen on. Defaults to $SNOWCHIVE_LISTEN, then 127.0.0.1:8080.") + root := flag.String("root", envOr("SNOWCHIVE_ROOT", "."), + "Archive directory holding data/ (the document corpus, one directory per document) and project/ "+ + "(the files about the archive as a whole: glossary.json, assemblies.json, changelog.json and related.json; only -rebuild-related writes there). "+ + "Defaults to $SNOWCHIVE_ROOT, then the working directory.") + check := flag.Bool("check", false, + "Load the corpus, print the warnings and the tally, then exit - non-zero when anything was reported. ") + noAPI := flag.Bool("no-api", envBool("SNOWCHIVE_NO_API"), + "Disable the read-only JSON API: every /api/ path answers a JSON 403 and the UI stops linking to it. Defaults on when $SNOWCHIVE_NO_API is true (1, t, true).") + flag.StringVar(&accelPrefix, "accel-files", os.Getenv("SNOWCHIVE_ACCEL_FILES"), + "Internal nginx location that aliases the data directory (e.g. /_files/). Document files are then answered with an X-Accel-Redirect into it and sent by nginx. "+ + "Only behind nginx. Defaults to $SNOWCHIVE_ACCEL_FILES.") + accelTTL := flag.Duration("accel-expires", envDuration("SNOWCHIVE_ACCEL_EXPIRES"), + "How long the proxy may cache a page served to a reader without cookies (e.g. 10m), sent as X-Accel-Expires; browsers keep the short max-age. "+ + "Clear the proxy cache after a restart with new data. Defaults to $SNOWCHIVE_ACCEL_EXPIRES.") + flag.Parse() + setRoot(*root) + if accelPrefix != "" { + if !strings.HasPrefix(accelPrefix, "/") || !strings.HasSuffix(accelPrefix, "/") { + log.Fatalf("-accel-files: %q must begin and end with a slash", accelPrefix) + } + } + if *accelTTL >= time.Second { + accelExpires = strconv.Itoa(int(*accelTTL / time.Second)) + } + if *baseFlag != "" { + b, err := parseCanonicalBase(*baseFlag) + if err != nil { + log.Fatalf("-base: %v", err) + } + canonicalBase = b + } + apiEnabled = !*noAPI + initAssets() + // Before the load: it pre-renders the big page bodies from these. + if !devMode { + if err := parseTemplates(); err != nil { + log.Fatalf("Fatal: Could not parse templates at startup: %v", err) + } + } + + loadData() + co := corpus() + + if *check { + if loadErrors > 0 || loadWarnings > 0 { + log.Printf("check failed: %d errors, %d warnings", loadErrors, loadWarnings) + os.Exit(1) + } + log.Printf("check passed: %d documents, nothing to report", len(co.archive.Docs)) + return + } + + if *rebuildRelated { + log.Println("Rebuilding related docs index...") + + existing := make(map[string][]string) + if b, err := os.ReadFile(relatedDocsPath()); err == nil { + if err := json.Unmarshal(b, &existing); err != nil { + warnf("could not parse existing related.json: %v - starting fresh", err) + existing = make(map[string][]string) + } else { + log.Printf("Loaded existing related.json with %d entries", len(existing)) + } + } + + stale := 0 + for id := range existing { + if co.unknownDoc(id) { + delete(existing, id) + stale++ + } + } + var missing []Document + for _, d := range co.archive.Docs { + if ids, ok := existing[d.ID]; !ok || slices.ContainsFunc(ids, co.unknownDoc) { + missing = append(missing, d) + } + } + + if len(missing) == 0 && stale == 0 { + log.Println("No new documents - related.json is up to date.") + return + } + + log.Printf("Computing related docs for %d documents...", len(missing)) + maps.Copy(existing, co.buildRelatedDocs(missing, co.archive.Docs)) + + if err := saveRelatedDocs(existing); err != nil { + log.Fatalf("Failed to save related docs: %v", err) + } + log.Printf("Saved related.json with %d entries (%d computed, %d stale dropped, other entries and manual edits preserved)", + len(existing), len(missing), stale) + return + } + + handler := withBaseHeaders(newMux()) + + srv := &http.Server{ + Addr: *listen, + Handler: handler, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: writeTimeout, // downloads set their own: fileWriteBudget + IdleTimeout: 60 * time.Second, + MaxHeaderBytes: 64 << 10, + } + + warnNonLoopback(*listen) + log.Printf("Archive Server listening on %s (data: %s, project: %s)", srv.Addr, dataDir, projectDir) + + errCh := make(chan error, 1) + go func() { errCh <- srv.ListenAndServe() }() + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM) + select { + case err := <-errCh: + log.Fatal(err) + case s := <-sig: + signal.Reset(os.Interrupt, syscall.SIGTERM) + log.Printf("%s received - draining connections (again to stop now)", s) + ctx, cancel := context.WithTimeout(context.Background(), writeTimeout) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { + log.Printf("Shutdown: %v", err) + } + } +} + +func (co *Corpus) prerenderPages() { + co.prerendered = map[string]template.HTML{} + co.docRowHTML = nil + if devMode { + return + } + exec := func(key, page, block string, data interface{}) { + t := tmplSet[page] + if t == nil { + return + } + var buf bytes.Buffer + if err := t.ExecuteTemplate(&buf, block, data); err != nil { + warnf("prerender %s/%s: %v - rendering it per request", page, block, err) + return + } + co.prerendered[key] = template.HTML(buf.String()) + } + + exec("ids", "ids.html", "id-rows", &idsPage{Rows: co.idRows}) + exec("archive-timeline", "browse.html", "archive-timeline", + &browsePage{ArchiveYears: co.archiveTimeline}) + exec("glossary", "glossary.html", "glossary-groups", + &glossaryPage{Groups: co.glossaryGroups}) + exec("articles", "articles.html", "article-groups", + &articlesPage{Groups: co.articleGroups}) + for key, v := range co.codewordViews { + exec("codewords:"+key, "codewords.html", "codeword-rows", + &codewordsPage{Rows: v.Rows}) + } + for set, tables := range co.exploreTables { + for i, t := range tables { + exec(exploreKey(set, i), "explore_table.html", "explore-rows", + &exploreTablePage{Rows: t.Rows}) + } + } + co.prerenderCategoryRows() + co.prerenderDocRows() + total := 0 + for _, h := range co.prerendered { + total += len(h) + } + log.Printf("Pre-rendered %d page bodies (%d KB)", len(co.prerendered), total/1024) +} + +func (co *Corpus) preHTML(key string) template.HTML { return co.prerendered[key] } + +func newMux() *http.ServeMux { + mux := http.NewServeMux() + mux.HandleFunc("GET /{$}", bind((*Corpus).handleIndex)) + mux.HandleFunc("GET /random", bind((*Corpus).handleRandom)) + mux.HandleFunc("GET /glossary", bind((*Corpus).handleGlossary)) + mux.HandleFunc("GET /preferences", handlePreferences) + mux.HandleFunc("POST /preferences/save", handlePreferencesSave) + mux.HandleFunc("GET /pinned", bind((*Corpus).handlePinnedPage)) + mux.HandleFunc("GET /pinned/export", bind((*Corpus).handlePinnedExport)) + mux.HandleFunc("POST /pin/{id}", bind((*Corpus).handlePinToggle)) + mux.HandleFunc("GET /articles", bind((*Corpus).handleArticles)) + mux.HandleFunc("GET /article/{key}", bind((*Corpus).handleArticleView)) + mux.HandleFunc("GET /changelog", bind((*Corpus).handleChangelog)) + mux.HandleFunc("POST /history/clear", handleHistoryClear) + mux.HandleFunc("POST /pinsearch", bind((*Corpus).handleSearchPinToggle)) + mux.HandleFunc("GET /robots.txt", handleRobots) + mux.HandleFunc("GET /sitemap.xml", bind((*Corpus).handleSitemap)) + mux.HandleFunc("/", notFound) + mux.HandleFunc("GET /browse", bind((*Corpus).handleBrowseRoot)) + mux.HandleFunc("GET /browse/{category}", bind((*Corpus).handleCategoryList)) + mux.HandleFunc("GET /explore", bind((*Corpus).handleExploreIndex)) + mux.HandleFunc("GET /explore/{metric}", bind((*Corpus).handleExploreMetric)) + mux.HandleFunc("GET /explore/terms", bind((*Corpus).handleCodewords)) + mux.HandleFunc("GET /explore/ids", bind((*Corpus).handleExploreIDs)) + mux.HandleFunc("GET /explore/ids.txt", bind((*Corpus).handleExploreIDsText)) + mux.HandleFunc("GET /explore/classification", bind((*Corpus).handleClassPatterns)) + mux.HandleFunc("GET /explore/portions", bind((*Corpus).handlePortions)) + mux.HandleFunc("GET /explore/outlets", bind((*Corpus).handleOutlets)) + mux.HandleFunc("GET /explore/assemblies", bind((*Corpus).handleAssemblies)) + mux.HandleFunc("GET /explore/assemblies/{id}", bind((*Corpus).handleAssemblyView)) + mux.HandleFunc("GET /browse/{category}/{value}", bind((*Corpus).handleCategoryFilter)) + mux.HandleFunc("GET /doc/{id}", bind((*Corpus).handleDocView)) + mux.HandleFunc("GET /doc/{id}/pages", bind((*Corpus).handleDocPages)) + mux.HandleFunc("GET /doc/{id}/versions/{slug}", bind((*Corpus).handleVersionView)) + mux.HandleFunc("GET /doc/{id}/versions/{slug}/pages", bind((*Corpus).handleVersionPages)) + mux.HandleFunc("GET /search", bind((*Corpus).handleSearch)) + mux.HandleFunc("GET /about", handleAbout) + mux.HandleFunc("GET /files/", bind((*Corpus).handleFiles)) + mux.HandleFunc("GET /static/", handleStatic) + + // JSON API (read-only) - see api.go for endpoint documentation. + if apiEnabled { + mux.HandleFunc("GET /api/docs", bind((*Corpus).handleAPIDocs)) + mux.HandleFunc("GET /api/doc/{id}", bind((*Corpus).handleAPIDoc)) + mux.HandleFunc("GET /api/search", bind((*Corpus).handleAPISearch)) + mux.HandleFunc("GET /api/facets", bind((*Corpus).handleAPIFacets)) + mux.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) { + apiError(w, http.StatusNotFound, "unknown API endpoint") + }) + } else { + mux.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) { + apiError(w, http.StatusForbidden, "the JSON API is disabled on this instance") + }) + } + + return mux +} + +// withBaseHeaders wraps a handler with the headers every response carries. +func withBaseHeaders(next http.Handler) http.Handler { + const csp = "default-src 'self'; img-src 'self'; script-src 'self'; " + + "style-src 'self' 'unsafe-inline'; object-src 'none'; base-uri 'none'; " + + "form-action 'self'; frame-ancestors 'self'" + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := w.Header() + h.Set("Content-Security-Policy", csp) + h.Set("X-Frame-Options", "SAMEORIGIN") + h.Set("X-Content-Type-Options", "nosniff") + h.Set("Referrer-Policy", "no-referrer") + switch r.Method { + case http.MethodGet, http.MethodHead, http.MethodPost: + default: + h.Set("Allow", "GET, HEAD, POST") + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + next.ServeHTTP(w, r) + }) +} + +func envOr(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func envBool(key string) bool { + v := os.Getenv(key) + if v == "" { + return false + } + b, err := strconv.ParseBool(v) + if err != nil { + log.Fatalf("$%s=%q is not a boolean (use 1/0 or true/false)", key, v) + } + return b +} + +func envDuration(key string) time.Duration { + v := os.Getenv(key) + if v == "" { + return 0 + } + d, err := time.ParseDuration(v) + if err != nil { + log.Fatalf("$%s=%q is not a duration (use e.g. 90s, 10m)", key, v) + } + return d +} + +// accelPath is the X-Accel-Redirect target for a file: its path relative to +// the data directory under accelPrefix, percent-encoded because nginx decodes +// the header as a URI. +func accelPath(file string) (string, bool) { + rel, err := filepath.Rel(dataDir, file) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", false + } + u := url.URL{Path: accelPrefix + filepath.ToSlash(rel)} + return u.EscapedPath(), true +} + +// handleFiles serves the files the loader indexed for a document, and no +// other file its directory may hold (Document.servedFile). +func (co *Corpus) handleFiles(w http.ResponseWriter, r *http.Request) { + rest := strings.TrimPrefix(r.URL.Path, "/files/") + id, sub, ok := strings.Cut(rest, "/") + if !ok || sub == "" { + http.NotFound(w, r) + return + } + doc := co.docIndex[id] + if doc == nil { + http.NotFound(w, r) + return + } + + sub = path.Clean("/" + sub) + rel := strings.TrimPrefix(sub, "/") + if strings.Contains(sub, "..") || !doc.servedFile(rel) { + http.NotFound(w, r) + return + } + + cacheIfFile(w, os.DirFS(doc.Dir), rel, "public, max-age=86400") + if isTranscript(rel) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + } + file := filepath.Join(doc.Dir, filepath.FromSlash(sub)) + if accelPrefix != "" { + if target, ok := accelPath(file); ok { + w.Header().Set("X-Accel-Redirect", target) + return + } + } + _ = http.NewResponseController(w).SetWriteDeadline(time.Now().Add(fileWriteBudget)) + http.ServeFile(w, r, file) +} + +const writeTimeout = 30 * time.Second + +// fileWriteBudget is how long a single download may take in total. A 123 MB +// PDF over a bad mobile link (~35 KB/s) fits in an hour, so in practice this +// bound is only reached by a reader that has gone away. +var fileWriteBudget = time.Hour + +func cacheIfFile(w http.ResponseWriter, fsys fs.FS, name, policy string) { + if fi, err := fs.Stat(fsys, name); err == nil && fi.Mode().IsRegular() { + w.Header().Set("Cache-Control", policy) + } +} + +func handleStatic(w http.ResponseWriter, r *http.Request) { + if path.Ext(r.URL.Path) == "" { + http.NotFound(w, r) + return + } + rel := strings.TrimPrefix(path.Clean("/"+strings.TrimPrefix(r.URL.Path, "/static/")), "/") + if h, ok := staticHashes[rel]; ok { + if r.URL.Query().Get("v") == h { + w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") + } + w.Header().Set("ETag", `"`+h+`"`) + } + http.StripPrefix("/static/", http.FileServerFS(staticFS)).ServeHTTP(w, r) +} + +func navSection(path string) string { + switch { + case path == "/": + return "home" + case path == "/browse" || strings.HasPrefix(path, "/browse/"): + return "browse" + case path == "/search": + return "search" + case path == "/articles" || strings.HasPrefix(path, "/article/"): + return "articles" + case path == "/glossary": + return "glossary" + case path == "/about": + return "about" + case path == "/pinned": + return "pinned" + case path == "/preferences": + return "preferences" + } + return "" +} + +func render(w http.ResponseWriter, r *http.Request, data pageData) { + renderCode(w, r, data, http.StatusOK) +} + +func crossSitePOST(r *http.Request) bool { + switch r.Header.Get("Sec-Fetch-Site") { + case "same-origin", "none": + return false + case "cross-site", "same-site": + return true + } + + if o := r.Header.Get("Origin"); o != "" && o != "null" { + if u, err := url.Parse(o); err == nil && !originIsHost(u, r.Host) { + return true + } + } + return false +} + +func originIsHost(origin *url.URL, host string) bool { + def := map[string]string{"http": "80", "https": "443"}[strings.ToLower(origin.Scheme)] + op := origin.Port() + if op == "" { + op = def + } + h, hp, err := net.SplitHostPort(host) + if err != nil { // no port + h, hp = strings.Trim(host, "[]"), def + } + return strings.EqualFold(origin.Hostname(), h) && op == hp +} + +func noStore(w http.ResponseWriter) { + w.Header().Set("Cache-Control", "private, no-store") +} + +func readPOSTForm(w http.ResponseWriter, r *http.Request) bool { + if crossSitePOST(r) { + http.Error(w, "cross-site request rejected", http.StatusForbidden) + return false + } + r.Body = http.MaxBytesReader(w, r.Body, 1<<16) + if err := r.ParseForm(); err != nil { + http.Error(w, "bad form", http.StatusBadRequest) + return false + } + return true +} + +func crossSiteFetch(r *http.Request) bool { + switch r.Header.Get("Sec-Fetch-Site") { + case "cross-site", "same-site": + return true + } + return false +} + +func renderCode(w http.ResponseWriter, r *http.Request, data pageData, code int) { + name := data.templateName() + if w.Header().Get("Cache-Control") == "" { + if (r.Method == http.MethodGet || r.Method == http.MethodHead) && + (code == http.StatusOK || code == http.StatusNotFound) && + r.Header.Get("Cookie") == "" { + w.Header().Set("Cache-Control", "public, max-age=10") + w.Header().Set("Vary", "Cookie") + if accelExpires != "" { + w.Header().Set("X-Accel-Expires", accelExpires) + } + } else { + w.Header().Set("Cache-Control", "private, no-cache") + } + } + + b := data.base() + b.Nav = navSection(r.URL.Path) + b.APIEnabled = apiEnabled + // A handler that read the reader's preferences passes them on + // (pageWith); they are only read here for one that had no use for them. + if !b.hasPrefs { + b.Prefs = readPrefs(r) + } + b.Theme = b.Prefs.ThemeAttr() + b.CardPagePreview = b.Prefs.CardPagePreview + + var t *template.Template + var err error + + if devMode { + t, err = parsePage(name) + if err != nil { + log.Printf("Template Parse Error (%s): %v", name, err) + serverError(w, err) + return + } + } else { + t = tmplSet[name] + if t == nil { + log.Printf("Template not found: %s", name) + serverError(w, fmt.Errorf("template not found: %s", name)) + return + } + } + + var buf bytes.Buffer + if err := t.ExecuteTemplate(&buf, name, data); err != nil { + log.Printf("Template Exec Error (%s): %v", name, err) + serverError(w, err) + return + } + if code != http.StatusOK { + w.WriteHeader(code) + } + w.Write(bytes.TrimSpace(buf.Bytes())) + w.Write([]byte("\n")) +} + +func notFound(w http.ResponseWriter, r *http.Request) { + renderCode(w, r, ¬FoundPage{Path: r.URL.Path}, http.StatusNotFound) +} + +func parsePage(name string) (*template.Template, error) { + return template.New(name).Funcs(funcMap).ParseFS(templateFS, "base.html", "partials.html", name) +} + +func parseTemplates() error { + pages, err := fs.Glob(templateFS, "*.html") + if err != nil { + return err + } + tmplSet = make(map[string]*template.Template) + for _, name := range pages { + if name == "base.html" || name == "partials.html" { + continue + } + t, err := parsePage(name) + if err != nil { + return fmt.Errorf("%s: %w", name, err) + } + tmplSet[name] = t + } + return nil +} + +func serverError(w http.ResponseWriter, err error) { + w.Header().Set("Cache-Control", "no-store") + if devMode { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + http.Error(w, "Internal Server Error", http.StatusInternalServerError) +} + +func warnNonLoopback(addr string) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return + } + if host == "localhost" { + return + } + if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() { + return + } + warnf("-listen %s is not loopback - anything that can route to this host reaches the archive directly, bypassing your proxy's caching, rate limits and logs.", addr) +} diff --git a/mdparser.go b/mdparser.go new file mode 100644 index 0000000..a7b82c4 --- /dev/null +++ b/mdparser.go @@ -0,0 +1,591 @@ +// mdparser.go - minimal dependency-free Markdown parser. +// +// Input trust: everything rendered through this parser is the archive's own +// curator-produced OCR never user-supplied input. + +package main + +import ( + "fmt" + "html" + "regexp" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// -- Inline patterns ---------------------------------------------------- + +var ( + reInlineCode = regexp.MustCompile("`(.+?)`") + reStrikethrough = regexp.MustCompile(`~~(.+?)~~`) + reHardBreak = regexp.MustCompile(`(?i)<br\s*/?>`) + reEscape = regexp.MustCompile("\\\\([!-/:-@\\[-`|{}~])") + reOrderedItem = regexp.MustCompile(`^(\s*)(\d{1,9})\.(?:\s+(.*))?\s*$`) + reUnorderedItem = regexp.MustCompile(`^(\s*)[-*+](?:\s+(.*))?\s*$`) + reTableSep = regexp.MustCompile(`^\|[-| :]+\|$`) +) + +// parseInline converts inline Markdown to HTML. +func parseInline(s string) string { + s = strings.ReplaceAll(s, "\x00", "") + s = strings.ReplaceAll(s, "\x01", "") + s = html.EscapeString(s) + + // Backslash escapes (CommonMark): "\*" is a literal "*", not emphasis + // syntax. Pull escaped ASCII punctuation out to placeholders BEFORE the + // emphasis passes so the character can't be interpreted as a marker, + // then restore it at the end. + var escaped []string + if strings.IndexByte(s, '\\') >= 0 { + s = reEscape.ReplaceAllStringFunc(s, func(m string) string { + // m is "\X": EscapeString leaves the backslash alone. Keep + // the char after it. + ch := m[len(m)-1:] + escaped = append(escaped, ch) + return "\x01" + strconv.Itoa(len(escaped)-1) + "\x01" + }) + } + + // Extract inline code spans next and replace them with placeholders so + // the strikethrough/bold/italic passes can't reformat code content + // (e.g. `*x*` must render as literal *x*, not x). Escape + // placeholders that landed INSIDE a code span are put back as their + // original "\X" source text first: backslash escapes are not escapes + // within code spans (CommonMark), so `\*` renders as \*, not *. The + // escape pass still runs before this one on purpose - that ordering is + // what lets \` be a literal backtick that cannot OPEN a code span. + var codes []string + if strings.IndexByte(s, '`') >= 0 { + s = reInlineCode.ReplaceAllStringFunc(s, func(m string) string { + inner := reInlineCode.FindStringSubmatch(m)[1] + for i, ch := range escaped { + inner = strings.ReplaceAll(inner, "\x01"+strconv.Itoa(i)+"\x01", "\\"+ch) + } + codes = append(codes, ""+inner+"") + return "\x00" + strconv.Itoa(len(codes)-1) + "\x00" + }) + } + + // After the code-span pass, so `
    ` inside backticks stays literal. + if strings.Contains(s, "<") { + s = reHardBreak.ReplaceAllString(s, "
    ") + } + if strings.Contains(s, "~~") { + s = reStrikethrough.ReplaceAllString(s, `$1`) + } + // ***x*** first, so it nests as rather than crossing. + s = emphasize(s, '*', 3, "", "") + s = emphasize(s, '*', 2, "", "") + s = emphasize(s, '_', 2, "", "") + s = emphasize(s, '*', 1, "", "") + s = emphasize(s, '_', 1, "", "") + + // Restore code spans. + for i, code := range codes { + s = strings.Replace(s, "\x00"+strconv.Itoa(i)+"\x00", code, 1) + } + // Restore backslash-escaped characters as their literal selves. + // (Placeholders that sat inside a code span were already restored to + // their "\X" source form up in the code-extraction pass and no longer + // occur here; Replace on an absent placeholder is a no-op.) + for i, ch := range escaped { + s = strings.Replace(s, "\x01"+strconv.Itoa(i)+"\x01", ch, 1) + } + return s +} + +// emphasize wraps the text between two runs of k delimiters c ('*' or '_') +// in open/close. It pairs them as CommonMark does: left to right, a run +// that can close gives its first k delimiters to the nearest unpaired +// opener before it, and a run that can open offers its last k to a later +// closer. Which runs can open or close follows CommonMark's flanking rules +// (delimiterRun). +func emphasize(s string, c byte, k int, open, close string) string { + var openers []int // offsets of unpaired openers, nearest last + tags := map[int]string{} // offset of k paired delimiters -> their tag + for i := 0; i < len(s); { + if s[i] != c { + i++ + continue + } + end := i + for end < len(s) && s[end] == c { + end++ + } + canOpen, canClose := delimiterRun(s, i, end, c) + n := end - i + if n >= k && canClose && len(openers) > 0 { + tags[openers[len(openers)-1]], tags[i] = open, close + openers = openers[:len(openers)-1] + n -= k + } + if n >= k && canOpen { + openers = append(openers, end-k) + } + i = end + } + if len(tags) == 0 { + return s + } + var b strings.Builder + for i := 0; i < len(s); i++ { + if t, ok := tags[i]; ok { + b.WriteString(t) + i += k - 1 + } else { + b.WriteByte(s[i]) + } + } + return b.String() +} + +// delimiterRun reports whether the run of delimiters c at s[start:end] +// can open and can close emphasis, by CommonMark's flanking rules, which +// look at the whole run. +func delimiterRun(s string, start, end int, c byte) (canOpen, canClose bool) { + prev, next := ' ', ' ' + if start > 0 { + prev, _ = utf8.DecodeLastRuneInString(s[:start]) + } + if end < len(s) { + next, _ = utf8.DecodeRuneInString(s[end:]) + } + left := !unicode.IsSpace(next) && + (!isPunct(next) || unicode.IsSpace(prev) || isPunct(prev)) + right := !unicode.IsSpace(prev) && + (!isPunct(prev) || unicode.IsSpace(next) || isPunct(next)) + if c == '_' { + return left && (!right || isPunct(prev)), right && (!left || isPunct(next)) + } + return left, right +} + +// isPunct is CommonMark's punctuation (Unicode P and S). The \x00/\x01 +// placeholders count too: they stand for a code span (opened by a +// backtick) or a backslash-escaped punctuation character. +func isPunct(r rune) bool { + return r == 0 || r == 1 || unicode.IsPunct(r) || unicode.IsSymbol(r) +} + +// -- Block-level parser -------------------------------------------- + +type parser struct { + lines []string + pos int + out strings.Builder + open []listLevel // the lists parseList is inside, outermost first + // numbered holds the last number of the ordered list at each indent, + // for as long as list lines keep coming (see endsItem). + numbered map[int]int + // top is the source's highest heading level and last the level of the + // heading emitted last (see parseHeading). + top, last int +} + +// listLevel is one open list: the indent of its items and their kind. +type listLevel struct { + indent int + ordered bool +} + +func newParser(src string) *parser { + src = strings.ReplaceAll(src, "\r\n", "\n") + src = strings.ReplaceAll(src, "\r", "\n") + p := &parser{lines: strings.Split(src, "\n"), top: 7, last: 1} + fence := "" + for _, line := range p.lines { + switch { + case fence != "": + if strings.HasPrefix(line, fence) { + fence = "" + } + case isFencedCodeStart(line): + fence = line[:3] + case isHeading(line): + p.top = min(p.top, headingLevel(line)) + } + } + return p +} + +func (p *parser) peek() (string, bool) { + if p.pos >= len(p.lines) { + return "", false + } + return p.lines[p.pos], true +} + +func (p *parser) next() (string, bool) { + l, ok := p.peek() + if ok { + p.pos++ + } + return l, ok +} + +func (p *parser) emit(s string) { p.out.WriteString(s) } + +// parse processes all lines and returns HTML. +func (p *parser) parse() string { + for p.pos < len(p.lines) { + line, _ := p.peek() + if !isListItem(line) { + p.numbered = nil // the run of list lines is over + } + + switch { + case isFencedCodeStart(line): + p.parseFencedCode() + case isHR(line): + p.next() + p.emit("
    \n") + case isHeading(line): + p.parseHeading() + case isTableRow(line): + p.parseTable() + case isListItem(line): + p.parseList(reOrderedItem.MatchString(line), indentOf(line)) + case strings.TrimSpace(line) == "": + p.next() + default: + p.parseParagraph() + } + } + return p.out.String() +} + +// -- Heading -------------------------------------------------- + +// headingLevel returns the heading level of an ATX heading line - one to +// six '#' followed by whitespace or the end of the line - or 0 when the +// line is not a heading. +func headingLevel(s string) int { + n := 0 + for n < len(s) && s[n] == '#' { + n++ + } + if n == 0 || n > 6 || (n < len(s) && s[n] != ' ' && s[n] != '\t') { + return 0 + } + return n +} + +func isHeading(s string) bool { return headingLevel(s) > 0 } + +// parseHeading emits a heading under the page's own

    : the source's top +// level becomes

    and no heading skips a level. The md-hN class keeps the +// size of the level the source wrote. +func (p *parser) parseHeading() { + line, _ := p.next() + level := headingLevel(line) + content := strings.TrimSpace(line[level:]) + tag := min(level-p.top+2, p.last+1, 6) + p.last = tag + p.emit(fmt.Sprintf("%s\n", tag, level, parseInline(content), tag)) +} + +// -- Horizontal rule ------------------------------------------------- + +// isHR reports whether s is a thematic break: three or more of the same +// '-', '*' or '_', optionally with spaces or tabs between them. +func isHR(s string) bool { + t := strings.TrimSpace(s) + if t == "" || (t[0] != '-' && t[0] != '*' && t[0] != '_') { + return false + } + n := 0 + for i := 0; i < len(t); i++ { + switch t[i] { + case t[0]: + n++ + case ' ', '\t': + default: + return false + } + } + return n >= 3 +} + +// Fenced code block --------------------------------------------------------- + +func isFencedCodeStart(s string) bool { + return strings.HasPrefix(s, "```") || strings.HasPrefix(s, "~~~") +} + +func (p *parser) parseFencedCode() { + opener, _ := p.next() + fence := opener[:3] + + var body strings.Builder + for { + line, ok := p.next() + if !ok || strings.HasPrefix(line, fence) { + break + } + body.WriteString(html.EscapeString(line)) + body.WriteByte('\n') + } + p.emit("
    " + body.String() + "
    \n") +} + +// -- Lists (unified, handles nesting) ----------------------------------------- + +// indentOf returns the number of leading whitespace characters in s. +func indentOf(s string) int { + return len(s) - len(strings.TrimLeft(s, " \t")) +} + +// isListItem returns true if the line is a ul or ol item at any indent. +// A thematic break wins over a bullet ("* * *" is a rule, not an item). +func isListItem(s string) bool { + // A list marker is the first non-space byte: rule out the rest (most + // lines) before any regex runs. + t := strings.TrimLeft(s, "\t\n\f\r ") + if t == "" || (t[0] != '-' && t[0] != '*' && t[0] != '+' && (t[0] < '0' || t[0] > '9')) { + return false + } + return !isHR(s) && (reUnorderedItem.MatchString(s) || reOrderedItem.MatchString(s)) +} + +// itemContent strips the list marker and returns the item text (possibly +// empty for a bare marker like "6."). +func itemContent(s string) string { + if sub := reUnorderedItem.FindStringSubmatch(s); sub != nil { + return sub[2] + } + if sub := reOrderedItem.FindStringSubmatch(s); sub != nil { + return sub[3] + } + return strings.TrimSpace(s) +} + +// orderedNum returns the source number of an ordered-list line ("7." -> 7) +// and true, or false if the line isn't an ordered item. "0." is a real +// number, so it cannot double as the "not ordered" answer. +func orderedNum(s string) (int, bool) { + if sub := reOrderedItem.FindStringSubmatch(s); sub != nil { + n, _ := strconv.Atoi(sub[2]) // at most nine digits: cannot fail + return n, true + } + return 0, false +} + +// startsList reports whether list-item line s may open a new list on the +// line after text, where it would otherwise continue that text. +func startsList(s string) bool { + if itemContent(s) == "" { + return false + } + n, ok := orderedNum(s) + return !ok || n == 1 +} + +// endsItem reports whether line s, met while collecting the lazily +// continued text of a list item, ends that item rather than continuing it. +func (p *parser) endsItem(s string) bool { + if isHR(s) || isHeading(s) || isFencedCodeStart(s) || isTableRow(s) { + return true + } + if !isListItem(s) { + return false + } + if startsList(s) { + return true + } + indent, ordered := indentOf(s), reOrderedItem.MatchString(s) + for _, l := range p.open { + if l.indent == indent && l.ordered == ordered { + return true + } + } + if n, ok := orderedNum(s); ok { + for _, last := range p.numbered { + if n > last && n <= last+2 { + return true + } + } + } + return false +} + +// parseList recursively parses a ul or ol at the given indent level. +func (p *parser) parseList(ordered bool, minIndent int) { + tag := "ul" + if ordered { + tag = "ol" + } + p.emit("<" + tag + ">\n") + p.open = append(p.open, listLevel{minIndent, ordered}) + defer func() { p.open = p.open[:len(p.open)-1] }() + + expected := 0 // running ordered-list counter, for value="N" decisions + + for { + line, ok := p.peek() + if !ok || strings.TrimSpace(line) == "" || !isListItem(line) { + break + } + indent := indentOf(line) + if indent < minIndent { + // Belongs to a parent list - stop. + break + } + if indent == minIndent && reOrderedItem.MatchString(line) != ordered { + // Bullets after numbers, or numbers after bullets: a new + // list (as in CommonMark). Kept in this one, a number would + // lose itself to a bullet and a bullet would gain a number. + break + } + if indent > minIndent { + // Deeper indent mid-list without a parent
  • - treat as nested. + p.parseList(reOrderedItem.MatchString(line), indent) + continue + } + + p.next() + + // Item body: the marker line plus any lazily-continued lines after it. + var body []string + if c := itemContent(line); c != "" { + body = append(body, c) + } + for { + cont, ok := p.peek() + if !ok || strings.TrimSpace(cont) == "" || p.endsItem(cont) { + break + } + p.next() + body = append(body, strings.TrimSpace(cont)) + } + + liOpen := "
  • " + if ordered { + n, _ := orderedNum(line) + expected++ + if n != expected { + liOpen = "
  • " + expected = n + } + if p.numbered == nil { + p.numbered = map[int]int{} + } + p.numbered[minIndent] = n + } + + var parts []string + for _, l := range body { + parts = append(parts, parseInline(l)) + } + text := strings.Join(parts, "
    \n") + + // Peek: if next line is deeper, nest a list inside this
  • - + // as many as follow, since bullets turning into numbers down + // there start a second nested list, which belongs here too. + next, hasNext := p.peek() + if hasNext && isListItem(next) && indentOf(next) > indent { + p.emit(liOpen + text + "\n") + for hasNext && isListItem(next) && indentOf(next) > indent { + p.parseList(reOrderedItem.MatchString(next), indentOf(next)) + next, hasNext = p.peek() + } + p.emit("
  • \n") + } else { + p.emit(liOpen + text + "

  • \n") + } + } + + p.emit("\n") +} + +// -- Table --------------------------------------------------------------------- + +func isTableRow(s string) bool { + t := strings.TrimSpace(s) + return len(t) >= 3 && t[0] == '|' && t[len(t)-1] == '|' && !strings.Contains(t, "\n") +} + +// splitCells splits a pipe-delimited row into trimmed cell strings. +func splitCells(row string) []string { + row = strings.TrimPrefix(strings.TrimSpace(row), "|") + if strings.HasSuffix(row, "|") && !strings.HasSuffix(row, `\|`) { + row = row[:len(row)-1] + } + var cells []string + var cell strings.Builder + for i := 0; i < len(row); i++ { + switch { + case row[i] == '\\' && i+1 < len(row) && row[i+1] == '|': + cell.WriteByte('|') + i++ + case row[i] == '|': + cells = append(cells, strings.TrimSpace(cell.String())) + cell.Reset() + default: + cell.WriteByte(row[i]) + } + } + return append(cells, strings.TrimSpace(cell.String())) +} + +func (p *parser) parseTable() { + headerLine, _ := p.next() + headers := splitCells(headerLine) + + // Consume separator row (|---|---|) if present. + if sep, ok := p.peek(); ok && reTableSep.MatchString(strings.TrimSpace(sep)) { + p.next() + } + + p.emit("
    \n\n\n\n") + for _, h := range headers { + p.emit("\n") + } + p.emit("\n\n\n") + + for { + line, ok := p.peek() + if !ok || !isTableRow(line) { + break + } + p.next() + p.emit("\n") + for _, cell := range splitCells(line) { + p.emit("\n") + } + p.emit("\n") + } + + p.emit("\n
    " + parseInline(h) + "
    " + parseInline(cell) + "
    \n
    \n") +} + +// -- Paragraph ----------------------------------------------------------------- + +func (p *parser) parseParagraph() { + var lines []string + for { + line, ok := p.peek() + if !ok || strings.TrimSpace(line) == "" { + break + } + if isHeading(line) || isFencedCodeStart(line) || isHR(line) || + (isListItem(line) && startsList(line)) || isTableRow(line) { + break + } + p.next() + lines = append(lines, strings.TrimSpace(line)) + } + if len(lines) > 0 { + var parts []string + for _, l := range lines { + parts = append(parts, parseInline(l)) + } + p.emit("

    " + strings.Join(parts, "
    \n") + "

    \n") + } +} + +// Render converts Markdown src to an HTML string. +func Render(src string) string { + return newParser(src).parse() +} diff --git a/outlets.go b/outlets.go new file mode 100644 index 0000000..bc43d8d --- /dev/null +++ b/outlets.go @@ -0,0 +1,132 @@ +// outlets.go - /explore/outlets: publication activity per media outlet and +// year. + +package main + +import ( + "net/http" + "sort" +) + +type outletCell struct { + Docs int + Articles int + URL string +} + +type outletRow struct { + Name string + URL string + Total int + Cells []outletCell +} + +type outletTable struct { + Years []string + YearLinks []facetEntry + Rows []outletRow + Totals []int +} + +func (co *Corpus) buildOutletTables() { + co.outletTables = map[string]outletTable{ + "": co.buildOutletTable(""), + collectionNews: co.buildOutletTable(collectionNews), + } +} + +func (co *Corpus) buildOutletTable(coll string) outletTable { + type key struct{ outlet, year string } + docs := map[key]int{} + stories := map[key]int{} + totals := map[string]int{} + yearSet := map[string]bool{} + display := map[string]string{} + canon := func(name string) string { + k := valueKey(name) + if d, ok := display[k]; ok { + return d + } + display[k] = name + return name + } + + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + if coll != "" && d.Collection != coll { + continue + } + for _, pub := range facetValues(d, "PU") { + totals[canon(pub)]++ + } + // A document counts for a publisher in the years of THAT publisher's + // own entries: another outlet's earlier publication does not make it + // this outlet's document of that year (publishedBy, the cell's page). + counted := map[key]bool{} + for _, p := range d.Publication { + if p.ReportingOnly || p.Publisher == "" || len(p.DatePublished) < 4 { + continue + } + k := key{canon(p.Publisher), p.DatePublished[:4]} + if !counted[k] { + counted[k] = true + docs[k]++ + yearSet[k.year] = true + } + } + } + for i := range co.articles { + a := &co.articles[i] + if a.Title == "" || a.Publisher == "" || len(a.Date) < 4 { + continue + } + // A story counts in a collection's table when one of its documents + // is in that collection. + if coll != "" && !a.inCollection(co, coll) { + continue + } + stories[key{canon(a.Publisher), a.Date[:4]}]++ + } + + years := make([]string, 0, len(yearSet)) + for y := range yearSet { + years = append(years, y) + } + sort.Strings(years) + + names := rankCounts(totals, -1) + + scope := wholeArchive + if coll != "" { + scope = scopeOf(map[string]bool{coll: true}) + } + t := outletTable{Years: years, Totals: make([]int, len(years))} + for _, y := range years { + t.YearLinks = append(t.YearLinks, facetEntry{Name: y, URL: scope.link(categoryValueURL("published", y))}) + } + for _, name := range names { + row := outletRow{Name: name, URL: scope.link(categoryValueURL("publisher", name)), Total: totals[name]} + for yi, y := range years { + k := key{name, y} + cell := outletCell{Docs: docs[k], Articles: stories[k]} + if cell.Docs > 0 { + cell.URL = scope.link(publisherYearURL(name, y)) + } + t.Totals[yi] += cell.Docs + row.Cells = append(row.Cells, cell) + } + t.Rows = append(t.Rows, row) + } + return t +} + +func (co *Corpus) handleOutlets(w http.ResponseWriter, r *http.Request) { + coll := "" + if r.URL.Query().Get("coll") == collectionNews { + coll = collectionNews + } + render(w, r, &outletsPage{ + Table: co.outletTables[coll], + NewsOnly: coll == collectionNews, + }) +} diff --git a/pages.go b/pages.go new file mode 100644 index 0000000..9c591d5 --- /dev/null +++ b/pages.go @@ -0,0 +1,393 @@ +package main + +import ( + "html/template" + "net/url" +) + +// pageData is a page struct, it embeds Base and names its template. +type pageData interface { + base() *Base + templateName() string +} + +// Base is what base.html reads. renderCode fills in everything but the +// category boxes. +type Base struct { + Nav string + APIEnabled bool + Theme string + CardPagePreview bool + Prefs Prefs + hasPrefs bool + ShowCatBoxes bool + CatCounts map[string]int + ActiveCategory string +} + +func (b *Base) base() *Base { return b } + +func pageWith(prefs Prefs) Base { return Base{Prefs: prefs, hasPrefs: true} } + +type cardView struct { + Docs []*Document + ShowThumbs bool + CardPagePreview bool + HideClassification bool + HideJournalists bool + HidePeople bool + HidePrograms bool + HideSummary bool + FOIAQS string // see Prefs.foiaLinkQS + ShowUnpin bool + ActiveCategory string + ActiveValue string + MarkReported bool // the "reported" badge on individually reported SIDtoday issues +} + +// listView is a page of documents as cards or as a table +type listView struct { + Docs []*Document + Total int + Page int + TotalPages int + View string + Sort string + Order string + DefaultSort string + BaseURL string + SortURL string + TableHeads []sortHead + RowsHTML template.HTML // the table view's rows, pre-rendered + Cards cardView + // MarkReported badges the individually reported SIDtoday issues, which + // only the SIDtoday tab tells apart from the bulk release. + MarkReported bool +} + +// docTableRow is a row of the document table +type docTableRow struct { + *Document + MarkReported bool +} + +func (lv listView) TableRows() []docTableRow { + rows := make([]docTableRow, len(lv.Docs)) + for i, d := range lv.Docs { + rows[i] = docTableRow{d, lv.MarkReported} + } + return rows +} + +type viewer struct { + Images []string + Sizes []imageSize + ImageBase string + Content template.HTML + Version *DocVersion +} + +func (v viewer) ImageSize(i int) template.HTMLAttr { + if i < len(v.Sizes) { + return v.Sizes[i].Attr() + } + return "" +} + +type notFoundPage struct { + Base + Path string +} + +type aboutPage struct{ Base } + +type indexPage struct { + Base + homeFigures + TrackRecent bool + RecentDocs []*Document + SnowdenTotal int + ArchiveTotal int + FOIATotal int + IncludeFOIA bool + ReportedTotal int + WithSIDtoday bool // the figures include the SIDtoday archive (/?sidtoday=1) +} + +// browsePage is /browse +type browsePage struct { + Base + listView + Collection string + CollectionCounts map[string]int + SIDTodayCount int + SelectedTypes map[string]bool + AllTypes map[string]int + Params url.Values + SeriesList []categoryEntry + ArchiveYears []archiveYear + TimelineHTML template.HTML +} + +// categoryPage is /browse/{category} +type categoryPage struct { + Base + Title string + Blurb string + Heads []sortHead + RowsHTML template.HTML + Entries []categoryEntry + CollCols bool + ShowCollCols bool + ShowAZ bool + Letters []string + SeeAlso []seeAlsoLink +} + +// listPage is /browse/{category}/{value} +type listPage struct { + Base + listView + Title string + CatLabel string + Blurb string + SearchLink string + SelectedTypes map[string]bool + AllTypes map[string]int + Params url.Values + GlossaryEntry *GlossaryEntry + Refinements []searchFacet + Breakdowns []bdGroup + Timeline []timelineBar + PubTimeline []timelineBar + YearScope string + YearScopeAllURL string + MonthNav []monthNavEntry + MonthNavYear string + MonthNavYearURL string + MonthNavYearActive bool + MonthlessCount int +} + +type docPage struct { + Base + viewer + Doc *Document + BrowseQS string + OGImageURL string + DocAbsURL string + Embeds docEmbeds + SeriesGroup *seriesGroup + Assemblies []*Assembly + SerialPrev *Document + SerialNext *Document + PublishedTogether []*Document + PublishedShown int + PublishedMore []*Article + PublishedCards cardView + RelatedDocs []*Document + RelatedCards cardView + Pinned bool +} + +type versionPage struct { + Base + viewer + Doc *Document + Publications []Publication + PDFURL string +} + +// pagesPage is the page-image grid of a document or of one of its versions. +type pagesPage struct { + Base + viewer + Doc *Document + TextOnly bool +} + +type articlesPage struct { + Base + Groups []articleYearGroup + GroupsHTML template.HTML + Total int + Books int +} + +type articlePage struct { + Base + Article Article + Total int + BrowseQS string + PublisherURL string + Cards cardView +} + +type assembliesPage struct { + Base + Assemblies []*Assembly + TextOnly bool +} + +type assemblyPage struct { + Base + Assembly *Assembly + TextOnly bool +} + +type changelogPage struct { + Base + Entries []ChangeEntry + Total int +} + +type codewordsPage struct { + Base + Rows []codewordRow + RowsHTML template.HTML + Total int + Shown int + Heads []sortHead +} + +type explorePage struct { + Base + Metrics []exploreMetric + HasAssemblies bool +} + +type exploreTablePage struct { + Base + Metric *exploreMetric + Rows []exploreRow + RowsHTML template.HTML + Total int + Shown int + Heads []sortHead +} + +type classPatternsPage struct { + Base + Cols []facetEntry + Levels []string + Strips []clsYearRow + Sparse []clsYearRow + MinMarks int + HeatRows []heatRow +} + +type idsPage struct { + Base + Rows []idRow + RowsHTML template.HTML + Shown int +} + +type glossaryPage struct { + Base + Groups []glossaryGroup + GroupsHTML template.HTML + Total int +} + +type outletsPage struct { + Base + Table outletTable + NewsOnly bool +} + +type portionsPage struct { + Base + Title string + Stats portionSummary + RowLimit int +} + +type pinnedPage struct { + Base + Total int + Searches []recentSearch + Full bool // a pin was refused: the cookie is full + Cards cardView +} + +type preferencesPage struct { + Base + MaxCards int + Saved bool + Reset bool +} + +type searchPage struct { + Base + FieldOptions []searchField + Facets []searchFacet + Queries []string + Fields []string + Conjs []string + Results []searchResult + SearchTypeCounts map[string]int + SelTypeSet map[string]bool + FormParams url.Values + Savable bool + ShowThumbs bool + FOIAQS string + Total int + Page int + TotalPages int + RecentSearches []recentSearch + SelColl map[string]bool + CollFiltered bool + CanonicalQS string + SearchPinned bool + Searched bool + QueryLabel string + DateDocStart string + DateDocEnd string + DatePubStart string + DatePubEnd string + Sort string + Order string + SearchBaseURL string + DateHist []timelineBar + SnippetsCapped bool + SnippetLimit int + OverClauses int + ClauseLimit int + OverWords int + WordLimit int +} + +func (*notFoundPage) templateName() string { return "404.html" } +func (*aboutPage) templateName() string { return "about.html" } +func (*indexPage) templateName() string { return "index.html" } +func (*browsePage) templateName() string { return "browse.html" } +func (*categoryPage) templateName() string { return "category.html" } +func (*listPage) templateName() string { return "list.html" } +func (*docPage) templateName() string { return "doc.html" } +func (*versionPage) templateName() string { return "version.html" } +func (*pagesPage) templateName() string { return "pages.html" } +func (*articlesPage) templateName() string { return "articles.html" } +func (*articlePage) templateName() string { return "article.html" } +func (*assembliesPage) templateName() string { return "assemblies.html" } +func (*assemblyPage) templateName() string { return "assembly.html" } +func (*changelogPage) templateName() string { return "changelog.html" } +func (*codewordsPage) templateName() string { return "codewords.html" } +func (*explorePage) templateName() string { return "explore.html" } +func (*exploreTablePage) templateName() string { return "explore_table.html" } +func (*classPatternsPage) templateName() string { return "classpatterns.html" } +func (*idsPage) templateName() string { return "ids.html" } +func (*glossaryPage) templateName() string { return "glossary.html" } +func (*outletsPage) templateName() string { return "outlets.html" } +func (*portionsPage) templateName() string { return "portions.html" } +func (*pinnedPage) templateName() string { return "pinned.html" } +func (*preferencesPage) templateName() string { return "preferences.html" } +func (*searchPage) templateName() string { return "search.html" } + +var pageTypes = []pageData{ + ¬FoundPage{}, &aboutPage{}, &indexPage{}, &browsePage{}, &categoryPage{}, &listPage{}, + &docPage{}, &versionPage{}, &pagesPage{}, &articlesPage{}, &articlePage{}, &assembliesPage{}, + &assemblyPage{}, &changelogPage{}, &codewordsPage{}, &explorePage{}, &exploreTablePage{}, + &classPatternsPage{}, &idsPage{}, &glossaryPage{}, &outletsPage{}, &portionsPage{}, + &pinnedPage{}, &preferencesPage{}, &searchPage{}, +} diff --git a/pins.go b/pins.go new file mode 100644 index 0000000..4519196 --- /dev/null +++ b/pins.go @@ -0,0 +1,145 @@ +// pins.go - pinned documents (a reading list). Cookie-local. + +package main + +import ( + "net/http" + "net/url" + "slices" +) + +const ( + pinnedCookie = "pinned_docs" + pinnedMax = 200 + pinnedSearchesCookie = "pinned_searches" + pinnedSearchesMax = 30 +) + +func (co *Corpus) readPinned(r *http.Request) []string { + var out []string + for _, id := range readIDListCookie(r, pinnedCookie, pinnedMax) { + if co.docIndex[id] != nil { + out = append(out, id) + } + } + return out +} + +func toggleEntry(list []string, e string, max int) []string { + var next []string + removed := false + for _, x := range list { + if x == e { + removed = true + continue + } + next = append(next, x) + } + if !removed && len(next) < max { + next = append(next, e) + } + return next +} + +func isPinned(r *http.Request, id string) bool { + return slices.Contains(readIDListCookie(r, pinnedCookie, pinnedMax), id) +} + +func (co *Corpus) handlePinToggle(w http.ResponseWriter, r *http.Request) { + if !readPOSTForm(w, r) { + return + } + id := r.PathValue("id") + if co.docIndex[id] == nil { + notFound(w, r) + return + } + + pinned := co.readPinned(r) + stored := writeListCookie(w, pinnedCookie, toggleEntry(pinned, id, pinnedMax)) + if pinRefused(pinned, stored, id) { + http.Redirect(w, r, "/pinned?full=1", http.StatusSeeOther) + return + } + + back := safeBackPath(r.FormValue("back")) + if back == "/" { + back = docPath(id) + } + http.Redirect(w, r, back, http.StatusSeeOther) +} + +func readPinnedSearchEntries(r *http.Request) []string { + return readIDListCookie(r, pinnedSearchesCookie, pinnedSearchesMax) +} + +func isPinnedSearch(r *http.Request, canonicalQS string) bool { + return slices.Contains(readPinnedSearchEntries(r), url.QueryEscape(canonicalQS)) +} + +func (co *Corpus) handleSearchPinToggle(w http.ResponseWriter, r *http.Request) { + if !readPOSTForm(w, r) { + return + } + qs := r.FormValue("qs") + if qs == "" { + http.Error(w, "bad query", http.StatusBadRequest) + return + } + v, err := url.ParseQuery(qs) + if err != nil { + http.Error(w, "bad query", http.StatusBadRequest) + return + } + + entry := url.QueryEscape(qs) + pinned := readPinnedSearchEntries(r) + if !co.savableSearch(v) && !slices.Contains(pinned, entry) { + http.Error(w, "this search cannot be pinned: it has no terms or filters, or is too long", http.StatusBadRequest) + return + } + stored := writeListCookie(w, pinnedSearchesCookie, toggleEntry(pinned, entry, pinnedSearchesMax)) + if pinRefused(pinned, stored, entry) { + http.Redirect(w, r, "/pinned?full=1", http.StatusSeeOther) + return + } + http.Redirect(w, r, safeBackPath(r.FormValue("back")), http.StatusSeeOther) +} + +func pinRefused(before, stored []string, e string) bool { + return !slices.Contains(before, e) && !slices.Contains(stored, e) +} + +func (co *Corpus) handlePinnedPage(w http.ResponseWriter, r *http.Request) { + prefs := readPrefs(r) + var docs []*Document + for _, id := range co.readPinned(r) { + docs = append(docs, co.docIndex[id]) + } + searches := co.parseSearchEntries(readPinnedSearchEntries(r)) + + data := &pinnedPage{ + Base: pageWith(prefs), + Total: len(docs), + Searches: searches, + Full: r.URL.Query().Get("full") != "", + Cards: newCardView(prefs, docs), + } + data.Cards.ShowUnpin = true + noStore(w) + render(w, r, data) +} + +func (co *Corpus) handlePinnedExport(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "private, no-store") + w.Header().Set("Content-Disposition", `attachment; filename="snowchive-pinned.json"`) + ids := co.readPinned(r) + out := make([]apiDocSummary, 0, len(ids)) + for _, id := range ids { + out = append(out, docSummary(*co.docIndex[id])) + } + writeJSON(w, http.StatusOK, map[string]interface{}{ + "count": len(out), + "docs": out, + }) +} diff --git a/portions.go b/portions.go new file mode 100644 index 0000000..475ffe9 --- /dev/null +++ b/portions.go @@ -0,0 +1,230 @@ +// portions.go - the portion markings inside the transcripts. +package main + +import ( + "html/template" + "log" + "net/http" + "regexp" + "strings" +) + +// A marking opens a line, optionally behind a bullet, a heading hash or a +// list number: "(U)", "* (S//SI) ...", "## (U//FOUO) Heading". +var rePortion = regexp.MustCompile(`^[ \t]{0,6}(?:[-*+][ \t]+|#{1,6}[ \t]+|\d+[.)][ \t]+)?\(([^)\n]{1,42})\)`) + +var portionLevels = map[string]string{ + "U": "U", "UNCLASSIFIED": "U", "FOUO": "U", + "C": "C", "CONFIDENTIAL": "C", + "S": "S", "SECRET": "S", + "TS": "TS", "TOP SECRET": "TS", +} + +var portionControls = map[string]bool{ + "SI": true, "TK": true, "GAMMA": true, "SI-GAMMA": true, "STLW": true, "TSP": true, + "HCS": true, "KLONDIKE": true, "ECI": true, "SI-ECI": true, "COMINT": true, + "FOUO": true, "NF": true, "NOFORN": true, "OC": true, "ORCON": true, + "PROPIN": true, "PR": true, "RELIDO": true, "RSEN": true, "FISA": true, + "IMCON": true, "DSEN": true, "REL": true, "FVEY": true, "NATO": true, + "X1": true, +} + +var portionLevelNames = []string{"U", "C", "S", "TS"} + +var portionLevelLabels = map[string]string{ + "U": "UNCLASSIFIED", "C": "CONFIDENTIAL", "S": "SECRET", "TS": "TOP SECRET", +} + +// reSlashRun folds "/", "//" and "///" to one separator. +var reSlashRun = regexp.MustCompile(`[ \t]*/+[ \t]*`) + +// portionMark is one marking, canonicalised. +type portionMark struct { + Raw string // as the transcript writes it, without the parentheses + Canonical string // "TS//SI//NF" + Level string // "TS" + Controls []string // ["SI", "NF"], in the order the marking writes them + OK bool // every part recognised +} + +func parsePortion(raw string) portionMark { + m := portionMark{Raw: strings.TrimSpace(raw)} + written := strings.Split(reSlashRun.ReplaceAllString(strings.Join(strings.Fields(m.Raw), " "), "//"), "//") + if lvl := strings.TrimSpace(written[0]); lvl == "" || lvl != strings.ToUpper(lvl) { + return m + } + parts := make([]string, len(written)) + for i := range written { + parts[i] = strings.ToUpper(strings.TrimSpace(written[i])) + } + lvl, isLevel := portionLevels[parts[0]] + if !isLevel { + return m + } + m.Level = lvl + if parts[0] == "FOUO" { + m.Controls = append(m.Controls, "FOUO") + } + known := true + for _, p := range parts[1:] { + if p == "" { + continue + } + m.Controls = append(m.Controls, p) + if !portionControlKnown(p) { + known = false + } + } + m.OK = known + m.Canonical = m.Level + if len(m.Controls) > 0 { + m.Canonical += "//" + strings.Join(m.Controls, "//") + } + return m +} + +func portionControlKnown(p string) bool { + if portionControls[p] { + return true + } + if strings.HasPrefix(p, "REL") { + return true + } + if strings.HasPrefix(p, "ECI ") || strings.HasPrefix(p, "SI-ECI") { + return true + } + return false +} + +func extractPortions(text string) []portionMark { + var out []portionMark + fenced := false + for _, line := range strings.Split(text, "\n") { + t := strings.TrimSpace(line) + if strings.HasPrefix(t, "```") || strings.HasPrefix(t, "~~~") { + fenced = !fenced + continue + } + if fenced || strings.HasPrefix(t, ">") { + continue + } + m := rePortion.FindStringSubmatch(line) + if m == nil { + continue + } + if p := parsePortion(m[1]); p.Level != "" { + out = append(out, p) + } + } + return out +} + +// --- the corpus view ------------------------------------------------------- + +type portionRow struct { + Name string // the canonical marking, a level, or a control + Label string // the spelled-out level, where there is one + Marks int // occurrences across the corpus + Docs int // documents carrying it + SearchURL template.URL +} + +type portionSummary struct { + Docs int // documents carrying at least one marking + Marks int // markings found + Distinct int // distinct canonical markings + Unrecog int // markings whose parts are not all known + UnrecogEg []portionRow + Levels []portionRow + Markings []portionRow + Controls []portionRow +} + +const portionRows = 200 + +func (co *Corpus) buildPortions() { + co.portionStats = portionSummary{} + markMarks, lvlMarks, ctlMarks := map[string]int{}, map[string]int{}, map[string]int{} + docsBy := map[string]map[string]int{"PL": {}, "PM": {}, "PC": {}} + unrecogMarks := map[string]int{} + + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + d.PortionLevels, d.PortionMarkings, d.PortionControls = nil, nil, nil + marks := extractPortions(d.FullText) + if len(marks) == 0 { + continue + } + co.portionStats.Docs++ + co.portionStats.Marks += len(marks) + for _, m := range marks { + if !m.OK { + co.portionStats.Unrecog++ + unrecogMarks[m.Raw]++ + continue + } + markMarks[m.Canonical]++ + lvlMarks[m.Level]++ + d.PortionMarkings = append(d.PortionMarkings, m.Canonical) + d.PortionLevels = append(d.PortionLevels, m.Level) + for _, c := range m.Controls { + ctlMarks[c]++ + d.PortionControls = append(d.PortionControls, c) + } + } + d.PortionLevels = normalizeList(d.PortionLevels) + d.PortionMarkings = normalizeList(d.PortionMarkings) + d.PortionControls = normalizeList(d.PortionControls) + for code, docs := range docsBy { + for _, v := range facetValues(d, code) { + docs[v]++ + } + } + } + + co.portionStats.Distinct = len(markMarks) + for _, l := range portionLevelNames { + if lvlMarks[l] == 0 { + continue + } + co.portionStats.Levels = append(co.portionStats.Levels, portionRow{ + Name: l, Label: portionLevelLabels[l], Marks: lvlMarks[l], Docs: docsBy["PL"][l], + SearchURL: portionSearchURL("PL", l), + }) + } + co.portionStats.Markings = portionRowsOf(markMarks, docsBy["PM"], portionRows) + for i := range co.portionStats.Markings { + r := &co.portionStats.Markings[i] + r.SearchURL = portionSearchURL("PM", r.Name) + } + co.portionStats.Controls = portionRowsOf(ctlMarks, docsBy["PC"], portionRows) + for i := range co.portionStats.Controls { + r := &co.portionStats.Controls[i] + r.SearchURL = portionSearchURL("PC", r.Name) + } + co.portionStats.UnrecogEg = portionRowsOf(unrecogMarks, nil, 25) + + log.Printf("Portion markings: %d in %d documents, %d distinct (%d unrecognised)", + co.portionStats.Marks, co.portionStats.Docs, co.portionStats.Distinct, co.portionStats.Unrecog) +} + +func portionSearchURL(code, value string) template.URL { + return template.URL(wholeArchive.link(searchURL([]searchClause{{Field: code, Value: exactClause(value)}}))) +} + +func portionRowsOf(marks, docs map[string]int, n int) []portionRow { + names := rankCounts(marks, n) + out := make([]portionRow, len(names)) + for i, k := range names { + out[i] = portionRow{Name: k, Marks: marks[k], Docs: docs[k]} + } + return out +} + +func (co *Corpus) handlePortions(w http.ResponseWriter, r *http.Request) { + render(w, r, &portionsPage{ + Title: "Portion markings", + Stats: co.portionStats, + RowLimit: portionRows, + }) +} diff --git a/prefs.go b/prefs.go new file mode 100644 index 0000000..1850453 --- /dev/null +++ b/prefs.go @@ -0,0 +1,262 @@ +// prefs.go - per-browser UI preferences stored in cookies, plus shared +// cookie helpers used by the pins and history features. + +package main + +import ( + "net/http" + "slices" + "strconv" + "strings" +) + +// setCookie / delCookie are the single place cookie attributes are decided. + +func setCookie(w http.ResponseWriter, name, val string) { + http.SetCookie(w, &http.Cookie{ + Name: name, + Value: val, + Path: "/", + MaxAge: 365 * 24 * 3600, + SameSite: http.SameSiteLaxMode, + HttpOnly: true, + }) +} + +func delCookie(w http.ResponseWriter, name string) { + http.SetCookie(w, &http.Cookie{ + Name: name, + Value: "", + Path: "/", + MaxAge: -1, + SameSite: http.SameSiteLaxMode, + HttpOnly: true, + }) +} + +const cookieValueMax = 3800 + +func writeListCookie(w http.ResponseWriter, name string, entries []string) []string { + n, size := 0, 0 + for _, e := range entries { + add := len(e) + if n > 0 { + add++ // the "|" separator + } + if size+add > cookieValueMax { + break + } + size += add + n++ + } + entries = entries[:n] + if len(entries) == 0 { + delCookie(w, name) + return entries + } + setCookie(w, name, strings.Join(entries, "|")) + return entries +} + +type Prefs struct { + DocView string // "images" (default), "split", "text" + Theme string // "auto" (default), "light", "dark" + PerPage int // docs per page (default 20); 0 means "all", which cardsPerPage bounds + + ShowClassification bool // classification/dissemination marking on document cards (default true) + ShowJournalists bool // journalist names on document cards (default true) + ShowPeople bool // people line on document cards (default true) + ShowPrograms bool // program names on document cards (default true) + ShowSummary bool // short summary (description) on document cards (default true) + ShowThumbs bool // thumbnails on search results & cards (default true) + ShowCollCols bool // category tables: per-collection count columns always visible (default false, toggle on the page) + TextOnly bool // suppress thumbnails and page images (default false) + CardPagePreview bool // desktop: full page preview on card-thumb hover (default false, opt-in) + ScrollDoc bool // doc view: cap pane height and scroll (default false) + InfoSidebar bool // doc view: metadata beside the document instead of above (default false); cookie is "above"/"beside", not a 0/1 flag + ProgramsAsList bool // doc view: programs one-per-line (default false, comma-separated) + TagsAsList bool // doc view: tags one-per-line (default false, comma-separated) + OpenInfo bool // doc view: "More info" expanded by default (default false) + CollapsePubTogether bool // doc view: collapse "published in the same report" (default false) + CollapseRelated bool // doc view: collapse related documents (default false) + TrackRecent bool // record recently viewed documents (default false) + TrackSearches bool // record recent searches (default false) + IncludeFOIA bool // FOIA releases in search, category totals, /random and related documents when the URL names no collection (default false, opt-in) +} + +func (p Prefs) thumbsOn() bool { return p.ShowThumbs && !p.TextOnly } + +const maxCardsPerPage = 500 + +func (p Prefs) cardsPerPage() int { + if p.PerPage <= 0 || p.PerPage > maxCardsPerPage { + return maxCardsPerPage + } + return p.PerPage +} + +func newCardView(p Prefs, docs []*Document) cardView { + return cardView{ + Docs: docs, + ShowThumbs: p.thumbsOn(), + CardPagePreview: p.CardPagePreview, + HideClassification: !p.ShowClassification, + HideJournalists: !p.ShowJournalists, + HidePeople: !p.ShowPeople, + HidePrograms: !p.ShowPrograms, + HideSummary: !p.ShowSummary, + FOIAQS: p.foiaLinkQS(), + } +} + +func (p Prefs) foiaLinkQS() string { + return pageScope(nil, p, false).including(collectionFOIA).suffix() +} + +var boolPrefs = []struct { + cookie string + form string + invert bool + field func(*Prefs) *bool +}{ + {"pref_hide_classification", "hide_classification", true, func(p *Prefs) *bool { return &p.ShowClassification }}, + {"pref_hide_journalists", "hide_journalists", true, func(p *Prefs) *bool { return &p.ShowJournalists }}, + {"pref_hide_people", "hide_people", true, func(p *Prefs) *bool { return &p.ShowPeople }}, + {"pref_hide_programs", "hide_programs", true, func(p *Prefs) *bool { return &p.ShowPrograms }}, + {"pref_hide_summary", "hide_summary", true, func(p *Prefs) *bool { return &p.ShowSummary }}, + {"pref_hide_thumbs", "hide_thumbs", true, func(p *Prefs) *bool { return &p.ShowThumbs }}, + {"pref_show_coll_cols", "show_coll_cols", false, func(p *Prefs) *bool { return &p.ShowCollCols }}, + {"pref_text_only", "text_only", false, func(p *Prefs) *bool { return &p.TextOnly }}, + {"pref_page_preview", "page_preview", false, func(p *Prefs) *bool { return &p.CardPagePreview }}, + {"pref_scroll_doc", "scroll_doc", false, func(p *Prefs) *bool { return &p.ScrollDoc }}, + {"pref_programs_list", "programs_list", false, func(p *Prefs) *bool { return &p.ProgramsAsList }}, + {"pref_tags_list", "tags_list", false, func(p *Prefs) *bool { return &p.TagsAsList }}, + {"pref_open_info", "open_info", false, func(p *Prefs) *bool { return &p.OpenInfo }}, + {"pref_collapse_pub", "collapse_pub", false, func(p *Prefs) *bool { return &p.CollapsePubTogether }}, + {"pref_collapse_related", "collapse_related", false, func(p *Prefs) *bool { return &p.CollapseRelated }}, + {"pref_track_recent", "track_recent", false, func(p *Prefs) *bool { return &p.TrackRecent }}, + {"pref_track_searches", "track_searches", false, func(p *Prefs) *bool { return &p.TrackSearches }}, + {"pref_include_foia", "include_foia", false, func(p *Prefs) *bool { return &p.IncludeFOIA }}, +} + +var valuePrefs = []struct { + cookie string + form string + allowed []string + def string + apply func(*Prefs, string) +}{ + {"pref_doc_view", "doc_view", []string{"images", "split", "text"}, "images", + func(p *Prefs, v string) { p.DocView = v }}, + {"pref_theme", "theme", []string{"auto", "light", "dark"}, "auto", + func(p *Prefs, v string) { p.Theme = v }}, + {"pref_per_page", "per_page", []string{"10", "20", "50", "100", "all"}, "20", + func(p *Prefs, v string) { + if v == "all" { + p.PerPage = 0 + return + } + if n, err := strconv.Atoi(v); err == nil { + p.PerPage = n + } + }}, + {"pref_info_position", "info_position", []string{"above", "beside"}, "above", + func(p *Prefs, v string) { p.InfoSidebar = v == "beside" }}, +} + +var prefCookies = func() []string { + var out []string + for _, vp := range valuePrefs { + out = append(out, vp.cookie) + } + for _, bp := range boolPrefs { + out = append(out, bp.cookie) + } + return append(out, recentDocsCookie, recentSearchesCookie) +}() + +func (p Prefs) ThemeAttr() string { + if p.Theme == "auto" { + return "" + } + return p.Theme +} + +func readPrefs(r *http.Request) Prefs { + var p Prefs + cookie := map[string]string{} + for _, c := range r.Cookies() { + cookie[c.Name] = c.Value + } + + for i := range boolPrefs { + bp := &boolPrefs[i] + v := cookie[bp.cookie] == "1" + if bp.invert { + v = !v + } + *bp.field(&p) = v + } + + for i := range valuePrefs { + vp := &valuePrefs[i] + v := cookie[vp.cookie] + if !slices.Contains(vp.allowed, v) { + v = vp.def + } + vp.apply(&p, v) + } + return p +} + +func handlePreferences(w http.ResponseWriter, r *http.Request) { + noStore(w) + q := r.URL.Query() + render(w, r, &preferencesPage{ + Base: pageWith(readPrefs(r)), + MaxCards: maxCardsPerPage, + Saved: q.Get("saved") == "1", + Reset: q.Get("reset") == "1", + }) +} + +func handlePreferencesSave(w http.ResponseWriter, r *http.Request) { + if !readPOSTForm(w, r) { + return + } + + if r.FormValue("reset") == "1" { + for _, name := range prefCookies { + delCookie(w, name) + } + http.Redirect(w, r, "/preferences?reset=1", http.StatusSeeOther) + return + } + + for _, vp := range valuePrefs { + v := r.FormValue(vp.form) + switch { + case v == vp.def: + delCookie(w, vp.cookie) + case slices.Contains(vp.allowed, v): + setCookie(w, vp.cookie, v) + } + } + + for _, bp := range boolPrefs { + if r.FormValue(bp.form) == "1" { + setCookie(w, bp.cookie, "1") + } else { + delCookie(w, bp.cookie) + } + } + + if r.FormValue("track_recent") != "1" { + delCookie(w, recentDocsCookie) + } + if r.FormValue("track_searches") != "1" { + delCookie(w, recentSearchesCookie) + } + + http.Redirect(w, r, "/preferences?saved=1", http.StatusSeeOther) +} diff --git a/search.go b/search.go new file mode 100644 index 0000000..a5858ac --- /dev/null +++ b/search.go @@ -0,0 +1,2644 @@ +package main + +import ( + "cmp" + "html/template" + "log" + "math" + "net/http" + "net/url" + "slices" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +type TFIDFIndex struct { + DocStats map[string]map[string]float64 + DocFreq map[string]int + N int +} + +// stopWords are excluded from tokenization to avoid recreating the map on +// every tokenize() call. +var stopWords = map[string]bool{ + "the": true, "and": true, "for": true, "that": true, + "this": true, "with": true, "from": true, "are": true, + "was": true, "were": true, "has": true, "have": true, + "not": true, "but": true, "they": true, "their": true, + "will": true, "been": true, "its": true, "also": true, + "into": true, "which": true, "such": true, "other": true, +} + +// --- Stemmer --- +// Porter-lite: suffix stripping for English. Applied at both index time and +// query time so "surveillance" and "surveilling" map to the same token. +// Intentionally conservative - only strips high-confidence suffixes to avoid +// over-stemming domain-specific terms like "SIGINT" or "collection". +func stem(w string) string { + // The last two steps apply whichever suffix rule returned: "flying" and + // "flyer" leave "fly", which must meet "fly" itself. + s := yToI(stripSuffixes(w)) + // fixStem keeps a doubled l ("falling" -> "fall"), so "controlled" and + // "controller" reach "controll" while "control" stays "control". A final + // "ll" after more than one vowel-consonant run is undoubled in every + // word alike (Porter's step 5b): control/controlled -> "control", + // install/installing -> "instal", while fall/falling stay "fall". + if strings.HasSuffix(s, "ll") && measure(s) > 1 { + return s[:len(s)-1] + } + return s +} + +func stripSuffixes(w string) string { + switch { + case strings.HasSuffix(w, "sses"): + w = w[:len(w)-2] + case strings.HasSuffix(w, "ies") && len(w) > 4: + w = w[:len(w)-3] + "y" // "studies" -> "study", "spies" -> "spy" + case strings.HasSuffix(w, "ies"): + w = w[:len(w)-1] // "ties" -> "tie": one letter before it, a word in "ie" + case strings.HasSuffix(w, "ss"): + // "process", "class": not a plural + case strings.HasSuffix(w, "s") && len(w) > 3: + w = w[:len(w)-1] + } + switch { + case strings.HasSuffix(w, "ied") && len(w) == 4: + return w[:3] // "tied" -> "tie", like "ties" + case strings.HasSuffix(w, "ied") && len(w) > 4: + w = w[:len(w)-3] + "y" // "studied" -> "study", "spied" -> "spy" + case strings.HasSuffix(w, "eed") && len(w) > 4: + return w[:len(w)-1] + case strings.HasSuffix(w, "ed") && len(w) > 4: + s := w[:len(w)-2] + return fixStem(s) + case strings.HasSuffix(w, "ing") && len(w) > 5: + s := w[:len(w)-3] + return fixStem(s) + } + switch { + case strings.HasSuffix(w, "ational") && len(w) > 8: + return w[:len(w)-5] // "operational" -> "operat", same as "operation" + case strings.HasSuffix(w, "ation") && len(w) > 6: + return w[:len(w)-3] + case strings.HasSuffix(w, "ence") && len(w) > 5: + return w[:len(w)-4] + case strings.HasSuffix(w, "ance") && len(w) > 5: + return w[:len(w)-4] + case strings.HasSuffix(w, "ment") && len(w) > 5: + return w[:len(w)-4] + case strings.HasSuffix(w, "ness") && len(w) > 5: + return w[:len(w)-4] + case strings.HasSuffix(w, "ity") && len(w) > 5: + return w[:len(w)-3] + case strings.HasSuffix(w, "ism") && len(w) > 5: + return w[:len(w)-3] + case strings.HasSuffix(w, "ist") && len(w) > 5: + return w[:len(w)-3] + case strings.HasSuffix(w, "ize") && len(w) > 5: + return w[:len(w)-3] + case strings.HasSuffix(w, "ise") && len(w) > 5: + return w[:len(w)-3] + case strings.HasSuffix(w, "ful") && len(w) > 5: + return w[:len(w)-3] + case strings.HasSuffix(w, "less") && len(w) > 5: + return w[:len(w)-4] + case strings.HasSuffix(w, "ly") && len(w) > 5 && w[len(w)-3] != 'p': + return w[:len(w)-2] + case strings.HasSuffix(w, "er") && len(w) > 4: + return w[:len(w)-2] + case strings.HasSuffix(w, "al") && len(w) > 4: + return w[:len(w)-2] + } + return w +} + +// yToI turns a final consonant+y into i, so singular/plural pairs unify +// with the "ies" rule: "study" -> "studi", "studies" -> "studi"; +// "agency" -> "agenci", "agencies" -> "agenci". +func yToI(w string) string { + if strings.HasSuffix(w, "y") && len(w) > 3 && !isVowel(rune(w[len(w)-2])) { + return w[:len(w)-1] + "i" + } + if strings.HasSuffix(w, "ie") && len(w) > 5 { + return w[:len(w)-1] + } + return w +} + +// measure is Porter's m: the number of vowel-consonant runs in w, with y a +// vowel after a consonant ("controll" 2, "fall" 1, "tree" 0). +func measure(w string) int { + m := 0 + prevVowel := false + for i := 0; i < len(w); i++ { + v := isVowel(rune(w[i])) || (w[i] == 'y' && i > 0 && !prevVowel) + if prevVowel && !v { + m++ + } + prevVowel = v + } + return m +} + +// fixStem handles consonant doubling after stripping -ed/-ing +// e.g. "hopping" -> "hopp" -> "hop". +func fixStem(s string) string { + if len(s) == 0 { + return s + } + if len(s) >= 2 { + last := s[len(s)-1] + secondLast := s[len(s)-2] + if last == secondLast && !isVowel(rune(last)) && last != 'l' && last != 's' && last != 'z' { + return s[:len(s)-1] + } + } + return s +} + +func isVowel(r rune) bool { + return strings.ContainsRune("aeiou", r) +} + +// foldTable maps accented Latin characters to their ASCII base form so +// "muller" finds "Müller-Maguhn" and "Sao Paulo" finds "São Paulo". The +// fold runs on BOTH the index and the query side, so it's symmetric. +var foldTable = map[rune]string{ + 'À': "A", 'Á': "A", 'Â': "A", 'Ã': "A", 'Ä': "A", + 'Å': "A", 'Æ': "AE", 'Ç': "C", 'È': "E", 'É': "E", 'Ê': "E", + 'Ë': "E", 'Ì': "I", 'Í': "I", 'Î': "I", 'Ï': "I", 'Ð': "D", + 'Ñ': "N", 'Ò': "O", 'Ó': "O", 'Ô': "O", 'Õ': "O", 'Ö': "O", + 'Ø': "O", 'Ù': "U", 'Ú': "U", 'Û': "U", 'Ü': "U", 'Ý': "Y", + 'Þ': "TH", 'ß': "ss", 'à': "a", 'á': "a", 'â': "a", 'ã': "a", + 'ä': "a", 'å': "a", 'æ': "ae", 'ç': "c", 'è': "e", 'é': "e", + 'ê': "e", 'ë': "e", 'ì': "i", 'í': "i", 'î': "i", 'ï': "i", + 'ð': "d", 'ñ': "n", 'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o", + 'ö': "o", 'ø': "o", 'ù': "u", 'ú': "u", 'û': "u", 'ü': "u", + 'ý': "y", 'þ': "th", 'ÿ': "y", 'Ā': "A", 'ā': "a", 'Ă': "A", + 'ă': "a", 'Ą': "A", 'ą': "a", 'Ć': "C", 'ć': "c", 'Ĉ': "C", + 'ĉ': "c", 'Ċ': "C", 'ċ': "c", 'Č': "C", 'č': "c", 'Ď': "D", + 'ď': "d", 'Đ': "D", 'đ': "d", 'Ē': "E", 'ē': "e", 'Ĕ': "E", + 'ĕ': "e", 'Ė': "E", 'ė': "e", 'Ę': "E", 'ę': "e", 'Ě': "E", + 'ě': "e", 'Ĝ': "G", 'ĝ': "g", 'Ğ': "G", 'ğ': "g", 'Ġ': "G", + 'ġ': "g", 'Ģ': "G", 'ģ': "g", 'Ĥ': "H", 'ĥ': "h", 'Ħ': "H", + 'ħ': "h", 'Ĩ': "I", 'ĩ': "i", 'Ī': "I", 'ī': "i", 'Ĭ': "I", + 'ĭ': "i", 'Į': "I", 'į': "i", 'İ': "I", 'ı': "i", 'Ĵ': "J", + 'ĵ': "j", 'Ķ': "K", 'ķ': "k", 'Ĺ': "L", 'ĺ': "l", 'Ļ': "L", + 'ļ': "l", 'Ľ': "L", 'ľ': "l", 'Ŀ': "L", 'ŀ': "l", 'Ł': "L", + 'ł': "l", 'Ń': "N", 'ń': "n", 'Ņ': "N", 'ņ': "n", 'Ň': "N", + 'ň': "n", 'ʼn': "n", 'Ō': "O", 'ō': "o", 'Ŏ': "O", 'ŏ': "o", + 'Ő': "O", 'ő': "o", 'Œ': "OE", 'œ': "oe", 'Ŕ': "R", 'ŕ': "r", + 'Ŗ': "R", 'ŗ': "r", 'Ř': "R", 'ř': "r", 'Ś': "S", 'ś': "s", + 'Ŝ': "S", 'ŝ': "s", 'Ş': "S", 'ş': "s", 'Š': "S", 'š': "s", + 'Ţ': "T", 'ţ': "t", 'Ť': "T", 'ť': "t", 'Ŧ': "T", 'ŧ': "t", + 'Ũ': "U", 'ũ': "u", 'Ū': "U", 'ū': "u", 'Ŭ': "U", 'ŭ': "u", + 'Ů': "U", 'ů': "u", 'Ű': "U", 'ű': "u", 'Ų': "U", 'ų': "u", + 'Ŵ': "W", 'ŵ': "w", 'Ŷ': "Y", 'ŷ': "y", 'Ÿ': "Y", 'Ź': "Z", + 'ź': "z", 'Ż': "Z", 'ż': "z", 'Ž': "Z", 'ž': "z", 'Ș': "S", + 'ș': "s", 'Ț': "T", 'ț': "t", 'ẞ': "SS", +} + +// foldMin is the smallest rune foldTable maps, every rune below it folds +// to itself so the folds skip the table lookup for it (all of ASCII). +var foldMin = func() rune { + m := utf8.MaxRune + for r := range foldTable { + m = min(m, r) + } + return m +}() + +// foldAccents replaces accented characters with their base form, +// preserving case for characters not in the table. +func foldAccents(s string) string { + needs := false + for _, r := range s { + if r < foldMin { + continue + } + if _, ok := foldTable[r]; ok { + needs = true + break + } + } + if !needs { + return s + } + var b strings.Builder + b.Grow(len(s)) + for _, r := range s { + if r >= foldMin { + if f, ok := foldTable[r]; ok { + b.WriteString(f) + continue + } + } + b.WriteRune(r) + } + return b.String() +} + +func normFoldString(s string) string { + return strings.ToLower(foldAccents(s)) +} + +// foldRune is normFoldString for one rune. +func foldRune(r rune) string { + if r < utf8.RuneSelf { + if 'A' <= r && r <= 'Z' { + r += 'a' - 'A' + } + return string(r) + } + if f, ok := foldTable[r]; ok { + return strings.ToLower(f) + } + return strings.ToLower(string(r)) +} + +func foldedLen(r rune) int { + if r < utf8.RuneSelf { + return 1 + } + if f, ok := foldTable[r]; ok { + return len(f) + } + return utf8.RuneLen(unicode.ToLower(r)) +} + +// normFold returns the normal form of s plus a position map: posMap[i] is +// the byte offset in s of the rune that produced norm byte i (with a final +// entry of len(s)). Lets match highlighting (markTerms) mark on the normal +// form but slice the ORIGINAL text, so a "muller" query highlights "Müller" +// as written. +func normFold(s string) (string, []int) { + var b strings.Builder + b.Grow(len(s)) + posMap := make([]int, 0, len(s)+1) + for i, r := range s { + if r < utf8.RuneSelf { + if 'A' <= r && r <= 'Z' { + r += 'a' - 'A' + } + b.WriteByte(byte(r)) + posMap = append(posMap, i) + continue + } + f := foldRune(r) + for j := 0; j < len(f); j++ { + posMap = append(posMap, i) + } + b.WriteString(f) + } + posMap = append(posMap, len(s)) + return b.String(), posMap +} + +// splitWords splits s into alphanumeric runs (it does not change case). +func splitWords(s string) []string { + return strings.FieldsFunc(s, func(r rune) bool { + return !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) +} + +func tokenize(s string) []string { + words := splitWords(normFoldString(s)) + filtered := words[:0] + for _, w := range words { + if len(w) > 1 && !stopWords[w] { + filtered = append(filtered, stem(w)) + } + } + return filtered +} + +// eachToken calls f with every token of a stemmed stream (tokenize's +// output joined by single spaces) in order. +func eachToken(stream string, f func(tok string)) { + for i := 0; i < len(stream); { + j := strings.IndexByte(stream[i:], ' ') + if j < 0 { + j = len(stream) - i + } + if j > 0 { + f(stream[i : i+j]) + } + i += j + 1 + } +} + +func buildTFIDF(docs []Document) TFIDFIndex { + idx := TFIDFIndex{ + DocStats: make(map[string]map[string]float64, len(docs)), + DocFreq: make(map[string]int), + N: len(docs), + } + + for i := range docs { + d := &docs[i] + // Title 3x, fulltext 2x, descriptions + metadata 1x. The metadata + // is every field the all-fields search reads (searchMeta), so a doc + // that matches only via metadata still gets a relevance score + // instead of always sorting last. Title, descriptions and body come + // from the stemmed forms loadData already built (tokenize() output + // joined by spaces). + tf := make(map[string]float64) + total := 0.0 + add := func(weight float64) func(string) { + return func(w string) { + if w == valueSep { + return + } + tf[w] += weight + total += weight + } + } + eachToken(d.StemmedTitle, add(3)) + eachToken(d.StemmedFull, add(2)) + eachToken(d.StemmedDesc, add(1)) + for _, w := range tokenize(searchMeta(d)) { + add(1)(w) + } + + for w := range tf { + tf[w] /= total // total > 0 whenever tf has a key + idx.DocFreq[w]++ + } + idx.DocStats[d.ID] = tf + } + + log.Printf("TF-IDF index built: %d docs, %d unique terms", len(docs), len(idx.DocFreq)) + return idx +} + +// scorer reads one positive term of a query into its relevance score for a +// document. The term's words go through the same tokenize() pipeline as the +// index (stemming + stopword removal), and matching against stemmed fields +// is by whole token. +func (idx TFIDFIndex) scorer(query string) func(doc *Document) float64 { + // A proximity clause scores as its phrase - the window is a matcher + // concern, not a relevance signal (and the raw "~10" tail would + // otherwise leak a junk "10" term into the tokenizer). + if _, phrase, _, ok := parseProximity(query); ok { + query = phrase + } + // An exact-value clause scores as its phrase with the phrase bonuses on + // whole words; quoted text earns them wherever it occurs. + contains := strings.Contains + if v, ok := exactValue(query); ok { + query, contains = `"`+v+`"`, containsWord + } + // Folded so the phrase bonuses are accent-consistent with matching, and + // so the content check can use the pre-folded body (no per-doc fold). + clean := strings.Trim(normFoldString(strings.TrimSpace(query)), `"`) + // Wildcard words score by where they occur, not through their stems: + // +5 when the title has one, else +1 when the document does. + var wild []queryWord + if !isQuotedQuery(query) { + var kept []string + for _, w := range strings.Fields(clean) { + if _, _, _, isWild := wildcard(w); isWild { + wild = append(wild, literalWord(w)) + } else { + kept = append(kept, w) + } + } + clean = strings.Join(kept, " ") + } + terms := tokenize(clean) + // A term's weight in a document is its share of the document times how + // rare it is in the corpus; a term no document holds weighs nothing. + idf := make([]float64, len(terms)) + for i, w := range terms { + if df := idx.DocFreq[w]; df != 0 { + idf[i] = math.Log(float64(idx.N) / float64(df)) + } + } + isPhrase := len(strings.Fields(clean)) > 1 + + return func(doc *Document) float64 { + stats, ok := idx.DocStats[doc.ID] + if !ok || (len(terms) == 0 && len(wild) == 0) { + return 0 + } + title := "" // for the wildcard and phrase bonuses + if len(wild) > 0 || isPhrase { + title = doc.titleFold() + } + + var score float64 + for i := range wild { + switch { + case wild[i].kind == wordNever: // matches nothing (see wildcard), so earns nothing + case wild[i].in(title): + score += 5 + case wild[i].in(doc.SearchText): + score++ + } + } + if len(terms) == 0 { + return score + } + for i, w := range terms { + score += stats[w] * idf[i] + } + + // bonus is whole when the field holds the exact phrase (or, for one + // word, the word), else all when it holds every term somewhere. + bonus := func(phraseIn string, holds func(term string) bool, whole, all float64) float64 { + if isPhrase && contains(phraseIn, clean) { + return whole + } + for _, w := range terms { + if !holds(w) { + return 0 + } + } + if isPhrase { + return all + } + return whole + } + // Title +10 or +5; the whole document +3 or +1. The document's + // entry in the index holds exactly the terms of its all-fields + // stream. + inTitle := func(w string) bool { return containsTokens(doc.StemmedTitle, w) } + inDoc := func(w string) bool { _, ok := stats[w]; return ok } + return score + bonus(title, inTitle, 10, 5) + bonus(doc.FoldedFull, inDoc, 3, 1) + } +} + +// similarity returns cosine similarity between two docs' TF-IDF vectors. +// Returns 0-1; used by relatedScore in data.go. +// TODO split this file or move this away from here? +func (idx TFIDFIndex) similarity(aID, bID string) float64 { + a, ok1 := idx.DocStats[aID] + b, ok2 := idx.DocStats[bID] + if !ok1 || !ok2 { + return 0 + } + + var dot, normA, normB float64 + for w, tfA := range a { + df := idx.DocFreq[w] + if df == 0 { + continue + } + idf := math.Log(float64(idx.N) / float64(df)) + tfidfA := tfA * idf + normA += tfidfA * tfidfA + if tfB, ok := b[w]; ok { + tfidfB := tfB * idf + dot += tfidfA * tfidfB + } + } + for w, tfB := range b { + df := idx.DocFreq[w] + if df == 0 { + continue + } + idf := math.Log(float64(idx.N) / float64(df)) + tfidfB := tfB * idf + normB += tfidfB * tfidfB + } + + if normA == 0 || normB == 0 { + return 0 + } + return dot / (math.Sqrt(normA) * math.Sqrt(normB)) +} + +// --- Clause values --- + +// isQuotedQuery returns true for "exact phrase" queries. +func isQuotedQuery(q string) bool { + q = strings.TrimSpace(q) + return len(q) >= 2 && strings.HasPrefix(q, `"`) && strings.HasSuffix(q, `"`) +} + +// --- Exact values and word boundaries --- +// +// A clause value has four shapes: words (cable tapping, matched with their +// other forms), quoted text ("cable tapping", found anywhere, like grep -i), +// an exact value (="cable tapping", or =tap for one word) and a wildcard +// (tap*, *apping). An exact value means "exactly this, nothing attached", on +// a metadata field it is one whole value, so agency:=NSA does not match +// "NSA, GCHQ"; on the text fields (all fields, title, content, description) +// it is whole words exactly as typed, so =tap finds "tap" but not "taps" or +// "strap". + +// exactValue reports whether q is an exact-value clause ("=") followed by +// a quoted value or a single bare word - and returns the value. +func exactValue(q string) (string, bool) { + q = strings.TrimSpace(q) + if !strings.HasPrefix(q, "=") { + return "", false + } + v := strings.TrimSpace(q[1:]) + if isQuotedQuery(v) { + v = strings.TrimSpace(v[1 : len(v)-1]) + } else if strings.ContainsAny(v, spaceBytes+`"`) { + return "", false + } + return v, v != "" +} + +// exactClause builds the ="..." clause value the generated links use, so a +// count and its click compare whole values the same way. +func exactClause(v string) string { return `="` + v + `"` } + +// isWordRune is the word-character test splitWords uses. +func isWordRune(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) } + +// indexWord returns the byte offset of the first occurrence of needle in s, +// at or after from, that stands on word boundaries (no letter or digit +// directly before or after it) or -1. A needle edge that is not a word +// character ("u.s.") needs no boundary on that side. s and needle must +// already share a normal form (normFoldString). +func indexWord(s, needle string, from int) int { + first, _ := utf8.DecodeRuneInString(needle) + last, _ := utf8.DecodeLastRuneInString(needle) + return indexBounded(s, needle, from, isWordRune(first), isWordRune(last)) +} + +// wildMinRunes is the fewest letters or digits a fragment behind a leading +// "*" must have: "*a" or "*ab" would match nearly every document. +const wildMinRunes = 3 + +// wildcard reports whether w is a wildcard word (a fragment with "*") at +// its end (tap*), its start (*apping) or both (*keys*), and returns the +// fragment in normal form plus which sides may run on into more of the +// word. +func wildcard(w string) (frag string, openStart, openEnd, ok bool) { + openStart, openEnd = strings.HasPrefix(w, "*"), strings.HasSuffix(w, "*") + if !openStart && !openEnd { + return "", false, false, false + } + frag = normFoldString(strings.Trim(w, "*")) + n := 0 + for _, r := range frag { + if isWordRune(r) { + n++ + } + } + if n == 0 || (openStart && n < wildMinRunes) || strings.Contains(frag, "*") { + return "", openStart, openEnd, true + } + return frag, openStart, openEnd, true +} + +// indexWild finds a wildcard fragment in s, both in normal form. A closed +// side must sit on a word boundary and an open side may continue into the +// word, so tap* finds "tapping" but not "strap". +func indexWild(s, frag string, from int, openStart, openEnd bool) int { + first, _ := utf8.DecodeRuneInString(frag) + last, _ := utf8.DecodeLastRuneInString(frag) + return indexBounded(s, frag, from, !openStart && isWordRune(first), !openEnd && isWordRune(last)) +} + +// wordBounds widens [start, end) in s to the whole word around it. +func wordBounds(s string, start, end int) (int, int) { + for start > 0 { + r, size := utf8.DecodeLastRuneInString(s[:start]) + if !isWordRune(r) { + break + } + start -= size + } + for end < len(s) { + r, size := utf8.DecodeRuneInString(s[end:]) + if !isWordRune(r) { + break + } + end += size + } + return start, end +} + +// indexBounded is indexWord with the boundary checks chosen by the caller: +// checkStart requires no letter or digit directly before the match, +// checkEnd none directly after it. +func indexBounded(s, needle string, from int, checkStart, checkEnd bool) int { + if from < 0 { + from = 0 + } + if needle == "" { + if from <= len(s) { + return from + } + return -1 + } + for from <= len(s)-len(needle) { + j := strings.Index(s[from:], needle) + if j < 0 { + return -1 + } + i := from + j + end := i + len(needle) + ok := true + if checkStart && i > 0 { + if r, _ := utf8.DecodeLastRuneInString(s[:i]); isWordRune(r) { + ok = false + } + } + if ok && checkEnd && end < len(s) { + if r, _ := utf8.DecodeRuneInString(s[end:]); isWordRune(r) { + ok = false + } + } + if ok { + return i + } + _, size := utf8.DecodeRuneInString(s[i:]) + from = i + size + } + return -1 +} + +// containsWord reports whether needle occurs in s on word boundaries. +func containsWord(s, needle string) bool { return indexWord(s, needle, 0) >= 0 } + +// containsTokens reports whether phrase (one or more stemmed tokens joined +// by single spaces) occurs in a space-joined token stream as WHOLE tokens. +func containsTokens(stream, phrase string) bool { + if phrase == "" { + return true + } + for i := 0; i+len(phrase) <= len(stream); { + j := strings.Index(stream[i:], phrase) + if j < 0 { + return false + } + p := i + j + end := p + len(phrase) + if (p == 0 || stream[p-1] == ' ') && (end == len(stream) || stream[end] == ' ') { + return true + } + i = p + 1 + } + return false +} + +// valueSep joins a field's values for word and phrase matching. It is not a +// word character, so words still split on it, and nobody types it, so a +// quoted phrase cannot run from one value into the next. +const valueSep = "\x1f" + +func valueKey(v string) string { return strings.ToLower(strings.TrimSpace(v)) } + +func sameValue(a, b string) bool { + a, b = strings.TrimSpace(a), strings.TrimSpace(b) + if !isASCII(a) || !isASCII(b) { + return strings.ToLower(a) == strings.ToLower(b) + } + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + x, y := a[i], b[i] + if 'A' <= x && x <= 'Z' { + x += 'a' - 'A' + } + if 'A' <= y && y <= 'Z' { + y += 'a' - 'A' + } + if x != y { + return false + } + } + return true +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} + +// --- Compiled clauses --- +// +// A search reads each clause once, into a test, and the scan runs the tests +// on every document. + +// queryWord is one word of an unquoted clause. +type queryWord struct { + kind int + text string // the fragment (wordWild), the word (wordWhole, wordSub) or its stemmed tokens joined by spaces (wordStems) + openStart, openEnd bool // wordWild: which sides of the fragment had "*" + tokens []string // wordStems: the tokens of text +} + +const ( + wordNever = iota // a wildcard with nothing to search for (wildcard): it matches nothing + wordWild // tap*, *apping, *keys*: part of a word, in the folded text + wordWhole // a whole word as written, in the folded text + wordSub // a word with internal punctuation ("u.s."), as written, anywhere in the folded text + wordStems // the word's stemmed tokens in sequence, in the token stream +) + +// literalWord reads a folded query word the way folded text can hold it: a +// wildcard, a whole word or, with internal punctuation, which word matching +// can't represent, a substring. +func literalWord(w string) queryWord { + if frag, openStart, openEnd, ok := wildcard(w); ok { + if frag == "" { + return queryWord{kind: wordNever} + } + return queryWord{kind: wordWild, text: frag, openStart: openStart, openEnd: openEnd} + } + if parts := splitWords(w); len(parts) == 1 { + return queryWord{kind: wordWhole, text: parts[0]} + } + return queryWord{kind: wordSub, text: w} +} + +// in reports whether the word, read literally, occurs in folded text. +func (w *queryWord) in(folded string) bool { + switch w.kind { + case wordWild: + return indexWild(folded, w.text, 0, w.openStart, w.openEnd) >= 0 + case wordWhole: + return containsWord(folded, w.text) + case wordSub: + return strings.Contains(folded, w.text) + } + return false +} + +// literalTest is a clause read for folded text that is not tokenized (the +// values of a metadata field). Quoted text is found anywhere, like grep -i +// ("ard" inside "guardian"); otherwise every word must occur as literalWord +// reads it. +type literalTest struct { + quoted bool + needle string + words []queryWord +} + +func compileLiteral(q string) literalTest { + q = normFoldString(strings.TrimSpace(q)) + if isQuotedQuery(q) { + return literalTest{quoted: true, needle: strings.Trim(q, `"`)} + } + var t literalTest + for _, w := range strings.Fields(q) { + t.words = append(t.words, literalWord(w)) + } + return t +} + +func (t *literalTest) match(folded string) bool { + if t.quoted { + return strings.Contains(folded, t.needle) + } + for i := range t.words { + if !t.words[i].in(folded) { + return false + } + } + return true +} + +// valuesTest is a clause read for a metadata field's values: ="..." is one +// whole value (the equality the browse pages and the refine tallies use +// the valueKey) and anything else the values' words and phrases +// (literalTest). +type valuesTest struct { + exact bool + key string // exact: the value's valueKey + text literalTest +} + +func compileValues(q string) valuesTest { + if v, ok := exactValue(q); ok { + return valuesTest{exact: true, key: valueKey(v)} + } + return valuesTest{text: compileLiteral(q)} +} + +// match tests the clause on a list of values. +func (t *valuesTest) match(vals []string) bool { + if t.exact { + for _, x := range vals { + if sameValue(x, t.key) { + return true + } + } + return false + } + return t.text.match(normFoldString(strings.Join(vals, valueSep))) +} + +func (t *valuesTest) matchIndexed(keys []string, folded string) bool { + if t.exact { + return slices.Contains(keys, t.key) + } + return t.text.match(folded) +} + +// compileWords reads an unquoted clause for a text field. Its words run +// through the same tokenize pipeline as the index, so stemming stays +// consistent with it. +func compileWords(q string) []queryWord { + var stems, literal []queryWord + for _, w := range strings.Fields(strings.ToLower(strings.TrimSpace(q))) { + if _, _, _, ok := wildcard(w); ok { + literal = append(literal, literalWord(w)) + } else if toks := tokenize(w); len(toks) == 0 { + literal = append(literal, literalWord(normFoldString(w))) + } else { + stems = append(stems, queryWord{kind: wordStems, text: strings.Join(toks, " "), tokens: toks}) + } + } + return append(stems, literal...) +} + +// compile reads clause q (one negation-free value of a row) into the +// test of it on the field. +func (f *searchField) compile(q string) func(*Document) bool { + if terms, phrase, win, ok := parseProximity(q); ok { + // "..."~N - see the proximity block below. + if f.Stemmed != nil && len(terms) >= 2 { + return func(d *Document) bool { + return d.hasTerms(terms) && proximityMatch(f.Stemmed(d), terms, win) + } + } + // A metadata field, or a phrase that boiled down to fewer than two + // searchable words, so it adds nothing, match the plain + // quoted phrase on the requested field instead. + q = phrase + } + if f.Values != nil { + test := compileValues(q) + return func(d *Document) bool { + if d.fields == nil { + return test.match(f.read(d)) + } + return test.matchIndexed(d.fields.keys[f.index], d.fields.folded[f.index]) + } + } + + // A text field: an exact value (=word) is a whole-word test and quoted + // text a substring test, both on the folded text; anything else runs on + // the stemmed tokens, and reads the folded text only for its wildcard + // and literal words. + if v, ok := exactValue(q); ok { + needle := normFoldString(v) + return func(d *Document) bool { return containsWord(f.Folded(d), needle) } + } + if isQuotedQuery(q) { + needle := strings.Trim(normFoldString(strings.TrimSpace(q)), `"`) + return func(d *Document) bool { return strings.Contains(f.Folded(d), needle) } + } + words := compileWords(q) + everyField := f.Code == "TE" + return func(d *Document) bool { + for i := range words { + w := &words[i] + switch { + case w.kind != wordStems: + if !w.in(f.Folded(d)) { + return false + } + case !d.hasTerms(w.tokens): + return false + case everyField && len(w.tokens) == 1 && d.terms != nil: + // stream needs no walking + case !containsTokens(f.Stemmed(d), w.text): + return false + } + } + return true + } +} + +// --- Proximity operator ---------------------------------------------------- +// +// "upstream collection"~10 matches documents where every quoted word occurs +// within a 10-word window, unordered and counted on the same stemmed, +// stopword-free token streams the bare-word matcher uses. + +// parseProximity reports whether q has the "..."~N shape (the whole clause, +// nothing after the digits). terms are the tokenized, stemmed, deduplicated +// phrase words; phrase is the plain quoted phrase for callers that degrade; +// win is N, capped at 1000 (wider than any window that could still mean +// anything beyond plain AND). +func parseProximity(q string) (terms []string, phrase string, win int, ok bool) { + q = strings.TrimSpace(q) + if len(q) < 4 || q[0] != '"' { + return nil, "", 0, false + } + end := strings.LastIndexByte(q, '"') + if end == 0 || end+2 >= len(q) || q[end+1] != '~' { + return nil, "", 0, false + } + n, err := strconv.Atoi(q[end+2:]) + if err != nil || n < 1 { + return nil, "", 0, false + } + if n > 1000 { + n = 1000 + } + inner := q[1:end] + seen := map[string]bool{} + for _, t := range tokenize(inner) { + if !seen[t] { + seen[t] = true + terms = append(terms, t) + } + } + return terms, `"` + inner + `"`, n, true +} + +// proximityMatch walks a space-joined stemmed stream once, tracking the most +// recent position of each term; whenever every term has been seen and the +// span from the oldest to the newest of those positions fits the window, a +// valid window exists. Last-seen positions minimize the span at every step, +// so no qualifying window is missed. One linear pass, no string splitting. +func proximityMatch(stream string, terms []string, win int) bool { + if len(terms) == 0 { + return false + } + last := make([]int, len(terms)) + for i := range last { + last[i] = -1 + } + need := len(terms) + pos := 0 + i := 0 + for i < len(stream) { + for i < len(stream) && stream[i] == ' ' { + i++ + } + if i >= len(stream) { + break + } + j := i + for j < len(stream) && stream[j] != ' ' { + j++ + } + tok := stream[i:j] + for t, term := range terms { + if tok != term { + continue + } + if last[t] < 0 { + need-- + } + last[t] = pos + if need == 0 { + lo, hi := last[0], last[0] + for _, p := range last[1:] { + if p < lo { + lo = p + } + if p > hi { + hi = p + } + } + if hi-lo <= win { + return true + } + } + } + pos++ + i = j + } + return false +} + +// --- Snippet helpers --- + +// getSnippets returns up to max non-overlapping +-60-char windows around +// query matches, in document order. The match runs on the normal form +// (folded + lowered) and each window's bounds are resolved back to the +// ORIGINAL text (one forward walk, see below), so accents and casing +// survive into the snippet. +func getSnippets(content, query string, max int) []string { + return snippetsIn(content, normFoldString(content), query, max) +} + +// bodySnippets is getSnippets over the document's transcript, on the fold +// of it the loader already holds. +func (d *Document) bodySnippets(query string, max int) []string { + return snippetsIn(d.FullText, d.foldedBody(), query, max) +} + +// anchorSource yields one needle's matches in a text, in document order and +// one at a time, so a search for three windows reads no further than them. +type anchorSource struct { + next func(from int) (pos, n, resume int) // pos < 0: no match from there on + from int + pos, n int // the match not yet taken; pos < 0 once there is none +} + +func (a *anchorSource) advance() { a.pos, a.n, a.from = a.next(a.from) } + +// snippetsIn is getSnippets for a caller that holds the normal form of +// content (normFoldString). +func snippetsIn(content, norm, query string, max int) []string { + if query == "" || content == "" || max <= 0 { + return nil + } + + searchQuery := strings.TrimSpace(query) + bounded := false // an exact value anchors on whole words only + if v, ok := exactValue(searchQuery); ok { + searchQuery, bounded = `"`+v+`"`, true + } + isExact := isQuotedQuery(searchQuery) + cleanQuery := normFoldString(strings.Trim(searchQuery, `"`)) + if cleanQuery == "" { + return nil + } + + var sources []*anchorSource + add := func(next func(from int) (pos, n, resume int)) { + src := &anchorSource{next: next} + src.advance() + sources = append(sources, src) + } + addAll := func(needle string, wordsOnly bool) { + if needle == "" { + return + } + add(func(from int) (int, int, int) { + i := -1 + if wordsOnly { + i = indexWord(norm, needle, from) + } else if j := strings.Index(norm[from:], needle); j >= 0 { + i = from + j + } + return i, len(needle), i + len(needle) + }) + } + found := func() bool { + for _, src := range sources { + if src.pos >= 0 { + return true + } + } + return false + } + if isExact || strings.Contains(cleanQuery, " ") { + addAll(cleanQuery, !isExact || bounded) + } + if !isExact { + var literal []string // stopwords, single letters, "u.s.": anchors of last resort + for _, w := range strings.Fields(cleanQuery) { + if frag, openStart, openEnd, ok := wildcard(w); ok { + if frag != "" { + add(func(from int) (int, int, int) { + i := indexWild(norm, frag, from, openStart, openEnd) + return i, len(frag), i + len(frag) + }) + } + continue + } + kept := 0 + for _, p := range splitWords(w) { + if len(p) <= 1 || stopWords[p] { + continue + } + kept++ + addAll(p, true) + s := stem(p) + n := 0 + for n < len(s) && n < len(p) && s[n] == p[n] { + n++ + } + if s == p || n < 2 { + continue + } + add(func(from int) (int, int, int) { + for { + i := indexWild(norm, p[:n], from, false, true) + if i < 0 { + return -1, 0, from + } + start, end := wordBounds(norm, i, i+n) + if word := norm[start:end]; word != p && stem(word) == s { + return start, end - start, end + } + from = end + } + }) + } + if kept == 0 { + literal = append(literal, w) + } + } + if !found() { + for _, w := range literal { + parts := splitWords(w) + if len(parts) == 1 { + w = parts[0] + } + addAll(w, len(parts) == 1) + } + } + } + if !found() { + return nil + } + + // First pass. Choose up to max non-overlapping +-60-byte windows, in + // NORM offsets, taking the anchors in document order. + type window struct{ start, end int } + var wins []window + lastEnd := -1 + for len(wins) < max { + var first *anchorSource + for _, src := range sources { + if src.pos >= 0 && (first == nil || src.pos < first.pos) { + first = src + } + } + if first == nil { + break + } + pos, n := first.pos, first.n + first.advance() + if pos < lastEnd { + continue // already inside the previous window + } + start := pos - 60 + if start < 0 { + start = 0 + } + end := pos + n + 60 + if end > len(norm) { + end = len(norm) + } + lastEnd = end + wins = append(wins, window{start, end}) + } + + // Second pass. Resolve those few norm offsets back to offsets in the + // original content with ONE forward walk. A norm offset resolves to the + // start of the rune whose folded form covers it; the offset just past + // the end resolves to len(content). + bounds := make([]int, 0, len(wins)*2) + for _, wnd := range wins { + bounds = append(bounds, wnd.start, wnd.end) + } + sort.Ints(bounds) // window paddings may overlap, so bounds aren't sorted yet + orig := make(map[int]int, len(bounds)) + bi, normOff := 0, 0 + for oi, r := range content { + if bi == len(bounds) { + break + } + l := foldedLen(r) + for bi < len(bounds) && bounds[bi] < normOff+l { + orig[bounds[bi]] = oi + bi++ + } + normOff += l + } + for ; bi < len(bounds); bi++ { + orig[bounds[bi]] = len(content) // the past-the-end boundary + } + + out := make([]string, 0, len(wins)) + for _, wnd := range wins { + oStart, oEnd := orig[wnd.start], orig[wnd.end] + prefix := "... " + if oStart == 0 { + prefix = "" + } + suffix := " ..." + if oEnd == len(content) { + suffix = "" + } + out = append(out, prefix+content[oStart:oEnd]+suffix) + } + return out +} + +// --- Search-result highlighting --- + +// Highlight term kinds +const ( + hlWord = iota // matches whole words + hlWild // marks the whole word a wildcard fragment matched in (tap*, *keys*) + hlPhrase // matches an exact value (=tap) at word boundaries + hlSub // matches raw substrings (quoted text, and punctuated terms) +) + +// hlTerm is one positive query term prepared for markTerms. +type hlTerm struct { + kind int + needle string // in normal form (folded + lowered) + stemmed string // the needle's stem when word stemming applies + openStart, openEnd bool // which sides of the fragment had "*" +} + +// buildHighlightTerms extracts the positive terms of the clause list in the +// shape markTerms consumes, mirroring how the matchers read a clause +// (literalWord, compileWords). A quoted value is one exact phrase, plain +// words match whole words (stem-unified, like the index), "word*" matches +// word prefixes, and words with internal punctuation fall back to substring +// matching. Negated clauses are skipped (their terms are what the results +// DON'T contain). +func buildHighlightTerms(queries, conjs []string) []hlTerm { + var out []hlTerm + seen := make(map[string]bool) + add := func(t hlTerm) { + k := strconv.Itoa(t.kind) + strconv.FormatBool(t.openStart) + strconv.FormatBool(t.openEnd) + "\x00" + t.needle + if t.needle == "" || seen[k] { + return + } + seen[k] = true + out = append(out, t) + } + + for _, q := range positiveTerms(queries, conjs) { + if _, phrase, _, ok := parseProximity(q); ok { + // Proximity clause. The match is word-based, not an exact + // substring, so highlight its individual words (with the + // same stemming eligibility bare words get below). + q = strings.Trim(phrase, `"`) + } + if v, ok := exactValue(q); ok { + add(hlTerm{kind: hlPhrase, needle: normFoldString(v)}) + continue + } + q = normFoldString(strings.TrimSpace(q)) + if q == "" { + continue + } + if isQuotedQuery(q) { + add(hlTerm{kind: hlSub, needle: strings.Trim(q, `"`)}) + continue + } + for _, w := range strings.Fields(q) { + if frag, openStart, openEnd, ok := wildcard(w); ok { + if frag != "" { + add(hlTerm{kind: hlWild, needle: frag, openStart: openStart, openEnd: openEnd}) + } + continue + } + parts := splitWords(w) + if len(parts) != 1 { + // Internal punctuation. The matchers read these as + // substrings, so highlight them the same way. + add(hlTerm{kind: hlSub, needle: w}) + continue + } + w = parts[0] + t := hlTerm{kind: hlWord, needle: w} + // Stem eligibility mirrors tokenize, so "taps" highlights + // "tapped" exactly when the index unified them. + if len(w) > 1 && !stopWords[w] { + t.stemmed = stem(w) + } + add(t) + } + } + return out +} + +// markTerms HTML-escapes s and wraps every occurrence of the given terms in +// tags, for the search results page. +// +// Matching runs on the normal form (normFold) so casing and accents can't +// hide a match, and the marked ranges are resolved back to the ORIGINAL +// text through normFold's position map (e.g. a "muller" query highlights +// "Müller" as written). Word terms match whole words, or words whose stem +// equals the term's stem (the same unification the index does); prefix +// terms match the start of a word; substring terms match anywhere. +func markTerms(s string, terms []hlTerm) template.HTML { + if s == "" || len(terms) == 0 { + return template.HTML(template.HTMLEscapeString(s)) + } + + norm, posMap := normFold(s) + + var wordTerms, subTerms []hlTerm + for _, t := range terms { + if t.kind != hlWord { + subTerms = append(subTerms, t) + } else { + wordTerms = append(wordTerms, t) + } + } + + type span struct{ start, end int } // byte offsets in norm + var spans []span + + if len(wordTerms) > 0 { + needStem := false + for _, t := range wordTerms { + if t.stemmed != "" { + needStem = true + break + } + } + ws := -1 // current word start; -1 = between words + flush := func(we int) { + if ws < 0 { + return + } + word := norm[ws:we] + wstem := "" + if needStem { + wstem = stem(word) + } + for _, t := range wordTerms { + if word == t.needle || (t.stemmed != "" && wstem == t.stemmed) { + spans = append(spans, span{ws, we}) + break + } + } + ws = -1 + } + for i, r := range norm { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + if ws < 0 { + ws = i + } + } else { + flush(i) + } + } + flush(len(norm)) + } + + for _, t := range subTerms { + for from := 0; ; { + i := -1 + switch t.kind { + case hlPhrase: + i = indexWord(norm, t.needle, from) + case hlWild: + i = indexWild(norm, t.needle, from, t.openStart, t.openEnd) + default: + if j := strings.Index(norm[from:], t.needle); j >= 0 { + i = from + j + } + } + if i < 0 { + break + } + start, end := i, i+len(t.needle) + from = end + if t.kind == hlWild { + start, end = wordBounds(norm, start, end) // mark the whole word + } + spans = append(spans, span{start, end}) + } + } + + if len(spans) == 0 { + return template.HTML(template.HTMLEscapeString(s)) + } + + // Merge overlapping and touching spans so nested or adjacent matches + // render as one contiguous highlight. + sort.Slice(spans, func(i, j int) bool { return spans[i].start < spans[j].start }) + merged := spans[:1] + for _, sp := range spans[1:] { + if last := &merged[len(merged)-1]; sp.start <= last.end { + if sp.end > last.end { + last.end = sp.end + } + } else { + merged = append(merged, sp) + } + } + + // Emit escaped gap / / escaped match / runs. Norm + // offsets resolve to original offsets via posMap; when a span boundary + // falls inside one rune's multi-byte fold (e.g. 'ẞ' -> "ss"), the whole + // rune is included. + var b strings.Builder + last := 0 + for _, sp := range merged { + oStart := posMap[sp.start] + j := sp.end + for j < len(norm) && posMap[j] == posMap[sp.end-1] { + j++ + } + oEnd := posMap[j] + if oStart < last { + oStart = last // rune snapping can nudge a span into the previous one + } + if oEnd <= oStart { + continue + } + b.WriteString(template.HTMLEscapeString(s[last:oStart])) + b.WriteString("") + b.WriteString(template.HTMLEscapeString(s[oStart:oEnd])) + b.WriteString("") + last = oEnd + } + b.WriteString(template.HTMLEscapeString(s[last:])) + return template.HTML(b.String()) +} + +// --- Search execution (shared by the HTML page and the JSON API) --- + +// ScoredDoc is one match, with its relevance score. Doc points into archive.Docs, which is +// read-only while serving: the match set is walked, filtered and sorted as +// pointers, and the page a reader sees wraps them in searchResults, which +// carry the highlighting. +type ScoredDoc struct { + Doc *Document + Score float64 +} + +// searchResult is one row of the search page: a document and what the +// query marked in it. +type searchResult struct { + Doc *Document + Title, Desc template.HTML // escaped, query matches in (markTerms) + Snippets []template.HTML // up to 3 match-context windows, -highlighted + VersionSnippets []VersionSnippet // windows from versions' new lines, labelled with their copy + MatchesIn []*DocVersion // versions that matched when the main transcript didn't +} + +const maxSnippetDocs = 100 +const maxSearchClauses = 24 +const maxSearchWords = 64 + +func searchWords(queries []string) int { + n := 0 + for _, q := range queries { + n += len(strings.Fields(q)) + } + return n +} + +// normalizeClauses caps the clause count and pads/truncates fields and +// conjs to the length of queries. +func normalizeClauses(queries, fields, conjs []string) ([]string, []string, []string) { + if len(queries) > maxSearchClauses { + queries = queries[:maxSearchClauses] + } + trimmed := make([]string, len(queries)) + for i, q := range queries { + trimmed[i] = strings.TrimSpace(q) + } + queries = trimmed + if len(fields) > len(queries) { + fields = fields[:len(queries)] + } + for len(fields) < len(queries) { + fields = append(fields, "TE") + } + if len(conjs) > len(queries) { + conjs = conjs[:len(queries)] + } + for len(conjs) < len(queries) { + conjs = append(conjs, "and") + } + return queries, fields, conjs +} + +// searchField describes one searchable field: its wire code (the fqf +// value), UI label, simple-box operator aliases, and what it searches. This +// table is the single source of truth. +type searchField struct { + Code string + Label string + Aliases []string + Stemmed, Folded func(d *Document) string + Values func(d *Document) []string + read func(d *Document) []string + index int // the field's place in searchFields +} + +type docFields struct { + values [][]string + keys [][]string + folded []string + pubYears []string // pubYearsOf +} + +func (co *Corpus) buildDocFields() { + for i := range co.archive.Docs { + d := &co.archive.Docs[i] + d.fields = nil // the readers below work everything out + f := &docFields{ + values: make([][]string, len(searchFields)), + keys: make([][]string, len(searchFields)), + folded: make([]string, len(searchFields)), + pubYears: pubYearsOf(d), + } + for fi := range searchFields { + sf := &searchFields[fi] + if sf.read == nil { + continue + } + vals := sf.read(d) + keys := make([]string, len(vals)) + for vi, v := range vals { + keys[vi] = valueKey(v) + } + f.values[fi], f.keys[fi] = vals, keys + f.folded[fi] = normFoldString(strings.Join(vals, valueSep)) + } + d.fields = f + } +} + +// searchFields is in the form dropdown's display order. +var searchFields = []searchField{ + // SearchText is stored pre-folded, so quoted text and exact words are + // direct tests against it. + {Code: "TE", Label: "All fields", Aliases: []string{"text"}, + Stemmed: func(d *Document) string { return d.StemmedText }, + Folded: func(d *Document) string { return d.SearchText }}, + {Code: "TI", Label: "Title", Aliases: []string{"title"}, + Stemmed: func(d *Document) string { return d.StemmedTitle }, + Folded: (*Document).titleFold}, + {Code: "FT", Label: "Content", Aliases: []string{"content"}, + Stemmed: func(d *Document) string { return d.StemmedFull }, + Folded: func(d *Document) string { return d.FoldedFull }}, + {Code: "DE", Label: "Description", Aliases: []string{"description"}, + Stemmed: func(d *Document) string { return d.StemmedDesc }, + Folded: (*Document).descFold}, + {Code: "CR", Label: "Agency", Aliases: []string{"agency"}, + Values: func(d *Document) []string { return single(d.DocMeta.Creator) }}, + {Code: "RE", Label: "Journalist", Aliases: []string{"journalist"}, + Values: journalistsOf}, + {Code: "PU", Label: "Publisher", Aliases: []string{"publisher"}, + Values: publishersOf}, + {Code: "CL", Label: "Classification", Aliases: []string{"classification"}, + Values: func(d *Document) []string { return d.DocMeta.Classification }}, + {Code: "PR", Label: "Program", Aliases: []string{"program"}, + Values: func(d *Document) []string { return d.DocMeta.Programs }}, + {Code: "DI", Label: "Dissemination", Aliases: []string{"dissemination"}, + Values: func(d *Document) []string { return d.DocMeta.Dissemination }}, + {Code: "BA", Label: "Banner", Aliases: []string{"banner"}, + Values: func(d *Document) []string { return single(d.DocMeta.Banner) }}, + {Code: "GE", Label: "Genre", Aliases: []string{"genre"}, + Values: func(d *Document) []string { return single(d.DocMeta.Genre) }}, + {Code: "SE", Label: "Series", Aliases: []string{"series"}, + Values: func(d *Document) []string { return single(d.DocMeta.Series.Name) }}, + {Code: "CO", Label: "Country", Aliases: []string{"country"}, + Values: func(d *Document) []string { return d.DocMeta.Countries }}, + {Code: "PE", Label: "People", Aliases: []string{"people"}, + Values: func(d *Document) []string { return d.DocMeta.People }}, + {Code: "TA", Label: "Tags", Aliases: []string{"tag"}, + Values: func(d *Document) []string { return d.Tags }}, + {Code: "TY", Label: "Type", Aliases: []string{"type"}, + Values: func(d *Document) []string { return single(d.Type) }}, + {Code: "PL", Label: "Portion level", + Values: func(d *Document) []string { return d.PortionLevels }}, + {Code: "PM", Label: "Portion marking", + Values: func(d *Document) []string { return d.PortionMarkings }}, + {Code: "PC", Label: "Portion control", + Values: func(d *Document) []string { return d.PortionControls }}, +} + +func single(v string) []string { + if v == "" { + return nil + } + return []string{v} +} + +var ( + fieldByCode = map[string]*searchField{} + simpleOps = map[string]string{} +) + +func init() { + for i := range searchFields { + f := &searchFields[i] + f.index = i + if read := f.Values; read != nil { + f.read = read + f.Values = func(d *Document) []string { + if d.fields != nil { + return d.fields.values[i] + } + return read(d) + } + } + fieldByCode[f.Code] = f + for _, a := range f.Aliases { + simpleOps[a] = f.Code + } + } +} + +type simpleDates struct { + DocStart, DocEnd, PubStart, PubEnd string +} + +// padDateEnd widens a partial date for use as an inclusive range end: +// 2013 -> 2013-12-31, 2013-06 -> 2013-06-31 (string comparison makes the +// over-wide day harmless). +func padDateEnd(v string) string { + switch len(v) { + case 4: + return v + "-12-31" + case 7: + return v + "-31" + } + return v +} + +// parseSimpleQuery turns a one-line query into fielded clauses plus date +// filters. +// +// program:XKEYSCORE country:Germany cable tapping +// +// becomes three AND clauses: PR=XKEYSCORE, CO=Germany, TE="cable tapping". +// Values can be double-quoted (journalist:"Poitras, Laura"); the quotes are +// KEPT on the clause value, so a quoted value has the same exact-match +// (substring/phrase) semantics it has when typed into the structured form, +// and a bare "quoted phrase" becomes its own exact all-fields clause rather +// than dissolving into the free-text words. +// +// A leading '-' negates: -term and -program:X become "and not" clauses +// (ordered after the positive ones, since the boolean evaluation is +// left-to-right). date:2013, date:2013-06, date:2013..2015 filter on the +// document date; published: does the same for publication dates. +func parseSimpleQuery(s string) (queries, fields, conjs []string, dates simpleDates) { + var freeText []string + type clause struct{ q, f string } + var negs []clause + + for i, neg, ok := nextTerm(s, 0); ok; i, neg, ok = nextTerm(s, i) { + // try operator: word characters followed by ':' + opEnd := i + for opEnd < len(s) && (isWordByte(s[opEnd])) { + opEnd++ + } + field := "" + dateOp := "" + valStart := i + if opEnd > i && opEnd < len(s) && s[opEnd] == ':' { + name := strings.ToLower(s[i:opEnd]) + if code, ok := simpleOps[name]; ok { + field = code + valStart = opEnd + 1 + } else if name == "date" || name == "published" { + dateOp = name + valStart = opEnd + 1 + } + } + + clauseVal, val, quoted, exact, next := readValue(s, valStart) + i = next + if val == "" { + continue + } + + if dateOp != "" { + // date filters can't be negated; '-' before them is ignored. + start, end := val, val + if a, b, ok := strings.Cut(val, ".."); ok { + start, end = strings.TrimSpace(a), strings.TrimSpace(b) + } + end = padDateEnd(end) + if dateOp == "date" { + dates.DocStart, dates.DocEnd = start, end + } else { + dates.PubStart, dates.PubEnd = start, end + } + continue + } + + val = clauseVal + + switch { + case field != "" && neg: + negs = append(negs, clause{val, field}) + case field != "": + queries = append(queries, val) + fields = append(fields, field) + case neg: + negs = append(negs, clause{val, "TE"}) + case quoted || exact: + queries = append(queries, val) + fields = append(fields, "TE") + default: + freeText = append(freeText, val) + } + } + + if len(freeText) > 0 { + queries = append(queries, strings.Join(freeText, " ")) + fields = append(fields, "TE") + } + if len(queries) == 0 { + queries = []string{""} + fields = []string{"TE"} + } + conjs = make([]string, len(queries)) + for i := range conjs { + conjs[i] = "and" + } + for _, n := range negs { + queries = append(queries, n.q) + fields = append(fields, n.f) + conjs = append(conjs, "not") + } + return queries, fields, conjs, dates +} + +func isWordByte(b byte) bool { + return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_' +} + +const spaceBytes = " \t\n\v\f\r" + +func isSpaceByte(b byte) bool { return strings.IndexByte(spaceBytes, b) >= 0 } + +func proxSuffix(s string, i int) (string, int) { + if i >= len(s) || s[i] != '~' { + return "", i + } + j := i + 1 + for j < len(s) && s[j] >= '0' && s[j] <= '9' { + j++ + } + if j == i+1 || (j < len(s) && !isSpaceByte(s[j])) { + return "", i + } + return s[i:j], j +} + +// readToken reads one search-value token starting at s[i]: either a +// "quoted span" returned with the quotes stripped and quoted=true, or +// a bare run up to the next whitespace. next is the index just past the +// token. This is THE tokenizer for search values: parseSimpleQuery and +// rowTerms both use it, behind nextTerm, so neither the quoting rule nor +// the negation rule can drift between them. +func readToken(s string, i int) (val string, quoted bool, next int) { + if i < len(s) && s[i] == '"' { + j := i + 1 + if k := strings.IndexByte(s[j:], '"'); k >= 0 { + return s[j : j+k], true, j + k + 1 + } + return s[i+1:], true, len(s) + } + j := i + for j < len(s) && !isSpaceByte(s[j]) { + j++ + } + return s[i:j], false, j +} + +// nextTerm finds the next term of a query from s[i] on: it skips the +// whitespace, and a leading '-' negates the term when it is attached to +// one. i is where the term's value (or its field: operator) starts; ok is +// false at the end of the string. +func nextTerm(s string, i int) (start int, neg, ok bool) { + for i < len(s) && isSpaceByte(s[i]) { + i++ + } + if i >= len(s) { + return i, false, false + } + if s[i] == '-' && i+1 < len(s) && !isSpaceByte(s[i+1]) { + return i + 1, true, true + } + return i, false, true +} + +// readValue reads one search value at s[i] +func readValue(s string, i int) (clause, inner string, quoted, exact bool, next int) { + if i < len(s) && s[i] == '=' { + exact = true + i++ + } + inner, quoted, next = readToken(s, i) + prox := "" + if quoted { + prox, next = proxSuffix(s, next) + } + inner = strings.TrimSpace(inner) + switch { + case inner == "": + case exact && quoted: + clause = `="` + inner + `"` + case exact: + clause = "=" + inner + case quoted: + clause = `"` + inner + `"` + prox + default: + clause = inner + } + return clause, inner, quoted, exact, next +} + +// rowTerms splits one search row into what it requires and what it +// excludes, the way parseSimpleQuery splits ?q=: each quoted phrase, +// proximity clause or exact value is a positive part of its own, the bare +// words together are one more (last), and google-style -term and -"phrase" +// tokens are exclusions on the row's own field. +// +// This is THE reading of a row: the matcher, the scorer, the snippet anchor +// and the highlighter all go through it, so "beta -gamma" is (beta AND NOT +// gamma) as one unit wherever the row sits. +func rowTerms(s string) (positives, negatives []string) { + var words []string + for i, neg, ok := nextTerm(s, 0); ok; i, neg, ok = nextTerm(s, i) { + tok, _, quoted, exact, next := readValue(s, i) + i = next + switch { + case tok == "" || tok == "-": + case neg: + negatives = append(negatives, tok) + case quoted || exact: + positives = append(positives, tok) + default: + words = append(words, tok) + } + } + if len(words) > 0 { + positives = append(positives, strings.Join(words, " ")) + } + return positives, negatives +} + +// executeSearch is the search pipeline the page and the API both run: over +// the whole archive, the collection filter, the dates and the boolean +// clause list, then the type filter, and the survivors ranked and sorted +// by p.Sort/p.Order, as a fresh slice the caller owns. The two filters are +// the browse pages' (inCollections, typeSelected). typeCounts tallies the +// types BEFORE the type filter. Only the documents that pass every filter +// are scored. +func (co *Corpus) executeSearch(p searchParams) (matches []ScoredDoc, typeCounts map[string]int) { + dateDocEnd := padDateEnd(p.DateDocEnd) + datePubEnd := padDateEnd(p.DatePubEnd) + + query := compileQuery(p.Queries, p.Fields, p.Conjs) + byEntry := entryRows(p.Queries, p.Fields, p.Conjs) + var scorers []func(*Document) float64 + for _, q := range positiveTerms(p.Queries, p.Conjs) { + scorers = append(scorers, co.tfidf.scorer(q)) + } + typeCounts = make(map[string]int) + for i := range co.archive.Docs { + doc := &co.archive.Docs[i] + if !inCollections(doc, p.Coll) || + !matchDates(doc, p.DateDocStart, dateDocEnd, p.DatePubStart, datePubEnd, byEntry) || + !query.matches(doc) { + continue + } + typeCounts[doc.Type]++ + if !typeSelected(doc, p.Types) { + continue + } + totalScore := 0.0 + for _, score := range scorers { + totalScore += score(doc) + } + matches = append(matches, ScoredDoc{Doc: doc, Score: totalScore}) + } + + slices.SortStableFunc(matches, func(a, b ScoredDoc) int { return cmp.Compare(b.Score, a.Score) }) + + return sortScored(matches, p.Sort, p.Order), typeCounts +} + +// --- Search handler (HTML) --- + +// --- Faceted search refinement ----------------------------------------------- + +type searchFacet struct { + Label string + Values []facetEntry +} + +func topFacetCounts(m map[string]int, total, n int, baseURL, field string) []facetEntry { + var out []facetEntry + for _, v := range rankCounts(m, -1) { + if len(out) == n { + break + } + if m[v] >= total { + continue + } + // ="value" -> a whole-value match on the field, the equality the + // tally counted, so the number is the number the click finds. + out = append(out, facetEntry{Name: v, Count: m[v], URL: appendClause(baseURL, field, exactClause(v))}) + } + return out +} + +func facetValues(d *Document, code string) []string { + if f := fieldByCode[code]; f != nil && f.Values != nil { + return f.Values(d) + } + return nil +} + +type facetGroup struct { + Label string + Code string +} + +func facetGroups(docs []*Document, groups []facetGroup, baseURL string, values func(*Document, string) []string) []searchFacet { + type tally struct{ count, first int } + tallies := make([]map[string]*tally, len(groups)) + for i := range tallies { + tallies[i] = map[string]*tally{} + } + seq := 0 + for i := range docs { + for gi, g := range groups { + for _, v := range values(docs[i], g.Code) { + t := tallies[gi][v] + if t == nil { + t = &tally{first: seq} + tallies[gi][v] = t + } + t.count++ + seq++ + } + } + } + out := make([]searchFacet, 0, len(groups)) + for gi, g := range groups { + type folded struct { + name string + count, first int + } + byKey := make(map[string]*folded, len(tallies[gi])) + for v, t := range tallies[gi] { + k := valueKey(v) + f := byKey[k] + if f == nil { + byKey[k] = &folded{v, t.count, t.first} + continue + } + f.count += t.count + if t.first < f.first { + f.name, f.first = v, t.first + } + } + counts := make(map[string]int, len(byKey)) + for _, f := range byKey { + counts[f.name] = f.count + } + if vals := topFacetCounts(counts, len(docs), 8, baseURL, g.Code); len(vals) > 0 { + out = append(out, searchFacet{Label: g.Label, Values: vals}) + } + } + return out +} + +func facetsWithinWords(facets []searchFacet, room int) []searchFacet { + out := facets[:0] + for _, f := range facets { + vals := f.Values[:0] + for _, v := range f.Values { + if searchWords([]string{exactClause(v.Name)}) <= room { + vals = append(vals, v) + } + } + if len(vals) > 0 { + f.Values = vals + out = append(out, f) + } + } + return out +} + +func computeSearchFacets(docs []*Document, baseURL string, values func(*Document, string) []string) []searchFacet { + return facetGroups(docs, []facetGroup{ + {"Programs", "PR"}, {"Agencies", "CR"}, {"Countries", "CO"}, {"Journalists", "RE"}, + }, baseURL, values) +} + +func collectionFilter(v url.Values) map[string]bool { + sel := map[string]bool{} + for _, c := range v["coll"] { + if validCollections[c] { + sel[c] = true + } + } + if len(sel) == 0 || len(sel) == len(validCollections) { + return nil + } + return sel +} + +func snippetAnchor(queries, fields, conjs []string) (term, field string) { + for i, q := range queries { + if q == "" || conjs[i] == "not" { + continue + } + pos, _ := rowTerms(q) + if len(pos) == 0 { + continue // only exclusions: nothing in the results to anchor on + } + q = pos[0] + if _, phrase, _, ok := parseProximity(q); ok { + if ws := strings.Fields(strings.Trim(phrase, `"`)); len(ws) > 0 { + return ws[0], fields[i] + } + } + return q, fields[i] + } + return "", "" +} + +// positiveTerms is what the query asks the results to contain: each +// non-empty row's positive part, skipping NOT-rows and every row's +// exclusions (rowTerms). The scorer and the highlighter read the query +// through it, so neither rewards or marks a term the results were selected +// for NOT containing. +func positiveTerms(queries, conjs []string) []string { + var out []string + for i, q := range queries { + if q == "" || (i < len(conjs) && conjs[i] == "not") { + continue + } + pos, _ := rowTerms(q) + out = append(out, pos...) + } + return out +} + +// searchParams is one search's effective inputs as read by readSearchParams. +type searchParams struct { + Queries, Fields, Conjs []string + DateDocStart, DateDocEnd, DatePubStart, DatePubEnd string + FromQ bool // a ?q= shortcut replaced the structured params + Coll map[string]bool // collectionFilter: nil filters nothing + Types map[string]bool // selectedTypes: empty filters nothing + Sort, Order string // Sort defaults to "relevance" + OverClauses int // the requested clause count when it exceeds maxSearchClauses: the search must not run + OverWords int // the word count (searchWords) when it exceeds maxSearchWords: the search must not run +} + +func (p searchParams) refused() bool { return p.OverClauses > 0 || p.OverWords > 0 } + +// readSearchParams reads a search from URL params. this intake is shared by +// the HTML page and the JSON API. +// +// A non-empty ?q= replaces any structured clause params. Plain words become +// a single all-fields clause. Field operators like program:XKEYSCORE or +// journalist:"Poitras, Laura" become their own AND clauses (see +// parseSimpleQuery). Its parsed dates override the URL's when either end is +// set - date:..2015 has an empty start but is still a real filter. +func readSearchParams(v url.Values) searchParams { + p := searchParams{ + Queries: v["fqv"], + Fields: v["fqf"], + Conjs: v["fqc"], + DateDocStart: v.Get("date_doc_start"), + DateDocEnd: v.Get("date_doc_end"), + DatePubStart: v.Get("date_pub_start"), + DatePubEnd: v.Get("date_pub_end"), + Coll: collectionFilter(v), + Sort: sortToken(v.Get("sort")), + Order: sortToken(v.Get("order")), + } + p.Types, _ = selectedTypes(v) + if docSortByKey[p.Sort] == nil { + p.Sort, p.Order = "relevance", "" + } + if s := v.Get("q"); s != "" { + var sd simpleDates + p.Queries, p.Fields, p.Conjs, sd = parseSimpleQuery(s) + if sd.DocStart != "" || sd.DocEnd != "" { + p.DateDocStart, p.DateDocEnd = sd.DocStart, sd.DocEnd + } + if sd.PubStart != "" || sd.PubEnd != "" { + p.DatePubStart, p.DatePubEnd = sd.PubStart, sd.PubEnd + } + p.FromQ = true + } + if n := len(p.Queries); n > maxSearchClauses { + p.OverClauses = n + } else if n := searchWords(p.Queries); n > maxSearchWords { + p.OverWords = n + } + p.Queries, p.Fields, p.Conjs = normalizeClauses(p.Queries, p.Fields, p.Conjs) + return p +} + +func (p searchParams) dates() [4]string { + return [4]string{p.DateDocStart, p.DateDocEnd, p.DatePubStart, p.DatePubEnd} +} + +// canonicalSearchQuery rebuilds the query params from the EFFECTIVE state +// of a search (the post-add/remove clause rows, the collection and type +// filters, the dates that a ?q= shortcut may have parsed out of the box) +// for every link the page generates (sorting, pagination, the histogram +// bars, the refine box, the export and the recent/pinned entries). +func canonicalSearchQuery(doSearch bool, p searchParams) url.Values { + q := url.Values{} + if doSearch { + // A ?q= entry has no do_search param, so the canonical form adds it + // explicitly. + q.Set("do_search", "1") + } + for i := range p.Queries { + q.Add("fqv", p.Queries[i]) + q.Add("fqf", p.Fields[i]) + q.Add("fqc", p.Conjs[i]) + } + if colls := collectionParams(p.Coll); len(colls) > 0 { + q["coll"] = colls + } + if len(p.Types) > 0 { + types := make([]string, 0, len(p.Types)) + for t := range p.Types { + types = append(types, t) + } + sort.Strings(types) + q["type"] = types + } + dates := p.dates() + for i, k := range []string{"date_doc_start", "date_doc_end", "date_pub_start", "date_pub_end"} { + if dates[i] != "" { + q.Set(k, dates[i]) + } + } + return q +} + +// searchDateHistogram is the year-bar strip under a result set. +func searchDateHistogram(results []*Document, q url.Values) []timelineBar { + counts := make(map[string]int) + undated := 0 + for i := range results { + if y := docYear(results[i]); y != "" { + counts[y]++ + } else { + undated++ + } + } + _, maxCount := sortedYears(counts) + if undated > maxCount { + maxCount = undated // the undated bar shares the scale + } + start, end := q.Get("date_doc_start"), q.Get("date_doc_end") + bars := yearBars(counts, func(y string) string { + from, to := y, y + if start > y { + from = start + } + if end != "" && padDateEnd(end) < padDateEnd(y) { + to = end + } + yq := cloneValues(q) + yq.Set("date_doc_start", from) + yq.Set("date_doc_end", to) + return "/search?" + yq.Encode() + }) + for i := range bars { + bars[i].Pct = barPct(bars[i].Count, maxCount, 1) + } + if undated > 0 { + bars = append(bars, timelineBar{ + Year: "-", + Count: undated, + Pct: barPct(undated, maxCount, 1), + }) + } + return bars +} + +// decorateResults marks the query's terms in the titles, descriptions and +// body snippets of the results on the page. +func decorateResults(docs []*Document, queries, fields, conjs []string) (results []searchResult, snippetsCapped bool) { + snippetTerm, snippetField := snippetAnchor(queries, fields, conjs) + hl := buildHighlightTerms(queries, conjs) + results = make([]searchResult, len(docs)) + for i, doc := range docs { + res := &results[i] + res.Doc = doc + res.Title = markTerms(doc.Title, hl) + res.Desc = markTerms(doc.Description, hl) + if snippetTerm == "" || i >= maxSnippetDocs { + continue + } + var raw []string + var hits []versionHit + mainHit := false + if snippetField == "TA" { + raw = getSnippets(strings.Join(doc.Tags, ", "), snippetTerm, 3) + } else { + raw = doc.bodySnippets(snippetTerm, 3) + mainHit = len(raw) > 0 + if len(raw) < 3 { + hits = versionHits(doc, snippetTerm, 3) + } + if !mainHit && len(hits) == 0 { + raw = getSnippets(doc.Description, snippetTerm, 1) + } + } + for _, s := range raw { + res.Snippets = append(res.Snippets, markTerms(s, hl)) + } + room := 3 - len(raw) + for _, h := range hits { + if !mainHit { + res.MatchesIn = append(res.MatchesIn, h.Version) + } + for _, s := range h.Snippets { + if room <= 0 { + break + } + res.VersionSnippets = append(res.VersionSnippets, + VersionSnippet{Slug: h.Version.Slug, Label: h.Version.Label, HTML: markTerms(s, hl)}) + room-- + } + } + } + return results, snippetTerm != "" && len(docs) > maxSnippetDocs +} + +func (co *Corpus) handleSearch(w http.ResponseWriter, r *http.Request) { + params := r.URL.Query() + action := params.Get("action") + sp := readSearchParams(params) + prefs := readPrefs(r) + scope := pageScope(params, prefs, true) + sp.Coll = scope.filter + queries, fields, conjs := sp.Queries, sp.Fields, sp.Conjs + sortBy, order := sp.Sort, sp.Order + + doSearch := (params.Get("do_search") == "1" || sp.FromQ) && !sp.refused() + + if len(queries) == 0 { + queries = []string{""} + fields = []string{"TE"} + conjs = []string{"and"} + } + + switch { + case strings.HasPrefix(action, "remove-"): + idxStr := strings.TrimPrefix(action, "remove-") + removeIdx, err := strconv.Atoi(idxStr) + if err == nil && removeIdx > 0 && removeIdx < len(queries) { + queries = append(queries[:removeIdx], queries[removeIdx+1:]...) + fields = append(fields[:removeIdx], fields[removeIdx+1:]...) + conjs = append(conjs[:removeIdx], conjs[removeIdx+1:]...) + } + case action == "add" && len(queries) < maxSearchClauses: + queries = append(queries, "") + fields = append(fields, "TE") + conjs = append(conjs, "and") + } + + sp.Queries, sp.Fields, sp.Conjs = queries, fields, conjs + + var scored []ScoredDoc + searchTypeCounts := map[string]int{} + if doSearch { + scored, searchTypeCounts = co.executeSearch(sp) + } + + selColl := scope.shown + collFiltered := scope.explicit() + selTypeSet := sp.Types + + results := make([]*Document, 0, len(scored)) + for _, s := range scored { + results = append(results, s.Doc) + } + + q := canonicalSearchQuery(doSearch, sp) + q["coll"] = scope.colls + canonicalQS := q.Encode() + searchBaseURL := "/search?" + canonicalQS + "&" + + formParams := cloneValues(q) + if sortBy != "relevance" { + formParams.Set("sort", sortBy) + } + if order != "" { + formParams.Set("order", order) + } + + page, _ := strconv.Atoi(params.Get("p")) + start, end, page, totalPages := paginate(len(results), page, prefs.cardsPerPage()) + pageResults, snippetsCapped := decorateResults(results[start:end], queries, fields, conjs) + + labelQ := q + if scope.defaulted { + labelQ = cloneValues(q) + delete(labelQ, "coll") + } + queryLabel, named := "", false // the page title's name for the search + if doSearch { + queryLabel, named = co.searchLabel(labelQ) + } + savable := named && len(canonicalQS) <= searchQSMax + var recorded []string + if prefs.TrackSearches && savable && + (action == "" || action == "search") && !crossSiteFetch(r) { + recorded = recordRecentSearch(w, r, canonicalQS) + } + var recentSearches []recentSearch + if prefs.TrackSearches { + if recorded != nil { + recentSearches = co.parseSearchEntries(recorded) + } else { + recentSearches = co.readRecentSearches(r) + } + } + + searchPinned := savable && isPinnedSearch(r, canonicalQS) + + var dateHist []timelineBar + if doSearch && len(results) > 0 { + dateHist = searchDateHistogram(results, q) + } + + var facets []searchFacet + if doSearch && len(results) >= 2 && len(queries) < maxSearchClauses { + values := facetValues + if !hasOrRow(queries, conjs) { + values = refineValues(sp.DatePubStart, sp.DatePubEnd) + } + facets = facetsWithinWords(computeSearchFacets(results, searchBaseURL, values), + maxSearchWords-searchWords(queries)) + } + + if len(recentSearches) > 0 { + noStore(w) + } + render(w, r, &searchPage{ + FieldOptions: searchFields, + Facets: facets, + Queries: queries, + Fields: fields, + Conjs: conjs, + Results: pageResults, + SearchTypeCounts: searchTypeCounts, + SelTypeSet: selTypeSet, + FormParams: formParams, + Savable: savable, + ShowThumbs: prefs.thumbsOn(), + FOIAQS: prefs.foiaLinkQS(), + Base: pageWith(prefs), + Total: len(results), + Page: page, + TotalPages: totalPages, + RecentSearches: recentSearches, + SelColl: selColl, + CollFiltered: collFiltered, + CanonicalQS: canonicalQS, + SearchPinned: searchPinned, + Searched: doSearch, + QueryLabel: queryLabel, + DateDocStart: sp.DateDocStart, + DateDocEnd: sp.DateDocEnd, + DatePubStart: sp.DatePubStart, + DatePubEnd: sp.DatePubEnd, + Sort: sortBy, + Order: order, + SearchBaseURL: searchBaseURL, + DateHist: dateHist, + SnippetsCapped: snippetsCapped, + SnippetLimit: maxSnippetDocs, + OverClauses: sp.OverClauses, + ClauseLimit: maxSearchClauses, + OverWords: sp.OverWords, + WordLimit: maxSearchWords, + }) +} + +// --- Date helpers --- + +func inDateRange(date, start, end string) bool { + if date == "" { + return start == "" && end == "" + } + if start != "" && date < start { + return false + } + if end != "" && date > end { + return false + } + return true +} + +func matchDates(doc *Document, dateDocStart, dateDocEnd, datePubStart, datePubEnd string, byEntry []entryRow) bool { + if dateDocStart != "" || dateDocEnd != "" { + if !inDateRange(doc.DocMeta.DateCreated, dateDocStart, dateDocEnd) { + return false + } + } + if datePubStart == "" && datePubEnd == "" { + return true + } + inRange := func(i int) bool { + return inDateRange(doc.Publication[i].DatePublished, datePubStart, datePubEnd) + } + if len(byEntry) == 0 { + for i := range doc.Publication { + if inRange(i) { + return true + } + } + return false + } + for _, r := range byEntry { + found := false + for i := range doc.Publication { + if inRange(i) && r.matches(doc, i) { + found = true + break + } + } + if !found { + return false + } + } + return true +} + +// entryRow is a journalist or publisher row a publication date is tied +// to, parsed once per search. +type entryRow struct { + field string // "RE" or "PU" + pos, negs []valuesTest +} + +func entryRows(queries, fields, conjs []string) []entryRow { + if hasOrRow(queries, conjs) { + return nil + } + var rows []entryRow + for i, q := range queries { + if q == "" { + continue + } + if conjs[i] != "not" && (fields[i] == "RE" || fields[i] == "PU") { + pos, negs := rowTerms(q) + rows = append(rows, entryRow{fields[i], entryTests(pos), entryTests(negs)}) + } + } + return rows +} + +func hasOrRow(queries, conjs []string) bool { + for i, q := range queries { + if q != "" && conjs[i] == "or" { + return true + } + } + return false +} + +func entryTests(terms []string) []valuesTest { + tests := make([]valuesTest, len(terms)) + for i, t := range terms { + if _, phrase, _, ok := parseProximity(t); ok { + t = phrase + } + tests[i] = compileValues(t) + } + return tests +} + +func entryValues(doc *Document, i int, field string) []string { + p := &doc.Publication[i] + switch { + case field == "RE": + return p.Journalists + case !p.ReportingOnly: + return []string{p.Publisher} + } + return nil +} + +func refineValues(pubStart, pubEnd string) func(*Document, string) []string { + if pubStart == "" && pubEnd == "" { + return facetValues + } + pubEnd = padDateEnd(pubEnd) + return func(d *Document, code string) []string { + if code != "RE" && code != "PU" { + return facetValues(d, code) + } + var vals []string + for i := range d.Publication { + if inDateRange(d.Publication[i].DatePublished, pubStart, pubEnd) { + vals = append(vals, entryValues(d, i, code)...) + } + } + return normalizeList(vals) + } +} + +// matches reports whether publication entry i matches the row on its own +// journalists or publisher (entryValues). +func (r entryRow) matches(doc *Document, i int) bool { + vals := entryValues(doc, i, r.field) + for t := range r.pos { + if !r.pos[t].match(vals) { + return false + } + } + for t := range r.negs { + if r.negs[t].match(vals) { + return false + } + } + return true +} + +// --- The boolean clause list --- + +// compiledQuery is a search's rows read once. The non-empty ones, in order, +// each with the tests of its positive values and of its exclusions on its +// field. +type compiledQuery []compiledRow + +type compiledRow struct { + conj string + pos, negs []func(*Document) bool +} + +func compileQuery(queries, fields, conjs []string) compiledQuery { + var rows compiledQuery + for i, q := range queries { + if q == "" { + continue + } + f := fieldByCode[fields[i]] + if f == nil { + f = fieldByCode["TE"] + } + row := compiledRow{conj: conjs[i]} + pos, negs := rowTerms(q) + for _, v := range pos { + row.pos = append(row.pos, f.compile(v)) + } + for _, v := range negs { + row.negs = append(row.negs, f.compile(v)) + } + rows = append(rows, row) + } + return rows +} + +// matches evaluates the rows against a document, left to right, each row +// as one unit. +func (rows compiledQuery) matches(d *Document) bool { + overall := true + for i := range rows { + row := &rows[i] + if i == 0 { + overall = row.matches(d) != (row.conj == "not") + continue + } + switch row.conj { + case "or": + overall = overall || row.matches(d) + case "not": + overall = overall && !row.matches(d) + default: // "and" + overall = overall && row.matches(d) + } + } + return overall +} + +// matches reports whether a document matches one row. +func (row *compiledRow) matches(d *Document) bool { + for _, test := range row.pos { + if !test(d) { + return false + } + } + for _, test := range row.negs { + if test(d) { + return false + } + } + return true +} + +// versionHit is one of a document's versions whose new lines match a query, +// with snippet windows from those lines. +type versionHit struct { + Version *DocVersion + Snippets []string +} + +// versionHits returns the versions of doc whose new lines (NovelText) match +// term in version order. +func versionHits(doc *Document, term string, max int) []versionHit { + var out []versionHit + for i := range doc.Versions { + v := &doc.Versions[i] + if s := getSnippets(v.NovelText, term, max); len(s) > 0 { + out = append(out, versionHit{Version: v, Snippets: s}) + } + } + return out +} diff --git a/seo.go b/seo.go new file mode 100644 index 0000000..7a26889 --- /dev/null +++ b/seo.go @@ -0,0 +1,233 @@ +// seo.go - robots.txt, sitemap.xml, and the per-document metadata embeds. + +package main + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "fmt" + "html/template" + "net/http" + "net/url" + "strings" +) + +var canonicalBase string + +func requestBase(r *http.Request) string { + if canonicalBase != "" { + return canonicalBase + } + scheme := "http" + if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { + scheme = "https" + } + return scheme + "://" + r.Host +} + +func handleRobots(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "public, max-age=86400") + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + fmt.Fprintf(w, `User-agent: * +Allow: / +Disallow: /search +Disallow: /api/ +Disallow: /random +Disallow: /pinned +Disallow: /preferences + +Sitemap: %s/sitemap.xml +`, requestBase(r)) +} + +type sitemapURL struct { + Loc string `xml:"loc"` +} + +type sitemapSet struct { + XMLName xml.Name `xml:"urlset"` + Xmlns string `xml:"xmlns,attr"` + URLs []sitemapURL `xml:"url"` +} + +func (co *Corpus) buildSitemapPaths() { + paths := []string{ + "/", "/browse", "/articles", "/glossary", "/changelog", "/about", + } + for _, c := range categories { + paths = append(paths, "/browse/"+c.Slug) + } + paths = append(paths, "/explore") + for _, m := range exploreMetrics { + paths = append(paths, "/explore/"+m.Slug) + } + paths = append(paths, "/explore/terms") + paths = append(paths, "/explore/classification") + paths = append(paths, "/explore/portions") + paths = append(paths, "/explore/outlets") + if len(co.assemblies) > 0 { + paths = append(paths, assembliesPath) + for _, a := range co.assemblies { + paths = append(paths, assemblyPath(a.ID)) + } + } + for _, d := range co.archive.Docs { + paths = append(paths, docPath(d.ID)) + } + for _, a := range co.articles { + if a.Title != "" { + paths = append(paths, "/article/"+a.Key) + } + } + + co.sitemapPaths = paths +} + +func (co *Corpus) marshalSitemap(base string) ([]byte, error) { + set := sitemapSet{Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9"} + set.URLs = make([]sitemapURL, len(co.sitemapPaths)) + for i, p := range co.sitemapPaths { + set.URLs[i] = sitemapURL{Loc: base + p} + } + out, err := xml.MarshalIndent(set, "", " ") + if err != nil { + return nil, err + } + return append([]byte(xml.Header), out...), nil +} + +func parseCanonicalBase(base string) (string, error) { + u, err := url.Parse(base) + if err != nil { + return "", fmt.Errorf("invalid base URL %q: %w", base, err) + } + if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || + (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.Fragment != "" { + return "", fmt.Errorf("base must be scheme://host with no path/query/fragment/userinfo, e.g. https://archive.example.org (got %q)", base) + } + return u.Scheme + "://" + u.Host, nil +} + +const sitemapBaseMark = "{base url}" + +func (co *Corpus) buildCanonicalSitemap() { + co.sitemapParts = nil + if canonicalBase == "" { + if b, err := co.marshalSitemap(sitemapBaseMark); err == nil { + co.sitemapParts = bytes.Split(b, []byte(sitemapBaseMark)) + } + return + } + b, err := co.marshalSitemap(canonicalBase) + if err != nil { + errorf("sitemap: %v - building it per request instead", err) + return + } + co.canonicalSitemap = b +} + +func (co *Corpus) handleSitemap(w http.ResponseWriter, r *http.Request) { + body := co.canonicalSitemap + var base bytes.Buffer + if body == nil && co.sitemapParts != nil { + xml.EscapeText(&base, []byte(requestBase(r))) + } else if body == nil { + var err error + body, err = co.marshalSitemap(requestBase(r)) + if err != nil { + serverError(w, err) + return + } + } + w.Header().Set("Cache-Control", "public, max-age=3600") + w.Header().Set("Content-Type", "application/xml; charset=utf-8") + if body != nil { + w.Write(body) + return + } + for i, part := range co.sitemapParts { + if i > 0 { + w.Write(base.Bytes()) + } + w.Write(part) + } +} + +// docEmbeds is handed to doc.html as data["Embeds"]. +type docEmbeds struct { + JSONLDTag template.HTML + COinS string +} + +func buildDocEmbeds(doc *Document, absURL, pdfURL string) docEmbeds { + type entity struct { + Type string `json:"@type"` + Name string `json:"name"` + } + type media struct { + Type string `json:"@type"` + ContentURL string `json:"contentUrl"` + EncodingFormat string `json:"encodingFormat"` + } + keywords := strings.Join(append(append([]string{}, doc.DocMeta.Programs...), doc.Tags...), ", ") + ld := struct { + Context string `json:"@context"` + Type string `json:"@type"` + Name string `json:"name"` + URL string `json:"url"` + Identifier string `json:"identifier"` + Description string `json:"description,omitempty"` + DateCreated string `json:"dateCreated,omitempty"` + DatePublished string `json:"datePublished,omitempty"` + InLanguage string `json:"inLanguage"` + Keywords string `json:"keywords,omitempty"` + Author *entity `json:"author,omitempty"` + Publisher entity `json:"publisher"` + AssociatedMedia media `json:"associatedMedia"` + }{ + Context: "https://schema.org", + Type: "DigitalDocument", + Name: doc.Title, + URL: absURL, + Identifier: absURL, + Description: doc.Description, + DateCreated: doc.DocMeta.DateCreated, + DatePublished: oldestPubDate(doc), + InLanguage: "en", + Keywords: keywords, + Publisher: entity{Type: "Organization", Name: "Snowden Archive"}, + AssociatedMedia: media{ + Type: "MediaObject", + ContentURL: pdfURL, + EncodingFormat: "application/pdf", + }, + } + if doc.DocMeta.Creator != "" { + ld.Author = &entity{Type: "Organization", Name: doc.DocMeta.Creator} + } + + var tag template.HTML + if b, err := json.Marshal(ld); err == nil { + tag = template.HTML(``) + } + + v := url.Values{} + v.Set("ctx_ver", "Z39.88-2004") + v.Set("rft_val_fmt", "info:ofi/fmt:kev:mtx:dc") + v.Set("rft.title", doc.Title) + if doc.DocMeta.Creator != "" { + v.Set("rft.creator", doc.DocMeta.Creator) + } + if doc.DocMeta.DateCreated != "" { + v.Set("rft.date", doc.DocMeta.DateCreated) + } + v.Set("rft.type", "Text") + v.Set("rft.format", "application/pdf") + v.Set("rft.identifier", absURL) + v.Set("rft.publisher", "Snowden Archive") + for _, p := range doc.DocMeta.Programs { + v.Add("rft.subject", p) + } + return docEmbeds{JSONLDTag: tag, COinS: v.Encode()} +} diff --git a/static/main.js b/static/main.js new file mode 100644 index 0000000..9aac94d --- /dev/null +++ b/static/main.js @@ -0,0 +1,415 @@ +// saved custom CSS; ?nocss=1 clears it + (function () { + try { + if (/[?&]nocss=1(?:&|$)/.test(location.search)) { + localStorage.removeItem('snowchive-custom-css'); + return; + } + var css = localStorage.getItem('snowchive-custom-css') || ''; + if (!css) return; + var el = document.createElement('style'); + el.id = 'snowchive-custom-css'; + el.textContent = css; + document.head.appendChild(el); + } catch (e) {} + })(); + +function snowchiveStorage() { + try { return window.localStorage || null; } catch (e) { return null; } +} + +function isTyping(el) { + return !!el && (/^(INPUT|TEXTAREA|SELECT)$/.test(el.tagName) || el.isContentEditable); +} + +// animated thumb rollover +(function () { + if (window.matchMedia && matchMedia('(prefers-reduced-motion: reduce)').matches) return; + const gifThumbs = Array.from(document.querySelectorAll('.thumb img[data-gif], .result-thumb img[data-gif]')); + gifThumbs.forEach(img => { + img.dataset.still = img.getAttribute('src'); + const gif = img.getAttribute('data-gif'); + + img.addEventListener('mouseenter', () => img.setAttribute('src', gif)); + img.addEventListener('mouseleave', () => img.setAttribute('src', img.dataset.still)); + + img.addEventListener('touchstart', () => { + gifThumbs.forEach(o => { + if (o !== img) o.setAttribute('src', o.dataset.still); + }); + img.setAttribute('src', gif); + }, { passive: true }); + }); + +})(); + +// card page-preview panel (opt-in, desktop) + (function () { + if (!window.matchMedia || !matchMedia('(hover: hover) and (pointer: fine)').matches) return; + if (!document.querySelector('[data-pp-id]')) return; + + var panel = null, activeEl = null, pages = [], idx = 0, docId = '', timer = null, dragging = false, pinned = false; + var openTimer = null; + var WKEY = 'snowchive.pp.width', WMIN = 300, WDEFAULT = 460; + + function imgURL(name) { return '/files/' + docId + '/images/' + name; } + function cancelClose() { if (timer) { clearTimeout(timer); timer = null; } } + function scheduleClose() { if (dragging || pinned) return; cancelClose(); timer = setTimeout(close_, 220); } + + function maxW() { return Math.round(window.innerWidth * 0.9); } + function clampW(w) { return Math.max(WMIN, Math.min(w, maxW())); } + function loadW() { try { var v = parseInt(localStorage.getItem(WKEY), 10); if (v) return v; } catch (e) {} return WDEFAULT; } + function saveW(w) { try { localStorage.setItem(WKEY, String(w)); } catch (e) {} } + function applyW(w) { if (panel) panel.style.width = clampW(w) + 'px'; } + + function build() { + panel = document.createElement('div'); + panel.className = 'pp-panel'; + panel.innerHTML = + '
    ' + + '
    ' + + '
    ' + + '
    ' + + '' + + 'h / ← prev · l / → next · s pin to resize' + + '📌 pinned · h / ← prev · l / → next · esc close' + + '
    '; + document.body.appendChild(panel); + applyW(loadW()); + initResize(); + panel.addEventListener('mouseenter', cancelClose); + panel.addEventListener('mouseleave', scheduleClose); + } + + function initResize() { + var resizer = panel.querySelector('.pp-resizer'); + function onMove(e) { applyW(window.innerWidth - e.clientX); } + function onUp() { + dragging = false; + resizer.classList.remove('pp-dragging'); + document.body.style.userSelect = ''; + document.removeEventListener('mousemove', onMove); + document.removeEventListener('mouseup', onUp); + saveW(parseInt(panel.style.width, 10)); + } + resizer.addEventListener('mousedown', function (e) { + e.preventDefault(); + dragging = true; + cancelClose(); + resizer.classList.add('pp-dragging'); + document.body.style.userSelect = 'none'; + document.addEventListener('mousemove', onMove); + document.addEventListener('mouseup', onUp); + }); + } + + function go(n) { + if (!pages.length) return; + idx = n < 0 ? 0 : (n >= pages.length ? pages.length - 1 : n); + panel.querySelector('.pp-img').src = imgURL(pages[idx]); + panel.querySelector('.pp-counter').textContent = (idx + 1) + ' / ' + pages.length; + var strip = panel.querySelector('.pp-strip'), i; + for (i = 0; i < strip.children.length; i++) { + strip.children[i].classList.toggle('pp-cur', i === idx); + } + var cur = strip.children[idx]; + if (cur) { + var cr = cur.getBoundingClientRect(), sr = strip.getBoundingClientRect(); + strip.scrollLeft += (cr.left + cr.width / 2) - (sr.left + sr.width / 2); + } + } + + function buildStrip() { + var strip = panel.querySelector('.pp-strip'); + strip.textContent = ''; + pages.forEach(function (name, i) { + var t = new Image(); + t.className = 'pp-th'; + t.loading = 'lazy'; + t.src = imgURL(name); + t.alt = 'Page ' + (i + 1); + t.addEventListener('click', function () { go(i); }); + strip.appendChild(t); + }); + } + + function open_(el) { + cancelClose(); + if (el === activeEl) return; + var imgs = (el.getAttribute('data-pp-imgs') || '').split(' ').filter(Boolean); + if (!imgs.length) { close_(); return; } + activeEl = el; + docId = el.getAttribute('data-pp-id'); + pages = imgs; + if (!panel) build(); + buildStrip(); + go(0); + panel.classList.add('pp-open'); + } + + function close_() { + cancelClose(); + pinned = false; + if (panel) { panel.classList.remove('pp-open'); panel.classList.remove('pp-pinned'); } + activeEl = null; + } + + document.addEventListener('mouseover', function (e) { + if (dragging || pinned) return; + var el = e.target.closest('[data-pp-id]'); + if (!el) return; + if (el === activeEl) { cancelClose(); return; } + clearTimeout(openTimer); + openTimer = setTimeout(function () { open_(el); }, 150); + }); + document.addEventListener('mouseout', function (e) { + var el = e.target.closest('[data-pp-id]'); + if (!el) return; + var to = e.relatedTarget; + if (to && (el.contains(to) || (panel && panel.contains(to)))) return; + clearTimeout(openTimer); + scheduleClose(); + }); + document.addEventListener('keydown', function (e) { + if (!activeEl) return; + if (e.altKey || e.ctrlKey || e.metaKey) return; + if (isTyping(document.activeElement)) return; + switch (e.key) { + case 'h': case 'ArrowLeft': go(idx - 1); e.preventDefault(); break; + case 'l': case 'ArrowRight': go(idx + 1); e.preventDefault(); break; + case 'Home': if (pinned) { go(0); e.preventDefault(); } break; + case 'End': if (pinned) { go(pages.length - 1); e.preventDefault(); } break; + case 's': case 'S': + pinned = !pinned; + panel.classList.toggle('pp-pinned', pinned); + if (!pinned && !(panel.matches(':hover') || (activeEl && activeEl.matches(':hover')))) close_(); + e.preventDefault(); + break; + case 'Escape': close_(); break; + } + }); + window.addEventListener('resize', function () { if (panel) applyW(parseInt(panel.style.width, 10) || WDEFAULT); }); + })(); + +// lightbox keyboard and swipe navigation +(function () { + var total = document.querySelectorAll('.lightbox:not(.lb-alt)').length; + if (!total) return; + document.addEventListener('keydown', function (e) { + if (e.altKey || e.ctrlKey || e.metaKey) return; + if (isTyping(e.target)) return; + if (e.key === 'Escape' && /^#lb-a\d+-\d+$/.test(location.hash)) { + location.hash = '#lb-closed'; + e.preventDefault(); + return; + } + var m = /^#lb-(\d+)$/.exec(location.hash); + if (!m) return; + var n = parseInt(m[1], 10); + switch (e.key) { + case 'ArrowLeft': case 'h': + if (n > 1) location.hash = '#lb-' + (n - 1); + e.preventDefault(); + break; + case 'ArrowRight': case 'l': + if (n < total) location.hash = '#lb-' + (n + 1); + e.preventDefault(); + break; + case 'Escape': + location.hash = '#lb-closed'; + e.preventDefault(); + break; + } + }); + + window.addEventListener('hashchange', function (e) { + var was = /#lb-(\d+)$/.exec(e.oldURL), is = /#lb-(\d+)$/.exec(e.newURL); + if (is) { + var close = document.querySelector('#lb-' + is[1] + ' .lb-close'); + if (close) close.focus({ preventScroll: true }); + } else if (was) { + var thumbs = document.querySelectorAll('a[href="#lb-' + was[1] + '"]'); + for (var i = 0; i < thumbs.length; i++) { + if (!thumbs[i].closest('.lightbox')) { thumbs[i].focus(); break; } + } + } + }); + + var tx = 0, ty = 0; + document.addEventListener('touchstart', function (e) { + if (e.touches.length > 1) { tx = NaN; return; } + tx = e.changedTouches[0].clientX; + ty = e.changedTouches[0].clientY; + }, { passive: true }); + document.addEventListener('touchend', function (e) { + var m = /^#lb-(\d+)$/.exec(location.hash); + if (!m) return; + var dx = e.changedTouches[0].clientX - tx; + var dy = e.changedTouches[0].clientY - ty; + if (!(Math.abs(dx) >= 50) || Math.abs(dy) > Math.abs(dx)) return; + var n = parseInt(m[1], 10); + if (dx < 0 && n < total) location.hash = '#lb-' + (n + 1); + else if (dx > 0 && n > 1) location.hash = '#lb-' + (n - 1); + }, { passive: true }); +})(); + +// private per-document notes +(function () { + var box = document.getElementById('note-box'); + var localStorage = snowchiveStorage(); + if (!box || !localStorage) return; + box.hidden = false; + + var key = 'snowchive-note:' + box.dataset.docId; + var ta = document.getElementById('note-text'); + var status = document.getElementById('note-status'); + + try { + var saved = localStorage.getItem(key); + if (saved) { + var obj = JSON.parse(saved); + ta.value = obj.text || ''; + if (ta.value) { box.open = true; status.textContent = '(saved)'; } + } + } catch (e) {} + + var t = null; + function save() { + clearTimeout(t); + t = null; + try { + if (ta.value.trim() === '') { + localStorage.removeItem(key); + status.textContent = ''; + } else { + localStorage.setItem(key, JSON.stringify({ + title: box.dataset.docTitle, + text: ta.value, + updated: new Date().toISOString() + })); + status.textContent = '(saved)'; + } + } catch (e) { status.textContent = '(could not save)'; } + } + ta.addEventListener('input', function () { + status.textContent = '...'; + clearTimeout(t); + t = setTimeout(save, 400); + }); + function flush() { if (t !== null) save(); } + ta.addEventListener('change', flush); + window.addEventListener('pagehide', flush); +})(); + +// the Pinned page's note listing +(function () { + var box = document.getElementById('note-list-box'); + var list = document.getElementById('note-list'); + if (!box || !list) return; + var localStorage = snowchiveStorage(); + if (!localStorage) return; + + var PREFIX = 'snowchive-note:'; + var items = []; + try { + for (var i = 0; i < localStorage.length; i++) { + var k = localStorage.key(i); + if (!k || k.indexOf(PREFIX) !== 0) continue; + try { + var obj = JSON.parse(localStorage.getItem(k)); + items.push({ id: k.slice(PREFIX.length), title: obj.title || k.slice(PREFIX.length), + text: obj.text || '', updated: obj.updated || '' }); + } catch (e) {} + } + } catch (e) {} + if (!items.length) return; + items.sort(function (a, b) { return a.updated < b.updated ? 1 : -1; }); + + box.hidden = false; + + items.forEach(function (it) { + var li = document.createElement('li'); + li.className = 'note-item'; + + var a = document.createElement('a'); + a.href = '/doc/' + encodeURIComponent(it.id); + a.textContent = it.title; + li.appendChild(a); + + if (it.updated) { + var d = document.createElement('span'); + d.className = 'faint note-date'; + d.textContent = ' ' + it.updated.slice(0, 10); + li.appendChild(d); + } + + var del = document.createElement('button'); + del.type = 'button'; + del.className = 'link-btn unpin-btn'; + del.textContent = 'delete'; + del.addEventListener('click', function () { + if (!confirm('Delete this note?')) return; + try { localStorage.removeItem(PREFIX + it.id); } catch (e) { return; } + items.splice(items.indexOf(it), 1); + li.remove(); + if (!list.children.length) box.hidden = true; + }); + li.appendChild(del); + + var p = document.createElement('div'); + p.className = 'note-text'; + p.textContent = it.text; + li.appendChild(p); + + list.appendChild(li); + }); + + document.getElementById('note-export').addEventListener('click', function () { + var blob = new Blob([JSON.stringify(items, null, 2)], { type: 'application/json' }); + var a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = 'snowchive-notes.json'; + a.click(); + // revoked later: Safari starts the download after click() returns + setTimeout(function () { URL.revokeObjectURL(a.href); }, 40000); + }); +})(); + +// the Preferences page's custom-CSS editor +(function () { + var localStorage = snowchiveStorage(); + if (!localStorage) return; + var fs = document.getElementById('custom-css-details'); + var ta = document.getElementById('custom-css'); + var status = document.getElementById('custom-css-status'); + if (!fs || !ta || !status) return; + fs.hidden = false; + + var KEY = 'snowchive-custom-css'; + try { + ta.value = localStorage.getItem(KEY) || ''; + } catch (e) {} + + function apply() { + var existing = document.getElementById('snowchive-custom-css'); + if (existing) existing.remove(); + if (ta.value) { + var el = document.createElement('style'); + el.id = 'snowchive-custom-css'; + el.textContent = ta.value; + document.head.appendChild(el); + } + } + + document.getElementById('custom-css-save').addEventListener('click', function () { + try { + if (ta.value) { + localStorage.setItem(KEY, ta.value); + } else { + localStorage.removeItem(KEY); + } + apply(); + status.textContent = ta.value ? 'Saved and applied.' : 'Cleared - custom CSS is off.'; + } catch (e) { status.textContent = 'Could not save.'; } + }); +})(); diff --git a/static/style.css b/static/style.css new file mode 100644 index 0000000..79a6ec7 --- /dev/null +++ b/static/style.css @@ -0,0 +1,2353 @@ +:root { + --bg: #ffffff; + --text: #222; + --text-strong: #000; + --muted: #666; + --faint: #6e6e6e; + --link: #2a2aff; + --link-hover: #0000b3; + --accent: #0066cc; + + --border: #ddd; + --border-soft: #eee; + --border-strong: #333; + + --surface: #fff; + --surface-alt: #f7f7f7; + --surface-hover: #f4f4f4; + --surface-inset: #fafafa; + --control-bg: #eee; + --control-border: #ccc; + + --row-hover: #fffbcc; + + --mark-bg: #fde68a; --mark-text: #451a03; + + --badge-foia-bg: #dbeafe; --badge-foia-text: #1d4ed8; --badge-foia-border: #bfdbfe; + --badge-sid-bg: #fef9c3; --badge-sid-text: #854d0e; --badge-sid-border: #fde68a; + --reported-bg: #e8f5e9; --reported-text: #1b5e20; --reported-border: #c8e6c9; + --banner-bg: #f4f6ff; --banner-border: #d6defc; + --saved-text: #1b5e20; + + --classified: #b3453c; + --redaction: #000; + + --pane-bg: #2f2f2f; + --btn-text: #fff; + --shadow: rgba(0, 0, 0, 0.05); + + color-scheme: light; +} + +@media screen { + [data-theme="dark"] { + --bg: #16181d; + --text: #d4d7dc; + --text-strong: #fff; + --muted: #9aa0a8; + --faint: #8d939b; + --link: #8aa4ff; + --link-hover: #fff; + --accent: #5b8def; + + --border: #353a43; + --border-soft: #272b32; + --border-strong: #5a606b; + + --surface: #1d2026; + --surface-alt: #1a1d23; + --surface-hover: #242830; + --surface-inset: #191c21; + --control-bg: #23262d; + --control-border: #3a3f49; + + --row-hover: #2e3038; + + --mark-bg: #6b5410; --mark-text: #ffe9a8; + + --badge-foia-bg: #1e3a5f; --badge-foia-text: #9cc3ff; --badge-foia-border: #2b4d7a; + --badge-sid-bg: #4a3f12; --badge-sid-text: #ffd97a; --badge-sid-border: #6b5a1d; + --reported-bg: #173a1f; --reported-text: #8fd99e; --reported-border: #26562f; + --banner-bg: #1c2435; --banner-border: #2c3a57; + --saved-text: #8fd99e; + + --classified: #f28b82; + --redaction: #d4d7dc; + + --pane-bg: #111; + --btn-text: #10141c; + --shadow: rgba(0, 0, 0, 0.4); + + color-scheme: dark; + } +} + +@media screen and (prefers-color-scheme: dark) { + :root:not([data-theme="light"]) { + --bg: #16181d; + --text: #d4d7dc; + --text-strong: #fff; + --muted: #9aa0a8; + --faint: #8d939b; + --link: #8aa4ff; + --link-hover: #fff; + --accent: #5b8def; + + --border: #353a43; + --border-soft: #272b32; + --border-strong: #5a606b; + + --surface: #1d2026; + --surface-alt: #1a1d23; + --surface-hover: #242830; + --surface-inset: #191c21; + --control-bg: #23262d; + --control-border: #3a3f49; + + --row-hover: #2e3038; + + --mark-bg: #6b5410; --mark-text: #ffe9a8; + + --badge-foia-bg: #1e3a5f; --badge-foia-text: #9cc3ff; --badge-foia-border: #2b4d7a; + --badge-sid-bg: #4a3f12; --badge-sid-text: #ffd97a; --badge-sid-border: #6b5a1d; + --reported-bg: #173a1f; --reported-text: #8fd99e; --reported-border: #26562f; + --banner-bg: #1c2435; --banner-border: #2c3a57; + --saved-text: #8fd99e; + + --classified: #f28b82; + --redaction: #d4d7dc; + + --pane-bg: #111; + --btn-text: #10141c; + --shadow: rgba(0, 0, 0, 0.4); + + color-scheme: dark; + } +} + +body { + font-family: sans-serif; + max-width: 1920px; + margin: 0 auto; + padding: 20px; + background: var(--bg); + color: var(--text); +} + +a { color: var(--link); } +:where(a):hover, +:where(a):focus-visible { color: var(--link-hover); text-decoration: underline; } + +h1, h2, h3, .doc-title { overflow-wrap: break-word; } + +h1, h2, h3, h4 { color: var(--text-strong); } + +.site-nav { + max-width: 1200px; + border-bottom: 2px solid var(--border-soft); + padding-bottom: 10px; + margin: 0 auto 20px; + display: flex; + gap: 15px; + flex-wrap: wrap; +} +.site-nav a { text-decoration: none; color: var(--link); } +.site-nav .nav-right { margin-left: auto; } +.site-nav a.active { font-weight: bold; text-decoration: underline; text-underline-offset: 3px; } +.site-nav a:hover, +.site-nav a:focus-visible { text-decoration: underline; text-underline-offset: 3px; } + +.skip-link { + position: absolute; + left: -9999px; + top: 8px; + z-index: 100; + background: var(--surface); + color: var(--link); + padding: 10px 16px; + border: 1px solid var(--border); + border-radius: 4px; +} +.skip-link:focus { left: 8px; } +main:focus { outline: none; } + +footer { + max-width: 1200px; + margin: 40px auto 0; + padding-top: 15px; + border-top: 2px solid var(--border-soft); + color: var(--faint); + font-size: 0.85rem; +} + +.active { font-weight: bold; text-decoration: underline; } + +.muted { color: var(--muted); } +.faint { color: var(--faint); } +.count-note { color: var(--muted); font-size: 0.6em; font-weight: normal; } +.hr-soft { border: 0; border-top: 1px solid var(--border-soft); margin: 20px 0; } +.section-rule { border: 0; border-top: 1px solid var(--border-soft); margin: 40px 0; } +.rule-top { border-top: 1px solid var(--border); padding-top: 10px; margin-top: 15px; } +.pub-sep { margin-top: 12px; padding-top: 12px; border-top: 1px dashed var(--border-soft); } + +.sr-only { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + border: 0; +} + +span.redacted { + background: var(--redaction); + color: var(--redaction); +} + +.pub-block .pub-title { line-height: 1.45; } +.pub-block .pub-people, .pub-block .pub-date { font-size: 0.92em; margin-top: 2px; } + +table { + width: 100%; + border-collapse: collapse; + border-spacing: 0; + table-layout: auto; +} + +th, td { + padding: 8px 12px; + text-align: left; + vertical-align: top; + min-width: 80px; + white-space: normal; + word-break: normal; + overflow-wrap: break-word; +} + +th { font-weight: 600; white-space: nowrap; } + +tr:nth-child(even) { background: var(--surface-alt); } + +.data-table { + width: 100%; + border-collapse: collapse; + font-size: 0.88em; +} +.data-table thead tr { text-align: left; } +.num, .num-col { text-align: right; white-space: nowrap; font-variant-numeric: tabular-nums; } +.data-table thead th { padding: 8px 10px; border-bottom: 2px solid var(--border); } +.data-table thead th a { text-decoration: none; color: var(--text-strong); } +.data-table thead th a:hover { text-decoration: underline; } +.sort-hint { color: var(--faint); } + +.type-badge { + display: inline-block; + font-size: 0.65rem; + font-weight: bold; + padding: 1px 6px; + border-radius: 10px; + vertical-align: middle; + letter-spacing: 0.03em; +} +.type-badge--foia { + background: var(--badge-foia-bg); + color: var(--badge-foia-text); + border: 1px solid var(--badge-foia-border); +} +.type-badge--sidtoday { + background: var(--badge-sid-bg); + color: var(--badge-sid-text); + border: 1px solid var(--badge-sid-border); +} +.type-filter-leak { font-size: 0.8em; color: var(--faint); } +.type-filter-count { font-size: 0.8em; } + +.issue-reported { + display: inline-block; + background: var(--reported-bg); + color: var(--reported-text); + border: 1px solid var(--reported-border); + border-radius: 10px; + font-size: 0.65rem; + font-weight: bold; + padding: 1px 6px; + margin-left: 6px; + vertical-align: middle; + letter-spacing: 0.03em; +} + +#cat-boxes { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(140px, max-content)); + gap: 15px; + margin-bottom: 40px; +} +#cat-boxes .cat-box { + padding: 10px 15px; + border: 1px solid var(--border); + border-radius: 10px; + text-decoration: none; + color: var(--text); + text-align: center; + background: var(--surface); +} +#cat-boxes .cat-box:hover { + border-color: var(--link); + box-shadow: 0 4px 12px var(--shadow); +} +#cat-boxes .cat-box strong { + font-size: 16px; + color: var(--link); + display: block; + margin-bottom: 5px; +} +#cat-boxes .cat-box .count { color: var(--muted); font-size: 0.9rem; } + +.cat-table { + width: 100%; + max-width: 800px; + margin: 20px auto 0; + border-collapse: collapse; +} +.cat-table thead tr { + text-align: left; + border-bottom: 2px solid var(--border-strong); +} +.cat-table th { padding: 12px; } +.cat-table th a { text-decoration: none; color: var(--text-strong); } +.cat-table th a:hover { text-decoration: underline; } +.cat-table td { padding: 10px; color: var(--text); } +.cat-table td a { text-decoration: none; font-weight: bold; } +.cat-table td a:hover, +.cat-table td a:focus-visible { text-decoration: underline; } +.cat-table tbody tr { border-bottom: 1px solid var(--border); } +.cat-table tbody tr:nth-child(even) { background: var(--surface-alt); } +.cat-table tbody tr:hover, +.codeword-table td:first-child { white-space: nowrap; } +.codeword-table tbody tr:hover, +.data-table tbody tr:hover { background: var(--row-hover); } + +.doc-grid { list-style: none; padding: 0; } + +.doc-card { + display: flex; + gap: 20px; + border: 1px solid var(--border); + border-radius: 8px; + margin-bottom: 20px; + padding: 20px; + background: var(--surface); + box-shadow: 0 2px 4px var(--shadow); +} + +.doc-grid .thumb-container { flex: 0 0 160px; } +.doc-grid .thumb-container .doc-img { + width: 160px; + height: 120px; + object-fit: cover; + border: 1px solid var(--border); + display: block; +} + +.doc-grid .doc-body { flex: 1; min-width: 0; } +.doc-grid .doc-body h3 { + margin: 0 0 6px 0; + font-size: 1.05rem; + line-height: 1.3; +} +.doc-pdf { + font-size: 0.72rem; + font-weight: normal; + color: var(--faint); + padding: 7px 3px; + margin: 0 -3px 0 1px; +} +.doc-grid .doc-body h3 .doc-title { text-decoration: none; } +.doc-grid .doc-body h3 .doc-title:hover, +.doc-grid .doc-body h3 .doc-title:focus-visible { text-decoration: underline; } +.doc-grid .doc-body .doc-content a:hover, +.doc-grid .doc-body .doc-content a:focus-visible { color: var(--link-hover); text-decoration: underline; } + +.doc-grid .doc-body .card-summary { + margin: 4px 0 0; + color: var(--muted); + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; +} +.doc-grid .doc-body .doc-content { + font-size: 0.82rem; + color: var(--muted); + line-height: 1.7; +} +.doc-grid .doc-body .doc-content .doc-creator.active { font-weight: bold; } +.doc-grid .doc-body .doc-content .doc-created.active, +.doc-grid .doc-body .doc-content .doc-programs.active, +.doc-grid .doc-body .doc-content .doc-people.active, +.doc-grid .doc-body .doc-content .pub-source.active, +.doc-grid .doc-body .doc-content .pub-link.active { + color: var(--muted); + font-weight: bold; +} +.doc-grid .doc-body .doc-content .doc-published { color: var(--muted); font-size: 0.9em; } +.doc-grid .doc-body .doc-content .doc-programs, +.doc-grid .doc-body .doc-content .doc-people, +.doc-grid .doc-body .doc-content .doc-classification { color: var(--faint); } +.doc-grid .doc-body .doc-content .doc-publication { + margin-top: 6px; + padding-top: 6px; + border-top: 1px solid var(--border-soft); + color: var(--muted); +} + +.doc-dot { color: var(--faint); margin: 0 4px; } + +details.sort-dropdown { + display: inline-block; + margin-bottom: 15px; + font-size: 0.9em; + position: relative; +} +details.sort-dropdown summary { + cursor: pointer; + list-style: none; + padding: 4px 10px; + border: 1px solid var(--control-border); + border-radius: 4px; + background: var(--control-bg); + user-select: none; +} +details.sort-dropdown .sort-options { + position: absolute; + top: 100%; + left: 0; + background: var(--surface); + border: 1px solid var(--control-border); + border-radius: 4px; + box-shadow: 0 4px 12px var(--shadow); + z-index: 100; + min-width: 160px; + padding: 4px 0; +} +details.sort-dropdown a { + display: block; + padding: 6px 14px; + text-decoration: none; + color: var(--text); +} +details.sort-dropdown a:hover { background: var(--surface-hover); } +details.sort-dropdown a.active { font-weight: bold; background: var(--surface-hover); } + +.pagination { + display: flex; + flex-wrap: wrap; + gap: 6px; + align-items: center; + justify-content: center; + margin: 30px 0; +} +.pag-btn { + padding: 8px 14px; + border: 1px solid var(--control-border); + text-decoration: none; + border-radius: 6px; + color: var(--text); + background: var(--surface); +} +a.pag-btn:hover { background: var(--surface-hover); border-color: var(--faint); } +.pag-current { + background: var(--border-strong); + border-color: var(--border-strong); + color: #fff; + font-weight: bold; +} +.pag-ellipsis { padding: 0 4px; color: var(--faint); } + +.collection-tabs { + display: flex; + gap: 4px; + border-bottom: 2px solid var(--border); + margin-bottom: 25px; + flex-wrap: wrap; +} +.collection-tab { + padding: 10px 18px; + text-decoration: none; + color: var(--muted); + border: 1px solid transparent; + border-bottom: none; + border-radius: 6px 6px 0 0; + position: relative; + top: 2px; +} +.collection-tab:hover { background: var(--surface-hover); } +.collection-tab.active-tab { + background: var(--bg); + border-color: var(--border); + border-bottom: 2px solid var(--bg); + font-weight: bold; + color: var(--text-strong); +} +.tab-count { color: var(--faint); font-size: 0.8em; margin-left: 4px; font-weight: normal; } + +.tab-title { font-size: 1.5em; margin: 0.83em 0; } +.tab-title small { color: var(--muted); font-size: 0.6em; font-weight: normal; } +.tab-desc { color: var(--muted); max-width: 700px; } + +.year-nav { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; + margin: 15px 0 25px; +} +.year-nav .year-nav-label { color: var(--muted); font-size: 0.9em; margin-right: 2px; } + +.search-facets { + border: 1px solid var(--border); + border-radius: 8px; + margin-bottom: 20px; + background: var(--surface); +} +.search-facets > summary { + cursor: pointer; + font-weight: bold; + font-size: 0.85em; + color: var(--muted); + text-transform: uppercase; + letter-spacing: 0.03em; + user-select: none; + padding: 12px 14px; +} +.facets-body { padding: 3px 14px 12px; } +.facet-group { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 6px 8px; + margin-bottom: 8px; +} +.facet-group:last-child { margin-bottom: 0; } +.facet-label { + font-size: 0.85em; + color: var(--muted); + font-weight: 600; + flex: 0 0 90px; +} +.facet-values { display: flex; flex-wrap: wrap; gap: 6px; } +.facet-link { + font-size: 0.85em; + padding: 1px 8px; + border: 1px solid var(--control-border); + border-radius: 11px; + background: var(--control-bg); + text-decoration: none; + color: var(--text); + white-space: nowrap; +} +.facet-link:hover { background: var(--surface-hover); } +.facet-count { color: var(--muted); margin-left: 5px; font-variant-numeric: tabular-nums; } +@media (max-width: 768px) { + .facet-label { flex-basis: 100%; } +} + +.az-nav { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 6px; + margin: 0 0 20px; +} +.az-nav .az-label { color: var(--muted); font-size: 0.9em; margin-right: 2px; } +.az-nav a { + padding: 2px 8px; + background: var(--control-bg); + border: 1px solid var(--control-border); + border-radius: 4px; + text-decoration: none; + color: var(--text); + font-size: 0.85em; + min-width: 1.2em; + text-align: center; +} +.az-nav a:hover { background: var(--surface-hover); } +.cat-table .az-row th { + background: var(--surface); + color: var(--muted); + font-size: 0.9em; + padding: 6px 10px; + scroll-margin-top: 10px; +} +.year-nav a { + padding: 4px 10px; + background: var(--control-bg); + border: 1px solid var(--control-border); + border-radius: 4px; + text-decoration: none; + color: var(--text); + font-size: 0.9em; +} +.year-nav a:hover { background: var(--surface-hover); } + +.archive-year { margin-bottom: 30px; } +.archive-year h2 { + font-size: 1.17em; + margin: 1em 0; + border-bottom: 2px solid var(--border-soft); + padding-bottom: 6px; +} +.archive-year h2 small { color: var(--faint); font-weight: normal; } +.archive-month h3 { font-size: 1em; color: var(--muted); margin: 18px 0 6px; } +.archive-issues { list-style: none; padding: 0; margin: 0; } +.archive-issues li { + padding: 5px 8px; + border-bottom: 1px solid var(--border-soft); + font-size: 0.92rem; +} +.archive-issues li:nth-child(even) { background: var(--surface-inset); } +.issue-date { + display: inline-block; + width: 100px; + color: var(--faint); + font-family: monospace; + font-size: 0.85em; +} + +.series-index { margin: 28px 0 50px; } +.series-index > summary { cursor: pointer; font-weight: bold; } +.series-index > summary .count { font-weight: normal; color: var(--muted); } +.series-index-list { + list-style: none; + padding: 10px 0 4px; + margin: 0; + display: grid; + grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); + gap: 2px 18px; +} +.series-index-list li { font-size: 0.9rem; padding: 2px 0; } +.series-index-list .count { color: var(--faint); font-size: 0.85em; margin-left: 4px; } + +.search-results { list-style: none; padding: 0; } +.search-result { display: block; padding: 14px 18px; } +.results-bar { display: flex; align-items: baseline; gap: 20px; margin-bottom: 15px; flex-wrap: wrap; } +.results-heading { font-size: 1.17em; margin: 0; } +.results-bar details.sort-dropdown { margin-bottom: 0; } +.result-title { margin: 0 0 4px 0; font-size: 1.05rem; } +.result-title-link { text-decoration: none; } +.result-meta { font-size: 0.82rem; color: var(--muted); line-height: 24px; margin-bottom: 4px; } +.result-desc { font-size: 0.92rem; color: var(--text); margin-bottom: 4px; } +.meta-link-soft { color: var(--muted); } +.meta-link-softer { color: var(--faint); } + +.search-result-snippet { + font-size: 0.85rem; + color: var(--muted); + font-style: italic; + margin-top: 6px; + padding: 4px 10px; + border-left: 2px solid var(--border); + background: var(--surface-inset); +} + +mark { + background: var(--mark-bg); + color: var(--mark-text); + border-radius: 2px; + padding: 0 1px; +} + +.search-form input, +.search-form select { + background: var(--control-bg); + border: 1px solid var(--control-border); + color: var(--text); + font-family: monospace; +} +.search-form input:focus, +.search-form select:focus { + outline: 2px solid var(--accent); + background: var(--surface); +} +:where(.search-form button) { color: var(--text); } +.remove-row-btn { + background: none; + border: none; + color: var(--faint); + cursor: pointer; + font-size: 1.1em; + padding: 0; + line-height: 1; +} +.reset-link { font-size: 0.9em; margin-left: 4px; } + +.date-hist { + display: flex; + gap: 4px; + align-items: flex-end; + margin: 10px 0 25px; + overflow-x: auto; + padding-bottom: 4px; +} + +.entity-timeline { margin: 0 0 18px; } +.entity-timeline > summary { + cursor: pointer; + color: var(--muted); + font-size: 0.9em; + user-select: none; +} +.entity-timeline-hist { margin: 12px 0 4px; } +.timeline-block { margin: 4px 0 14px; } +.timeline-label { margin: 8px 0 2px; font-size: 0.78em; font-weight: 600; color: var(--muted); text-transform: uppercase; letter-spacing: 0.03em; } +.hist-col { + display: flex; + flex-direction: column; + align-items: center; + text-decoration: none; + min-width: 30px; +} +.hist-track { + height: 60px; + width: 26px; + display: flex; + align-items: flex-end; +} +.hist-bar { + width: 100%; + background: var(--accent); + border-radius: 2px 2px 0 0; + display: block; +} +.hist-col:hover .hist-bar { background: var(--link); } +.hist-col-undated { cursor: default; } +.hist-col-undated .hist-bar { background: var(--muted); opacity: 0.5; } +.hist-col-undated:hover .hist-bar { background: var(--muted); } +.hist-year { font-size: 0.7rem; color: var(--muted); margin-top: 3px; } +.hist-count { font-size: 0.65rem; color: var(--faint); margin-bottom: 2px; } + +.home-intro { font-size: 1.05rem; } +.home-actions { display: flex; flex-wrap: wrap; gap: 10px; margin: 25px 0 120px; } +.home-stats-note { margin: 0; } +.home-stats-note + .home-section { margin-top: 8px; } +.home-action { + padding: 10px 16px; + border: 1px solid var(--border); + border-radius: 8px; + text-decoration: none; + background: var(--surface); + color: var(--text); +} +.home-action:hover { border-color: var(--accent); box-shadow: 0 4px 12px var(--shadow); } +.home-section { margin: 45px 0; } +.home-section h2 { font-size: 1.15rem; border-bottom: 2px solid var(--border-soft); padding-bottom: 6px; } +.see-all { font-size: 0.75em; font-weight: normal; text-decoration: none; margin-left: 8px; } +.home-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); + gap: 0 30px; +} +.stat-list { list-style: none; padding: 0; margin: 10px 0; } +.stat-list li { + display: flex; + justify-content: space-between; + gap: 10px; + padding: 4px 6px; + border-bottom: 1px solid var(--border-soft); +} +.stat-list a { text-decoration: none; } +.pill-list { display: flex; flex-wrap: wrap; gap: 8px; padding: 10px 0; } +.pill { + display: inline-flex; + align-items: center; + gap: 5px; + background: var(--control-bg); + border: 1px solid var(--control-border); + border-radius: 20px; + padding: 4px 10px; + text-decoration: none; + color: var(--text); +} +.pill:hover { border-color: var(--accent); } +.pill-count { + font-size: 0.75em; + color: var(--muted); + background: var(--surface-hover); + border-radius: 10px; + padding: 1px 6px; +} + +.letter-nav { display: flex; flex-wrap: wrap; gap: 6px; margin: 15px 0 25px; } +.letter-nav a { + padding: 3px 9px; + background: var(--control-bg); + border-radius: 4px; + text-decoration: none; + color: var(--text); + font-size: 0.9em; +} +.letter-nav a:hover { background: var(--surface-hover); } +.glossary-group h2 { border-bottom: 2px solid var(--border-soft); padding-bottom: 4px; } +.glossary-list dt { + font-weight: bold; + margin-top: 14px; + color: var(--text-strong); +} +.glossary-aka { font-weight: normal; font-size: 0.9em; } +.glossary-search { font-size: 0.8em; font-weight: normal; margin-left: 8px; text-decoration: none; } +.glossary-list dd { margin: 4px 0 0 0; color: var(--text); max-width: 800px; } +.glossary-see { font-size: 0.85em; margin-top: 3px; } + +.pref-form { max-width: 560px; } +.pref-form fieldset { + border: 1px solid var(--border); + border-radius: 8px; + margin-bottom: 18px; + padding: 12px 16px; +} +.pref-form legend { font-weight: bold; padding: 0 6px; } +.pref-form label { display: block; padding: 4px 0; cursor: pointer; } +.pref-form input[type="checkbox"] { vertical-align: middle; } +.pref-form select { + background: var(--control-bg); + border: 1px solid var(--control-border); + color: var(--text); + padding: 4px 8px; +} +.pref-saved { color: var(--saved-text); font-weight: bold; } +.pref-actions { display: flex; gap: 14px; align-items: center; } +.pref-save { padding: 8px 18px; cursor: pointer; font-weight: bold; } + +.doc-container { + max-width: 1920px; + margin: 0 auto; + padding: 20px; + font-family: sans-serif; + display: grid; +} +.doc-header { + max-width: 1200px; + margin: 0 auto; + width: 100%; +} +.header-main { margin-bottom: 20px; } +.content-wrapper { + display: flex; + gap: 20px; + align-items: flex-start; + max-width: 100%; + overflow: hidden; +} +.pane { + height: 70vh; + overflow-y: auto; + position: sticky; + top: 10px; +} +.pane.images-pane { background: var(--pane-bg); color: #c7cedb; } +.images-pane a { color: #c7cedb; } +.images-pane a:visited { color: #c7cedb; } +.images-pane a:hover { color: #fff; } +.images-pane .muted, +.images-pane .faint { color: #9aa3b2; } + +.long-description-text { + white-space: pre-line; + padding-bottom: 20px; + line-height: 1.6; +} + +.doc-container input[type="radio"], +.meta-checkbox { + position: absolute; + opacity: 0; + width: 0; + height: 0; + margin: 0; + pointer-events: none; +} + +.view-picker { + background: var(--control-bg); + padding: 5px; + border-radius: 8px; + display: flex; + gap: 5px; + justify-self: flex-start; +} +.view-picker label { + padding: 8px 16px; + cursor: pointer; + background: var(--control-bg); + color: var(--muted); + border-radius: 4px; + transition: all 0.1s ease; + display: inline-block; + font-size: 0.9rem; + border: 2px solid transparent; +} +.view-picker label:hover { background: var(--surface-hover); } + +#view-images:focus-visible ~ .view-picker .btn-img, +#view-split:focus-visible ~ .view-picker .btn-split, +#view-text:focus-visible ~ .view-picker .btn-txt, +.meta-checkbox:focus-visible ~ .meta-label { + outline: 2px solid var(--accent); + outline-offset: 2px; + border-radius: 4px; +} + +#view-images:checked ~ .view-picker .btn-img, +#view-split:checked ~ .view-picker .btn-split, +#view-text:checked ~ .view-picker .btn-txt { + background: var(--surface-hover); + color: var(--text-strong); + font-weight: bold; +} + +#view-images:checked ~ .content-wrapper .text-pane { display: none; } +#view-images:checked ~ .content-wrapper .images-pane { + flex: 1; + max-width: min(1200px, 100%); + margin: 0 auto; + position: relative; + height: auto; +} +#view-text:checked ~ .content-wrapper .images-pane { display: none; } +#view-text:checked ~ .content-wrapper .text-pane { + flex: 1; + max-width: min(1000px, 100%); + margin: 0 auto; + position: relative; + height: auto; +} +#view-split:checked ~ .content-wrapper .images-pane { flex: 1; background: var(--pane-bg); } +#view-split:checked ~ .content-wrapper .text-pane { flex: 1; } + +.meta-label { + color: var(--link); + cursor: pointer; + font-weight: bold; + display: inline-block; + padding: 4px 8px 4px 0; +} +.meta-label:hover { background: var(--surface-hover); } +.meta-content { display: none; margin-top: 15px; } +.meta-checkbox:checked ~ .meta-content { display: block; } +.expanded-text { display: none; } +.meta-checkbox:checked ~ .meta-label .collapsed-text { display: none; } +.meta-checkbox:checked ~ .meta-label .expanded-text { display: inline; } + +.related-section { max-width: 1200px; margin: 40px auto; padding: 0 20px; } +.doc-extra { max-width: 1200px; margin: 0 auto; padding: 0 20px; } +.collapse-section > summary { + cursor: pointer; + list-style: none; + display: flex; + align-items: baseline; + gap: 8px; + border-bottom: 2px solid var(--border-soft); + padding-bottom: 8px; + user-select: none; +} +.collapse-section > summary::-webkit-details-marker { display: none; } +.collapse-section > summary h2 { font-size: 1.17em; margin: 0; border: 0; padding: 0; } +.collapse-section > summary small { font-weight: normal; color: var(--faint); } +.collapse-section > summary .collapse-ind { color: var(--link); font-weight: bold; } +.collapse-section > summary .collapse-ind::before { content: "[+]"; } +.collapse-section[open] > summary .collapse-ind::before { content: "[\2212]"; } +.collapse-section > summary:hover .collapse-ind { background: var(--surface-hover); } +.collapse-section { margin-bottom: 30px; } + +.image-list { + display: flex; + flex-direction: column; + gap: 30px; + align-items: center; + padding: 30px; +} +.image-list img { display: block; max-width: 100%; height: auto; } +.prose { + line-height: 1.8; + font-size: 1.15rem; + color: var(--text); + font-family: Georgia, serif; + padding: 0 20px; + overflow-wrap: anywhere; +} +.prose code { white-space: pre-wrap; } +.prose .md-h1 { font-size: 2em; margin: 0.67em 0; color: var(--text-strong); } +.prose .md-h2 { font-size: 1.5em; margin: 0.83em 0; color: var(--text-strong); } +.prose .md-h3 { font-size: 1.17em; margin: 1em 0; color: var(--text-strong); } +.prose .md-h4 { font-size: 1em; margin: 1.33em 0; color: var(--text-strong); } +.prose .md-h5 { font-size: 0.83em; margin: 1.67em 0; color: var(--text); } +.prose .md-h6 { font-size: 0.67em; margin: 2.33em 0; color: var(--text); } +.prose .table-scroll { max-width: 100%; } +.prose pre { max-width: 100%; overflow-x: auto; } + +.tag { + display: inline-flex; + align-items: center; + box-sizing: border-box; + min-height: 24px; + background: var(--control-bg); + padding: 2px 8px; + border-radius: 12px; + margin: 2px; + font-size: 0.85rem; +} + +.doc-top-btn { + display: inline-block; + margin: 0 10px 0 0; + padding: 3px 10px; + font-size: 0.85rem; + text-decoration: none; + border: 1px solid var(--control-border); + border-radius: 5px; + background: var(--control-bg); + color: var(--text); + vertical-align: middle; + line-height: 1.5; +} +.doc-top-btn:hover { background: var(--surface-hover); border-color: var(--faint); } + +.search-link { display: inline-block; font-size: 0.85em; margin-top: 2px; } + +.header-main .doc-top-btn { margin-bottom: 0; } +.header-main .doc-info { margin-top: 18px; } +.header-main h1 { margin-bottom: 14px; } + +.doc-pdf-link { + background: var(--accent); + border-color: var(--accent); + color: var(--btn-text); + font-weight: 600; +} +.doc-pdf-link:hover { opacity: .9; background: var(--accent); border-color: var(--accent); } + +.doc-info { line-height: 1.25; } +.doc-info strong { + display: inline-block; + min-width: 10.5em; + font-weight: 500; + font-size: 0.82em; + letter-spacing: .04em; + text-transform: uppercase; + color: var(--muted); +} + +.doc-banner-link { + font-size: 0.82em; + font-weight: 500; + letter-spacing: .03em; + background: var(--banner-bg); + border: 1px solid var(--banner-border); + border-radius: 4px; + padding: 1px 8px; + text-decoration: none; +} + +.doc-info a.tag.classification { + background: transparent; + border: 1px solid var(--classified); + color: var(--classified); + font-weight: 600; + letter-spacing: .02em; +} + +.doc-layout.info-sidebar { + display: flex; + align-items: flex-start; + max-width: 1920px; + margin: 0 auto; +} + +.info-sidebar > .doc-header { + flex: 0 0 auto; + width: 330px; + min-width: 240px; + max-width: 46vw; + margin: 0; + padding: 0 16px 12px 0; + position: sticky; + top: 10px; + max-height: calc(100vh - 20px); + overflow: auto; + overscroll-behavior: contain; + scrollbar-width: thin; + resize: horizontal; + border-right: 1px solid var(--border-soft); +} + +.info-sidebar > .doc-container { + flex: 1 1 auto; + min-width: 0; + max-width: none; + margin: 0; + padding: 0 0 20px 20px; +} + +.info-sidebar .header-main { margin-bottom: 0; padding-bottom: 72px; } + +.info-collapse-checkbox { + position: absolute; + opacity: 0; + width: 0; + height: 0; + margin: 0; + pointer-events: none; +} +.info-toggle-label { + flex: 0 0 auto; + position: sticky; + top: 10px; + z-index: 3; + margin: 0 3px; + padding: 10px 4px; + cursor: pointer; + user-select: none; + font-size: 0.8rem; + line-height: 1; + color: var(--muted); + background: var(--control-bg); + border: 1px solid var(--control-border); + border-radius: 5px; +} +.info-toggle-label:hover { color: var(--text); border-color: var(--faint); } +.info-toggle-label::before { content: "\00AB"; } +.info-collapse-checkbox:checked ~ .info-toggle-label::before { content: "\00BB"; } +.info-collapse-checkbox:focus-visible ~ .info-toggle-label { + outline: 2px solid var(--accent); + outline-offset: 2px; +} +.info-collapse-checkbox:checked ~ .doc-header { display: none; } + +.info-sidebar .doc-header h1 { font-size: 1.3rem; line-height: 1.3; margin: 0 0 10px; } +.info-sidebar .doc-info, +.info-sidebar .series-box, +.info-sidebar .pub-block { font-size: 0.88rem; } +.info-sidebar .header-main .doc-pdf-link, +.info-sidebar .header-main .doc-top-btn { margin: 0 6px 8px 0; } +.info-sidebar .checksum { font-size: 0.68rem; } +.info-sidebar .long-description-text { font-size: 0.88rem; line-height: 1.55; } +.info-sidebar .serial-nav { flex-direction: column; gap: 4px; font-size: 0.82rem; } +.info-sidebar .serial-next { margin-left: 0; } +.info-sidebar .meta-grid { display: block; } +.info-sidebar .meta-wide { grid-column: auto; } +.info-sidebar #view-text:checked ~ .content-wrapper .text-pane, +.info-sidebar #view-images:checked ~ .content-wrapper .images-pane { + max-width: none; +} +.info-sidebar .prose { font-size: 1.05rem; } + +.info-sidebar #view-split:checked ~ .content-wrapper .images-pane, +.info-sidebar #view-split:checked ~ .content-wrapper .text-pane { + flex: 1 1 50%; + min-width: 0; +} +.pref-form label:has(select) { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 6px; +} +.pref-form label:has(select) .pref-note { flex: 1 1 100%; } + +@media (max-width: 900px) { + .doc-layout.info-sidebar { display: block; } + .info-sidebar > .doc-header { + width: auto; + max-width: 1200px; + position: static; + max-height: none; + overflow: visible; + resize: none; + border-right: 0; + padding-right: 0; + margin: 0 auto; + } + .info-sidebar > .doc-container { padding: 20px; } + .info-toggle-label { display: none; } + .info-sidebar .header-main { margin-bottom: 20px; } +} + +@media print { + .doc-layout.info-sidebar { display: block; } + .info-sidebar > .doc-header { + position: static !important; + max-height: none !important; + overflow: visible !important; + width: auto !important; + border-right: 0; + } + .info-toggle-label { display: none !important; } +} +.header-main .rule-top > p { + line-height: 1.65; +} + +.pub-block .pub-entry { + max-width: 70ch; + padding: 10px 14px; + border: 1px solid var(--border); + border-radius: 8px; + background: var(--surface); +} +.pub-block .pub-entry + .pub-entry { margin-top: 10px; } +.pub-block .pub-sep { padding-top: 10px; border-top: 1px solid var(--border); } +.pub-block .pub-head { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 2px 10px; + font-size: 0.85em; +} +.pub-block .pub-head .pub-original { margin-left: 0; font-weight: 600; } +.pub-block .pub-head .pub-date { margin-top: 0; font-size: 1em; } +.pub-block .pub-head + .pub-title { margin-top: 4px; } +.pub-block .pub-people { margin-top: 7px; } + +.header-main .meta-label { + font-weight: normal; + font-size: 0.85rem; +} + +@media (max-width: 900px) { + #cat-boxes { + grid-template-columns: repeat(auto-fill, minmax(120px, max-content)); + gap: 5px; + } + #cat-boxes .cat-box { padding: 7px 10px; text-align: left; } + #cat-boxes .cat-box strong { font-size: 14px; } +} + +@media (max-width: 768px) { + html { font-size: 14px; } + body { padding: 12px; } + h1 { font-size: 1.5rem; } + h2 { font-size: 1.2rem; } + .browse-header .browse-title, .tab-title { font-size: 1.2rem; } + .prose .md-h1 { font-size: 1.5rem; } + .prose .md-h2 { font-size: 1.2rem; } + + #cat-boxes { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-bottom: 20px; + } + #cat-boxes .cat-box { + display: inline-flex; + align-items: baseline; + gap: 5px; + padding: 4px 10px; + border-radius: 14px; + text-align: left; + } + #cat-boxes .cat-box strong { + display: inline; + font-size: 0.82rem; + margin: 0; + } + #cat-boxes .cat-box .count { font-size: 0.72rem; } + + .doc-card { flex-direction: column; gap: 8px; } + .doc-grid .thumb-container { flex: 0 0 auto; } + .content-wrapper { flex-direction: column; align-items: stretch; } + .content-wrapper > .pane { min-width: 0; max-width: 100%; } + .pane { position: static; height: auto; } + th, td { width: auto; } +} + +.search-title { margin-bottom: 10px; } + +.search-panel { + background: var(--surface); + border: 1px solid var(--border); + border-radius: 10px; + padding: 16px 18px; + max-width: 860px; + box-shadow: 0 2px 4px var(--shadow); +} + +.query-rows { display: flex; flex-direction: column; gap: 8px; } +.query-row { display: flex; align-items: center; gap: 8px; } +.qr-conj { width: 90px; flex: 0 0 90px; } +.qr-conj select { width: 100%; } +.qr-term { + flex: 1; + min-width: 0; + padding: 7px 10px; + border-radius: 6px; + font-size: 0.95rem; +} +.qr-field { display: flex; align-items: center; gap: 6px; flex: 0 0 auto; } +.qr-in { font-size: 0.85em; } +.search-panel select { + background: var(--control-bg); + border: 1px solid var(--control-border); + color: var(--text); + padding: 6px 6px; + border-radius: 6px; +} +.remove-row-spacer { width: 22px; flex: 0 0 22px; } +.remove-row-btn { width: 22px; flex: 0 0 22px; } +.add-row-btn { + background: none; + border: 1px dashed var(--control-border); + color: var(--muted); + border-radius: 6px; + padding: 5px 12px; + cursor: pointer; +} +.add-row-btn:hover { color: var(--text); border-color: var(--faint); } + +.date-filters { margin-top: 17px; } +.date-filters summary { + cursor: pointer; + color: var(--muted); + font-size: 0.9em; + user-select: none; +} +.date-filter-rows { display: flex; flex-direction: column; gap: 6px; margin-top: 10px; } +.date-filter-row { display: flex; align-items: center; gap: 8px; } +.df-label { width: 80px; flex: 0 0 80px; font-size: 0.9em; } +.date-filter-row input { + width: 120px; + padding: 5px 8px; + border-radius: 6px; + font-family: monospace; + font-size: 0.85rem; +} + +.search-actions { + display: flex; + align-items: center; + gap: 14px; + margin-top: 16px; + flex-wrap: wrap; +} +.search-btn { + font-weight: bold; + padding: 8px 22px; + cursor: pointer; + border-radius: 6px; + border: 1px solid var(--accent); + background: var(--accent); + color: var(--btn-text); +} +.search-btn:hover { opacity: 0.9; } + +.list-header h1 { margin-bottom: 6px; } + +.list-controls { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 12px 16px; + margin-bottom: 18px; +} +.list-controls details.sort-dropdown, +.list-controls .type-filter { + margin-bottom: 0; +} +.pref-reset { + padding: 8px 14px; + cursor: pointer; + background: none; + border: 1px solid var(--control-border); + border-radius: 4px; + color: var(--muted); +} +.pref-reset:hover { color: var(--text); border-color: var(--faint); } + +@media (max-width: 700px) { + .query-row { flex-wrap: wrap; } + .qr-conj { flex-basis: 100%; width: auto; } + .qr-term { flex-basis: 100%; } +} + +.inline-form { display: inline; margin: 0; } +.link-btn { + background: none; + border: none; + padding: 0; + cursor: pointer; + font-size: 0.85em; + color: var(--link); + text-decoration: underline; +} +.link-btn.faint { color: var(--faint); } +.unpin-btn { font-size: 0.72rem; margin-left: 4px; } +.pin-btn { cursor: pointer; font-family: inherit; } + +.export-link { margin-left: 12px; } + +.recent-searches { + margin: 10px 0 14px; + font-size: 0.9rem; +} +.recent-searches > summary { + cursor: pointer; + color: var(--muted); + font-size: 0.9em; + user-select: none; +} +.recent-list { + display: flex; + flex-wrap: wrap; + gap: 8px; + align-items: center; + margin-top: 10px; +} +.recent-searches .pill { font-size: 0.85em; } + +.glossary-box { + background: var(--banner-bg); + border: 1px solid var(--banner-border); + border-radius: 6px; + padding: 10px 14px; + margin: 0 0 18px; + max-width: 800px; + font-size: 0.95rem; +} + +.checksum { + font-size: 0.72rem; + word-break: break-all; + background: var(--surface-inset); + border: 1px solid var(--border-soft); + border-radius: 4px; + padding: 3px 6px; + display: inline-block; +} + +.cat-blurb { max-width: 900px; font-size: 0.92rem; margin: 6px 0 14px; } + +.coll-scope { margin-top: 6px; } + +.pub-more { margin: 10px 0 2px; } + +.id-table { font-size: 0.82em; table-layout: fixed; } +.id-table .c-id { width: 24%; } +.id-table .c-coll { width: 9%; } +.id-table .c-pages { width: 5%; } +.id-table .c-pdf { width: 25%; } +.id-table .c-size { width: 7%; } +.id-table .c-sha { width: 30%; } +.id-table td:first-child { overflow-wrap: anywhere; } +.id-table td { padding: 5px 10px; vertical-align: top; border-bottom: 1px solid var(--border-soft); } +.id-table code { font-size: 0.92em; word-break: break-all; } +.id-table .checksum { font-size: 0.68rem; padding: 1px 4px; } +.id-table .id-version td { background: var(--surface-alt); } +.id-table .id-version td:first-child { padding-left: 22px; } + +.changelog-list { list-style: none; padding: 0; max-width: 900px; } +.changelog-entry { + padding: 7px 4px; + border-bottom: 1px solid var(--border-soft); + font-size: 0.95rem; +} +.changelog-date { font-family: monospace; font-size: 0.85em; margin-right: 8px; } +.changelog-type { + display: inline-block; + font-size: 0.65rem; + font-weight: bold; + text-transform: uppercase; + letter-spacing: 0.04em; + border-radius: 10px; + padding: 1px 7px; + margin-right: 6px; + background: var(--control-bg); + border: 1px solid var(--control-border); + color: var(--muted); +} +.changelog-type-added { background: var(--reported-bg); color: var(--reported-text); border-color: var(--reported-border); } +.changelog-type-removed { background: var(--badge-sid-bg); color: var(--badge-sid-text); border-color: var(--badge-sid-border); } +.changelog-type-updated { background: var(--badge-foia-bg); color: var(--badge-foia-text); border-color: var(--badge-foia-border); } + +.pref-note { font-weight: normal; font-size: 0.85em; } +.home-section h2 .link-btn { margin-left: 10px; } + +@media print { + :root { + --bg: #fff; + --text: #000; + --text-strong: #000; + --muted: #444; + --faint: #666; + --link: #000; + --border: #999; + --border-soft: #ccc; + --surface: #fff; + --surface-alt: #fff; + --surface-inset: #fff; + --pane-bg: #fff; + --btn-text: #fff; + } + + .site-nav, + footer, + .view-picker, + .doc-pdf-link, + .pin-btn, + .inline-form, + .export-link, + .related-section, + .thumb-container, + .sort-dropdown, + .view-toggle, + .pagination, + .recent-searches, + .search-panel, + #cat-boxes, + .doc-top-btn, + .pp-panel, + .search-facets, + .type-filter, + button { + display: none !important; + } + + body { max-width: 100%; padding: 0; } + a { color: #000; } + .prose a, .doc-content a { text-decoration: underline; } + + .pane { + position: static !important; + height: auto !important; + overflow: visible !important; + } + .content-wrapper { display: block !important; min-height: 0; } + #view-images:checked ~ .content-wrapper .text-pane, + #view-split:checked ~ .content-wrapper .text-pane, + #view-text:checked ~ .content-wrapper .text-pane { + display: block !important; + max-width: 100% !important; + } + #view-images:checked ~ .content-wrapper .images-pane, + #view-split:checked ~ .content-wrapper .images-pane, + #view-text:checked ~ .content-wrapper .images-pane { + display: none !important; + } + .meta-content { display: block !important; } + .meta-label { display: none !important; } +} + +.article-year h2 { border-bottom: 2px solid var(--border-soft); padding-bottom: 4px; } +.article-list { list-style: none; padding: 0; margin: 6px 0 22px; max-width: 980px; } +.article-item { + display: flex; + gap: 14px; + padding: 7px 4px; + border-bottom: 1px solid var(--border-soft); + align-items: baseline; +} +.article-date { + flex: 0 0 92px; + font-family: monospace; + font-size: 0.82em; +} +.article-main { min-width: 0; } +.article-title { text-decoration: none; font-weight: 600; } +.article-title:hover { text-decoration: underline; } +.article-meta { font-size: 0.82em; display: block; margin-top: 1px; } +.article-kind { text-transform: uppercase; letter-spacing: 0.08em; font-size: 0.9em; color: var(--muted); } + +.article-header h1 { margin: 2px 0 6px; max-width: 900px; } +.article-kicker { + text-transform: uppercase; + letter-spacing: 0.08em; + font-size: 0.75rem; + margin: 0; +} +.article-byline { margin: 0 0 10px; } + +.pub-article-title { font-style: italic; } + +.home-search-form { + display: flex; + gap: 10px; + margin: 25px 0 30px; +} +.home-search { + flex: 1; + padding: 13px 16px; + font-size: 1.05rem; + border: 1px solid var(--control-border); + border-radius: 8px; + background: var(--surface); + color: var(--text); +} +.home-search:focus { outline: 2px solid var(--accent); } + +.cat-box-active { + border-color: var(--accent) !important; + box-shadow: 0 0 0 1px var(--accent); +} + +.pub-original { + font-size: 0.85em; + margin-left: 6px; + white-space: nowrap; +} + +.coll-filter-row { + display: flex; + flex-wrap: wrap; + gap: 14px; + align-items: center; + padding: 10px 0 4px; +} +.coll-check { + display: inline-flex; + align-items: center; + gap: 5px; + font-size: 0.9rem; + cursor: pointer; + white-space: nowrap; +} +.coll-check input { margin: 0; } + +.pinned-searches-heading { margin-top: 35px; } +.pinned-searches { list-style: none; padding: 0; max-width: 800px; } +.pinned-search-item { + display: flex; + align-items: center; + gap: 10px; + padding: 6px 2px; + border-bottom: 1px solid var(--border-soft); +} + +.page-jump { + position: sticky; + top: 0; + z-index: 2; + background: var(--pane-bg); + max-height: 56px; + overflow-y: auto; + scrollbar-width: thin; + padding: 6px 8px; + border-bottom: 1px solid var(--border-soft); + font-size: 0.78rem; + display: flex; + flex-wrap: wrap; + gap: 0 2px; + align-items: center; +} +.page-jump a { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 24px; + min-height: 24px; + text-decoration: none; +} +.page-jump a:hover { text-decoration: underline; } + +.notfound { max-width: 700px; margin: 40px auto; } +.notfound h1 { margin-bottom: 8px; } + +@media print { + .page-jump { display: none !important; } +} + +.about-page { max-width: 760px; } +.about-page p { line-height: 1.7; } +.about-page section { margin-bottom: 28px; } +.about-page h2 { + font-size: 1.25rem; + margin: 0 0 10px; + padding-bottom: 4px; + border-bottom: 1px solid var(--border-soft); + scroll-margin-top: 12px; +} +.about-page .toc { + background: var(--surface); + border: 1px solid var(--border); + border-radius: 8px; + padding: 12px 16px; + margin: 0 0 28px; +} +.about-page .toc strong { + display: block; + font-size: 0.85em; + text-transform: uppercase; + letter-spacing: 0.03em; + color: var(--muted); + margin-bottom: 6px; +} +.about-page .toc ul { margin: 0; padding-left: 18px; } +.about-page .toc li { margin: 2px 0; } +.about-page h3 { font-size: 1.1rem; margin: 26px 0 8px; scroll-margin-top: 12px; } +.about-page h4 { font-size: 1rem; margin: 20px 0 6px; } +.about-page ul ul { margin: 4px 0; } +.about-page li { margin: 3px 0; line-height: 1.6; } +.about-page pre { + background: var(--surface-inset); + border: 1px solid var(--border-soft); + border-radius: 6px; + padding: 10px 12px; + overflow-x: auto; + font-size: 0.85em; +} +.about-page details > summary { cursor: pointer; } +.about-collapse > summary > h2, +.about-collapse > summary > h3 { display: inline; } +.about-table td, .about-table th { padding: 8px 10px; vertical-align: top; border-bottom: 1px solid var(--border-soft); } +.about-table td:first-child { white-space: nowrap; } +.field-dict dt { margin-top: 14px; font-weight: 600; } +.field-dict dd { margin: 4px 0 0 20px; line-height: 1.6; } + +@media (max-width: 768px) { + .collection-tabs { gap: 2px; flex-wrap: nowrap; } + .collection-tab { + padding: 7px 8px; + font-size: 0.82rem; + white-space: nowrap; + flex: 0 1 auto; + min-width: 0; + } + .collection-tab .tab-count { font-size: 0.7rem; } + + .header-main .doc-top-btn { + margin-right: 8px; + margin-bottom: 12px; + } + + #view-split:checked ~ .content-wrapper { flex-direction: row; } + #view-split:checked ~ .content-wrapper .images-pane, + #view-split:checked ~ .content-wrapper .text-pane { + flex: 0 0 50%; + max-width: 50%; + min-width: 0; + } + + .text-pane { overflow-x: auto; max-width: 100%; } + .prose { padding: 0 10px; min-width: 0; } + + .date-filter-row { flex-wrap: wrap; } + .date-filter-row input { + width: auto; + flex: 1 1 90px; + min-width: 0; + max-width: 140px; + box-sizing: border-box; + } + .df-label { flex: 0 0 70px; width: 70px; } + .coll-filter-row { gap: 8px 12px; } +} + +.note-box { + max-width: 900px; + margin: 14px 0; + border: 1px solid var(--border-soft); + border-radius: 6px; + padding: 8px 12px; + background: var(--surface); +} +.note-box > summary { + cursor: pointer; + font-weight: 600; + user-select: none; +} + +.note-box textarea { + width: 100%; + box-sizing: border-box; + margin-top: 8px; + padding: 8px 10px; + border: 1px solid var(--control-border); + border-radius: 4px; + background: var(--surface-inset); + color: var(--text); + font: inherit; + font-size: 0.9rem; +} +.note-status { font-size: 0.75em; font-weight: normal; } + +.note-list-box { margin-top: 35px; } +.note-list { list-style: none; padding: 0; max-width: 900px; } +.note-item { + padding: 8px 4px; + border-bottom: 1px solid var(--border-soft); +} +.note-item > a { font-weight: 600; } +.note-date { font-size: 0.8em; margin: 0 8px; } +.note-text { + white-space: pre-wrap; + overflow-wrap: anywhere; + font-size: 0.9rem; + color: var(--muted); + margin-top: 3px; +} + +.serial-nav { + display: flex; + justify-content: space-between; + gap: 12px; + max-width: 900px; + margin: 8px 0 4px; + font-size: 0.9rem; +} +.serial-nav a { text-decoration: none; } +.serial-nav a:hover { text-decoration: underline; } +.serial-next { margin-left: auto; } + +.search-ops-hint { max-width: 800px; font-size: 0.85rem; margin: 0 0 14px; } +.search-ops-hint code { + background: var(--surface-inset); + border: 1px solid var(--border-soft); + border-radius: 3px; + padding: 0 4px; +} + +@media print { + .note-box, .serial-nav { display: none !important; } +} + +.archive-note { + max-width: 900px; + margin: 14px 0; + padding: 10px 14px; + border-left: 3px solid var(--accent); + background: var(--banner-bg); + border-radius: 0 6px 6px 0; + font-size: 0.95rem; + word-break: break-word; +} + +.page-link-list { list-style: none; padding: 8px 12px; margin: 0; } +.page-link-list li { padding: 3px 0; border-bottom: 1px solid var(--border-soft); } +.textonly-note { padding: 8px 12px; margin: 0; font-size: 0.85rem; } + +.build-info { font-size: 0.8em; } + +.ctl, +.export-link, +.view-toggle a, +.pin-search-btn { + display: inline-flex; + align-items: center; + gap: 4px; + font-size: 0.78rem; + font-weight: normal; + font-family: inherit; + line-height: 1; + padding: 5px 10px; + border: 1px solid var(--control-border); + border-radius: 5px; + background: var(--control-bg); + color: var(--muted); + text-decoration: none; + cursor: pointer; + vertical-align: middle; +} +.ctl:hover, +.export-link:hover, +.view-toggle a:hover, +.pin-search-btn:hover { + color: var(--text); + border-color: var(--faint); +} +.view-toggle { display: inline-flex; gap: 4px; } +.view-toggle a.active-toggle { + background: var(--border-strong); + color: #fff; + border-color: var(--border-strong); +} +@media (max-width: 768px) { + .view-toggle a { padding: 5px 8px; white-space: nowrap; } +} + +.type-filter { + display: inline-flex; + gap: 12px; + align-items: center; + margin-bottom: 15px; + font-size: 0.88em; + flex-wrap: wrap; +} +.type-filter-option { + display: inline-flex; + align-items: center; + gap: 4px; + cursor: pointer; +} +.note-subtle { + border: none; + background: none; + padding: 0; + margin: 6px 0 0; + max-width: 700px; +} +.note-subtle > summary { + display: inline-block; + font-weight: normal; + font-size: 0.85rem; + color: var(--link); + text-decoration: underline; + cursor: pointer; +} +.note-subtle textarea { font-size: 0.85rem; } + +.search-ops-hint { margin: 0 0 14px; } +.search-ops-hint > summary { + cursor: pointer; + font-size: 0.85rem; + color: var(--muted); +} +.search-ops-hint p { max-width: 800px; font-size: 0.85rem; } + +.search-results.has-thumbs .search-result { + display: flex; + gap: 14px; + align-items: flex-start; +} +.result-thumb { + flex: 0 0 auto; + display: block; + width: 120px; +} +.result-thumb .doc-img { + width: 120px; + height: auto; + border: 1px solid var(--border-soft); + border-radius: 3px; + display: block; +} +.result-body { flex: 1 1 auto; min-width: 0; } + +@media (max-width: 768px) { + .result-thumb, .result-thumb .doc-img { width: 88px; } +} + +@media (max-width: 768px) { + .browse-controls { width: 100%; } +} + +.doc-untitled { color: var(--faint); font-style: italic; font-weight: normal; } + +.browse-header { + display: flex; + justify-content: space-between; + align-items: baseline; + gap: 10px 16px; + flex-wrap: wrap; + margin-bottom: 25px; +} +.browse-title { font-size: 1.5em; margin: 0; } +.browse-controls-spaced { margin-bottom: 15px; } +.sort-bar-spaced details.sort-dropdown { margin-top: 15px; } +.browse-controls { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} + +#view-images:checked ~ .content-wrapper.scroll-panes .images-pane, +#view-text:checked ~ .content-wrapper.scroll-panes .text-pane { + height: 90vh; + overflow-y: auto; +} + +@media (max-width: 768px) { + #view-images:checked ~ .content-wrapper.scroll-panes .images-pane, + #view-text:checked ~ .content-wrapper.scroll-panes .text-pane { + flex: 0 0 auto; + height: 85vh; + overflow-y: auto; + } +} + +.meta-value-list { + list-style: none; + margin: 4px 0 0; + padding: 0; +} +.meta-value-list li { padding: 2px 0; } + +.collection-desc { + max-width: 720px; + margin: -10px 0 22px; + color: var(--muted); +} + +.custom-css-details { margin: 30px 0; border-top: 1px solid var(--border-soft); padding-top: 15px; } +.custom-css-details > summary { cursor: pointer; font-weight: bold; } +.custom-css-details > summary::marker { color: var(--muted); } +.custom-css-details > * { margin-top: 10px; } + +#custom-css { + width: 100%; + box-sizing: border-box; + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 0.85rem; + padding: 8px 10px; + border: 1px solid var(--control-border); + border-radius: 4px; + background: var(--surface-inset); + color: var(--text); +} +.custom-css-actions { + display: flex; + gap: 10px; + align-items: center; + margin-top: 8px; + flex-wrap: wrap; +} +.custom-css-status { font-size: 0.8rem; } + +.pp-panel { + position: fixed; + z-index: 1000; + top: 0; + right: 0; + height: 100vh; + width: 460px; + display: flex; + flex-direction: column; + background: var(--bg); + border-left: 1px solid var(--border-strong); + box-shadow: -8px 0 28px var(--shadow); + padding: 8px; + overflow: hidden; + transform: translateX(100%); + opacity: 0; + pointer-events: none; + transition: transform 0.16s ease, opacity 0.16s ease; +} +.pp-panel.pp-open { transform: translateX(0); opacity: 1; pointer-events: auto; } + +.pp-resizer { + position: absolute; + top: 0; + left: 0; + width: 7px; + height: 100%; + z-index: 2; + cursor: ew-resize; +} +.pp-resizer:hover, .pp-resizer.pp-dragging { background: var(--accent); opacity: 0.4; } + +.pp-stage { + position: relative; + flex: 1 1 auto; + min-height: 0; + margin-left: 6px; + background: var(--pane-bg); + border: 1px solid var(--border-soft); + border-radius: 4px; + overflow: hidden; +} +.pp-img { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: contain; } + +.pp-foot { + flex: 0 0 auto; + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + padding: 6px 4px 2px; + margin-left: 6px; + margin-bottom: 10px; + font-size: 0.76em; + color: var(--muted); +} +.pp-counter { font-variant-numeric: tabular-nums; white-space: nowrap; } +.pp-hint { white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +.pp-hint b { color: var(--text); font-weight: 600; } +.pp-hint-pinned { display: none; } +.pp-panel.pp-pinned .pp-hint-free { display: none; } +.pp-panel.pp-pinned .pp-hint-pinned { display: inline; } +.pp-panel.pp-pinned { border-left-color: var(--accent); } +.pp-panel.pp-pinned .pp-resizer { background: var(--accent); opacity: 0.25; } +.pp-strip { + flex: 0 0 auto; + display: flex; + gap: 4px; + overflow-x: auto; + padding: 2px; + margin-left: 6px; + scrollbar-width: thin; +} +.pp-th { + height: 54px; + width: 42px; + flex: 0 0 auto; + object-fit: contain; + border: 2px solid transparent; + border-radius: 2px; + cursor: pointer; + opacity: 0.6; + background: var(--pane-bg); +} +.pp-th:hover { opacity: 1; } +.pp-th.pp-cur { border-color: var(--accent); opacity: 1; } + +.table-scroll { overflow-x: auto; } +.cat-table-colls .num-col { padding-left: 18px; } + +.coll-cols-checkbox { + position: absolute; + opacity: 0; + width: 0; + height: 0; + margin: 0; + pointer-events: none; +} +.coll-cols-label { + color: var(--link); + cursor: pointer; + font-weight: bold; + display: inline-block; + padding: 4px 8px 4px 0; + margin-bottom: 6px; +} +.coll-cols-label:hover { background: var(--surface-hover); } +.coll-cols-checkbox:focus-visible ~ .coll-cols-label { + outline: 2px solid var(--accent); + outline-offset: 2px; + border-radius: 4px; +} +.coll-col { display: none; } +.coll-cols-checkbox:checked ~ .table-scroll .coll-col { display: table-cell; } +.coll-cols-checkbox:checked ~ .coll-cols-label .collapsed-text { display: none; } +.coll-cols-checkbox:checked ~ .coll-cols-label .expanded-text { display: inline; } + +.explore-list { list-style: none; padding: 0; max-width: 800px; } +.explore-list li { margin: 12px 0; line-height: 1.5; } +.explore-table td { vertical-align: top; } +.explore-date { white-space: nowrap; color: var(--muted); } +.explore-link { margin: 30px 0 10px; font-size: 1.1em; } +.crumb { margin: 0 0 4px; font-size: 0.85em; } + +.page-open { display: block; } +.lightbox { display: none; } +.lightbox:target { + display: flex; + position: fixed; + inset: 0; + z-index: 200; + background: rgba(0, 0, 0, 0.92); + align-items: center; + justify-content: center; +} +.lightbox img { + width: auto; + height: auto; + max-width: 96vw; + max-height: 94vh; + object-fit: contain; + position: relative; + z-index: 201; +} +.lightbox .lb-backdrop { + inset: 0; + z-index: 200; + padding: 0; + border-radius: 0; +} +.lightbox .lb-backdrop:hover, .lightbox .lb-backdrop:focus-visible { background: none; } +.lightbox a { + position: fixed; + color: #ddd; + text-decoration: none; + line-height: 1; + padding: 14px 18px; + z-index: 202; +} +.lightbox a:hover, .lightbox a:focus-visible { color: #fff; background: rgba(255, 255, 255, 0.12); border-radius: 6px; } +.lb-close { top: 8px; right: 12px; font-size: 2.2rem; } +.lb-prev { left: 4px; top: 50%; transform: translateY(-50%); font-size: 3rem; } +.lb-next { right: 4px; top: 50%; transform: translateY(-50%); font-size: 3rem; } +.lb-counter { + position: fixed; + bottom: 10px; + left: 50%; + transform: translateX(-50%); + color: #aaa; + font-size: 0.85rem; +} +@media print { + .lightbox { display: none !important; } +} + +.page-sheet { + list-style: none; + padding: 0; + margin: 20px 0; + display: grid; + grid-template-columns: repeat(auto-fill, minmax(150px, 1fr)); + gap: 12px; +} +.page-sheet li { position: relative; } +.page-sheet a { + display: block; + border: 1px solid var(--border); + border-radius: 4px; + overflow: hidden; + background: var(--surface); +} +.page-sheet a:hover, .page-sheet a:focus-visible { border-color: var(--accent); } +.page-sheet img { + display: block; + width: 100%; + height: 210px; + object-fit: contain; + background: #fff; +} +.page-sheet .page-num { + position: absolute; + bottom: 4px; + right: 8px; + font-size: 0.8rem; + background: rgba(0, 0, 0, 0.45); + color: #eee; + padding: 1px 6px; + border-radius: 3px; +} + +.assembly-note { margin: 8px 0 0; } +.assembly-list { list-style: none; padding: 0; margin: 20px 0; } +.assembly-list li { display: flex; gap: 16px; padding: 12px 0; border-bottom: 1px solid var(--border-soft); } +.assembly-thumb { flex: 0 0 180px; } +.assembly-thumb img { display: block; width: 100%; height: auto; background: #fff; border: 1px solid var(--border); border-radius: 4px; } +.page-sheet .assembly-cap { margin: 6px 0 0; font-size: 0.8rem; line-height: 24px; overflow-wrap: anywhere; } +.page-sheet .assembly-cap a { display: inline; border: none; border-radius: 0; background: none; overflow: visible; } +.lightbox .lb-caption { z-index: 202; max-width: 90vw; padding: 2px 8px; text-align: center; color: #ccc; background: rgba(0, 0, 0, 0.7); border-radius: 4px; } +.lightbox .lb-caption a { position: static; padding: 0; color: #fff; text-decoration: underline; } +@media (max-width: 600px) { + .assembly-list li { flex-direction: column; } + .assembly-thumb { flex-basis: auto; } +} + +.month-nav { + display: flex; + flex-wrap: wrap; + gap: 9px 14px; + align-items: baseline; + margin: 0 0 18px; + font-size: 0.92rem; +} +.month-nav a { text-decoration: none; white-space: nowrap; } +.month-nav a:hover, .month-nav a:focus-visible { text-decoration: underline; } +.month-nav a.active { font-weight: bold; text-decoration: underline; text-underline-offset: 3px; } +.month-nav .month-nav-note { font-size: 0.85em; } +.month-nav .facet-count { + font-size: 0.78em; + background: var(--control-bg); + border: 1px solid var(--border); + border-radius: 9px; + padding: 0 6px; + margin-left: 3px; + color: var(--muted); +} + +.doc-pages { color: var(--muted); white-space: nowrap; } + +.series-box { font-size: 0.95rem; } +.series-list { margin: 6px 0 0; padding-left: 2.2em; } +.series-list li { margin: 2px 0; } +.series-list strong[aria-current] { font-weight: 600; } + +@media (prefers-reduced-motion: reduce) { + *, *::before, *::after { + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + transition-duration: 0.01ms !important; + scroll-behavior: auto !important; + } +} + +.data-table thead th, +.cat-table thead th { + position: sticky; + top: 0; + background: var(--surface-hover); + z-index: 2; +} +@media (min-width: 1024px) { + .table-scroll { overflow-x: visible; } +} + +@media print { + main a[href^="http"]::after { + content: " (" attr(href) ")"; + font-size: 0.85em; + color: #444; + word-break: break-all; + } +} + +.meta-wide { grid-column: span 2; } + +.hist-toggles a.active { font-weight: bold; } +.breakdown-bars { + display: grid; + gap: 4px; + margin: 6px 0 14px; + max-width: 500px; +} +.bd-row { + display: grid; + grid-template-columns: minmax(120px, 38%) 1fr auto; + gap: 8px; + align-items: center; + min-height: 20px; + text-decoration: none; + font-size: 0.9rem; +} +.bd-row:hover .bd-name, .bd-row:focus-visible .bd-name { text-decoration: underline; } +.bd-name { overflow-wrap: anywhere; } +.bd-track { background: var(--control-bg); border-radius: 3px; height: 10px; } +.bd-bar { display: block; height: 100%; background: var(--accent); border-radius: 3px; } +.bd-count { color: var(--muted); font-variant-numeric: tabular-nums; } + +.cls-legend { display: flex; flex-wrap: wrap; gap: 12px; margin: 8px 0 12px; font-size: 0.85rem; } +.cls-key { display: inline-flex; align-items: center; gap: 5px; } +.cls-swatch { width: 12px; height: 12px; border-radius: 2px; display: inline-block; } +.cls-row { display: grid; grid-template-columns: 52px minmax(0, 640px) auto; gap: 10px; align-items: center; margin: 3px 0; } +.cls-year { text-decoration: none; font-variant-numeric: tabular-nums; } +.cls-strip { display: flex; height: 24px; border-radius: 3px; overflow: hidden; background: var(--control-bg); } +.cls-seg { display: block; height: 100%; } +.cls-seg:hover, .cls-seg:focus-visible { outline: 2px solid var(--text); outline-offset: -2px; } +.cls-total { font-variant-numeric: tabular-nums; } +.seg-0 { background: #b3453c; } +.seg-1 { background: #d98a3d; } +.seg-2 { background: #c9b458; } +.seg-3 { background: #5b84a8; } +.heat-table td { text-align: right; } +.heat-table tfoot th { text-align: right; font-weight: normal; } +.heat-table tfoot th:first-child { text-align: left; } +.outlet-table small { margin-left: 1px; } +.cat-note { font-size: 0.85em; } +.heat-table td:first-child { text-align: left; } +.heat-cell a { text-decoration: none; display: block; } +.result-pubs { font-size: 0.85rem; line-height: 24px; } + +.pub-block .pub-note { font-size: 0.92em; margin-top: 4px; white-space: pre-line; } +.pub-block .pub-reporting-only { font-size: 0.92em; margin-top: 2px; font-style: italic; } +.pub-block .pub-copy { + display: flex; + flex-wrap: wrap; + gap: 2px 14px; + font-size: 0.92em; + margin-top: 9px; +} +.pub-block .pub-shown { font-style: italic; } +.version-actions { margin: 8px 0 4px; } +.version-snippet .snippet-source { font-weight: 600; margin-right: 0.4em; } +.result-matches-in { font-size: 0.9em; margin-top: 4px; } + +.page-sheet li.page-changed a { border-color: var(--accent); box-shadow: 0 0 0 2px var(--accent); } +.page-sheet .page-changed-badge { position: absolute; top: 4px; left: 4px; background: var(--accent); color: var(--btn-text); font-size: 0.75em; padding: 1px 6px; border-radius: 3px; } +.changed-pages-note { margin: 6px 0; } + +.image-list .page-open.page-changed { outline: 3px solid var(--accent); outline-offset: 2px; position: relative; } +.image-list .page-open .page-changed-badge { top: 6px; left: 6px; } +.page-jump a.page-changed-jump { font-weight: 700; color: #ffd479; } diff --git a/templates/404.html b/templates/404.html new file mode 100644 index 0000000..d97a374 --- /dev/null +++ b/templates/404.html @@ -0,0 +1,14 @@ +{{ define "title" }}Page not found - Snowden Archive{{ end }} +{{ define "content" }} +
    +

    404 - Page not found

    +

    Nothing lives at {{ .Path }}. It may have been moved, removed, or mistyped.

    + + + +

    ← Home · Browse the archive

    +
    +{{ end }} +{{ template "base.html" . }} diff --git a/templates/about.html b/templates/about.html new file mode 100644 index 0000000..4e6b8c4 --- /dev/null +++ b/templates/about.html @@ -0,0 +1,544 @@ +{{ define "title" }}About - Snowden Archive{{ end }} +{{ define "content" }} +
    +

    About this archive

    + +

    This is a searchable archive of all published documents from the 2013 + Snowden disclosures. The archive includes documents published alongside + news reporting, The Intercept's bulk SIDtoday release, and FOIA material + (official releases from FOIA litigation and FISC and ODNI + declassifications).

    +

    Every document carries its original PDF, page images, extracted text, + and curated metadata - agencies, programs, countries, classification + markings, people, and the journalists and articles that published it.

    +

    Use Browse to explore by category, + Search for fielded boolean queries across text and + metadata, Articles to follow the disclosures story + by story, the Glossary to decode codenames and + acronyms, and Explore to dig deeper into ranked + views and lists.

    + + + +
    +

    Privacy

    +

    By default, the server keeps no state about its visitors. Pins and + history live only in your own browser's cookies, and history features + are off unless you opt in via Preferences. + Anonymous page views set no cookies at all. All core features on this + website work without JavaScript.

    +

    Note that you can't verify that this server doesn't log and analyze + your visits. Use Tor to hide your real IP address, or + set up your own instance on your machine to + browse fully offline.

    +
    + +
    +

    What's in this archive?

    +

    By default, two collections are shown as tabs on the + Browse page: Reported documents + (leaked files as the outlets that broke the stories presented them) and + the SIDtoday Archive (The Intercept's bulk release of + the NSA's internal SIDtoday newsletters). The third collection, + FOIA (official releases from FOIA litigation and FISC + and ODNI declassifications), can be included via + Preferences.

    +

    The metadata and many of the extracted texts are curated by hand and + may contain errors or omissions, so treat them as a guide to the + documents rather than as established fact, and check the original + documents and reporting before relying on them.

    +
    + + + +
    + {{ if .APIEnabled }} +

    Downloading data as JSON

    +

    API endpoints:

    +
    + + + + + + + + +
    EndpointReturns
    /api/docsMetadata for every document in the archive, with pagination.
    /api/doc/{id}A single document's metadata and extracted text, where {id} is the document's identifier.
    /api/search?q=xkeyscoreSearch results for term xkeyscore. It accepts the same parameters as the Search page, so you can also pass fqf/fqv/fqc clause parameters for fielded queries.
    /api/facetsCounts for every browsable category (programs, agencies, countries, people, and so on).
    +
    +

    Responses are plain JSON. For paginated responses, use the URL + parameter p to navigate pages (e.g. + /api/docs?p=2).

    +

    See the Downloading the archive + section on how to download the whole archive.

    +

    Important if you are a crawler, robot or AI trying to + download large amounts of data from this website: please use + the downloadable datasets from the + Downloading the archive section. The + information on this website is identical to the datasets, so you gain + nothing by crawling this website.

    + {{ end }} + +
    + {{ if .APIEnabled }}

    Metadata structure

    {{ else }}

    Metadata structure

    {{ end }}
    +

    Every document ships a curated metadata.json:

    + +

    The document record

    +
    +
    id
    +
    The document's stable identifier: the name of its directory + and the last part of its URL (/doc/{id}). It is not + written inside metadata.json - the directory name is + the id.
    + +
    aka_id
    +
    Other names the same document was released under, most often + the file name another publisher or a mirror gave its PDF. A list + of entries, each described below.
    + +
    aka_id[].id
    +
    The other name itself. Searchable, so a document can be found + by a file name it circulated under elsewhere.
    + +
    aka_id[].doc_source
    +
    The web page where the document was released under that name, + linked as “page” next to the alias.
    + +
    aka_id[].doc_url
    +
    The direct link to the PDF released under that name, linked + as “PDF” next to the alias.
    + +
    title
    +
    The document's title, written by the curators - usually the + title the document gives itself or the one its publisher used.
    + +
    description
    +
    A short summary of what the document is and says, shown under + the publication list and in search results and cards. + Descriptions are the work of archivists without inside knowledge - + corrections welcome.
    + +
    long_description
    +
    A longer account of the contents where one has been written, + behind “Show more info” on the document page; often + empty. A description: search looks in both + descriptions.
    + +
    tags
    +
    Free-text subjects added by the curators. Unlike the other + lists they are not necessarily taken from the document's own + vocabulary, so they are the loosest thing to browse or search by. +
    + +
    type
    +
    The kind of release: REPORTED (a leaked document + as an outlet published it), SIDTODAY (an article of + the NSA's internal SIDtoday newsletter from The Intercept's bulk + release) or FOIA (obtained through a public-records + request). When it is empty, a document filed in the SIDtoday or FOIA + folder takes that type and any other is REPORTED.
    + +
    collection
    +
    Which of the three collections - the tabs on + Browse - the document belongs to: + news (Reported), sidtoday-archive or + foia. Documents filed under the SIDtoday and FOIA + folders belong to those collections whatever this says; elsewhere + an empty value means Reported, or FOIA for a document of type + FOIA. FOIA releases are left out of + browsing and search unless you include them in + Preferences.
    + +
    pages
    +
    The page count of the PDF. When it is missing, the number of + page images is used instead.
    + +
    pdf_sha256
    +
    The SHA-256 of the PDF as archived.
    + +
    archive_comment
    +
    A note from the curators about this archive's copy - how it + was assembled, where it was found, what differs between releases, + or a known problem with it.
    +
    + +

    The document itself: document

    +
    +
    document.creator
    +
    The agency that wrote the document, in its short form + (NSA, GCHQ, FISC, + DOJ, CSE ...) - not the agency it is + about and not the one that published it. Empty when the author is + not known.
    + +
    document.date_created
    +
    When the document was written, as precise as the document + allows: 2013-03-20, 2013-03, + 2013, or empty when unknown. Nothing is invented to + fill the gaps, and nothing is guessed from the publication date. +
    + +
    document.banner
    +
    The marking line verbatim - what the + document says about itself, qualifiers and all. Intranet pages + often carry a ceiling banner (“DYNAMIC PAGE - HIGHEST + POSSIBLE CLASSIFICATION IS ...”) while their content is + portion-marked unclassified; the banner records the claim, not an + assessment. Empty means the artifact has no marking line - nothing + is reconstructed from portion marks.
    + +
    document.classification
    +
    The curator's assessment of the content's + classification, as component values: the level + (TOP SECRET, SECRET, + CONFIDENTIAL, UNCLASSIFIED) and any + control systems (COMINT, SI, + TK, STRAP 1 ...), judged from portion + marks. If a document contains multiple classifications, highest is + used here.
    + +
    document.dissemination
    +
    The curator's assessment of the handling controls, as + component values (FOUO, NOFORN, + ORCON, REL TO ...). Assessed the same + way as the classification.
    + +
    document.genre
    +
    The document's form, from a fixed vocabulary: + slides, report, wiki, + manual, guide, + newsletter, other.
    + +
    document.series.name
    +
    For a document that prints a series roster (currently only in + SIDtoday): the series name verbatim, as the document prints it. + Documents with the same name are listed together on the document + page and under Series.
    + +
    document.series.part
    +
    This document's 1-based position in the printed roster. Order + comes from the artifact; parts of a series not yet in the archive + show as gaps in the numbering, never renumbered away.
    + +
    document.programs
    +
    The codenames and programs the document names + (XKEYSCORE, PRISM, ...), as curated + values for browsing and search; the + Glossary explains many of them.
    + +
    document.countries
    +
    The countries the document mentions, by their English names. + A mention, not a claim that the country was a target or a partner. +
    + +
    document.people
    +
    The people who appear in the document: named in its text, or + recognised in its images - for example + Mobile Networks + in My NOC World lists King Charles and Queen Camilla, + who appear only in a photograph on one of its slides. Usually + written “Surname, Given name”. A person appearing in + these documents implies nothing about them beyond that appearance. + The journalists who published the document are under + publication[].reporters, not here.
    +
    + +

    Where it was published: publication[]

    +

    One entry per story or book that published (or reported) the + document. Browse-by-publication-date counts a document under its + oldest dated entry; a publication-date search matches + any entry's date - or, combined with a journalist or + publisher, that entry's date.

    +
    +
    publication[].publisher
    +
    Who published it: a news outlet, a book's publisher, or for a + FOIA release the organization that obtained and released it. + Browsable under Publishers - except + entries marked reporting_only.
    + +
    publication[].title
    +
    The headline of the article, or the book's title. Shown on the + document page and names the story on + Articles.
    + +
    publication[].link
    +
    The URL of the article. It is the story's identity: documents + whose entries share a link were published in the same report and + are listed together on its article page.
    + +
    publication[].date_published
    +
    When this publication appeared, in the same + YYYY-MM-DD / YYYY-MM / + YYYY forms as the document date.
    + +
    publication[].reporters
    +
    The journalists bylined on the story, written “Surname, + Given name”. Browsable under + Journalists and searchable with + journalist:.
    + +
    publication[].contributors
    +
    People credited on the story without a byline - additional + reporting, research, technical analysis. Shown on the document + page and found by an all-fields search, but not listed among the + journalists.
    + +
    publication[].note
    +
    Anything particular to this publication: that it showed only + some of the pages, published them in a different order, redacted + differently, or printed them in a book appendix.
    + +
    publication[].version
    +
    When this publication released a different copy from the one + shown on the document page, the name of that copy in the + document's versions/ folder; its entry then links to + “view this copy”. Without it, the entry published the + copy shown on the page.
    + +
    publication[].doc_source
    +
    The web page where this publication presented the document + (the outlet's own document page, a DocumentCloud page), linked as + “document page”. Where a copy came from is recorded + here, on the publication that released it, not on the + document.
    + +
    publication[].doc_url
    +
    The direct link to the PDF file this publication released, + linked as “original PDF”.
    + +
    publication[].reporting_only
    +
    true when the outlet reported the story without + publishing the document - a reporting partner. The entry keeps its + byline and its article, but the outlet is not counted as one of + the document's publishers and no copy is linked from it.
    +
    + +

    Other copies: versions

    +

    Other published copies of a document (e.g. less redacted, with extra + pages, or a different scan) are kept in its versions/ + folder, one sub-folder per copy, each with its own PDF, text and page + images. The versions object adds details about them, + keyed by the copy's folder name.

    +
    +
    versions.{name}.changed_pages
    +
    The pages of that copy, by its own page numbers, that differ + from the copy shown on the document page. They are linked as + “changed pages” next to the copy.
    + +
    versions.{name}.pdf_sha256
    +
    The SHA-256 of that copy's PDF.
    +
    +
    +
    + +
    +

    Downloading the archive

    +

    Each release comes in two packages:

    +
      +
    • full, ~18 GB: all the PDFs, page images (the + PDF pages as JPEG files), extracted texts, and metadata,
    • +
    • pdfs, ~3.5 GB: the same without the page + images.
    • +
    +

    Both extract to data/ and project/, ready + to run your own copy. Each release also + includes the source of the server version it was published with, and + the last six releases stay online. For only the extracted texts and + metadata (~58 MB), see below.

    + +

    Downloading all data

    + +

    HTTP

    +

    https://data.snowchive.org/files/

    +
    + See a curl example +
    +# download the latest release:
    +rel=$(curl -fsS https://data.snowchive.org/files/latest.txt)
    +base="https://data.snowchive.org/files/$rel"
    +curl -f -C - -O "$base/snowchive-$rel-full.tar" # "pdfs" instead of "full" for no page images
    +
    +# verify (optional):
    +curl -fsS -O "$base/SHA256SUMS" -O "$base/SHA256SUMS.gpg"
    +curl -fsS -O "https://libroot.org/public/libroot.asc"
    +# "gpg --show-keys libroot.asc" should show 3B054BC2E7B2CC825FF23E94108F97B8FC5B2EBC
    +gpg --import libroot.asc
    +gpg --verify SHA256SUMS.gpg SHA256SUMS
    +sha256sum --check --ignore-missing SHA256SUMS
    +
    +# extract to ./snowchive-data/:
    +mkdir -p snowchive-data && tar -xf "snowchive-$rel-full.tar" -C snowchive-data
    +
    +
    + +

    Torrent

    +

    Each release folder on + https://data.snowchive.org/files/ + has a .torrent for both packages.

    + +

    Downloading only extracted texts and metadata

    +

    Repository for the extracted texts and metadata: + https://git.disroot.org/libroot/snowchive-meta

    +
    +git clone https://git.disroot.org/libroot/snowchive-meta
    +cd snowchive-meta/data
    +
    +# grep only the extracted texts:
    +grep "fiber optic" -ri --include=*.md
    +
    +# grep only the metadata:
    +grep "fiber optic" -ri --include=*.json
    +
    +
    + +
    +

    Can I run my own copy or mirror this?

    +

    Yes. The server software behind this archive is free software with + no database and no external dependencies, designed to run offline or, + for example, as a Tor hidden service.

    +

    First, get a release package + and extract it: full for everything, or + pdfs for a site that works the same but shows no page + images.

    +

    Then get the server (Go required). Each release folder has the + source of the server version it was published with, + snowchive-server-<commit>.tar.gz, covered by the same + SHA256SUMS - the version known to work with that data:

    +
       
    +tar -xzf snowchive-server-<commit>.tar.gz
    +cd snowchive-server-<commit>
    +go run . -root=/path/to/snowchive-data/ -listen=127.0.0.1:8081
    +
    +

    Or clone the latest version from the + snowchive-server + repository:

    +
       
    +git clone https://git.disroot.org/libroot/snowchive-server
    +cd snowchive-server
    +go run . -root=/path/to/snowchive-data/ -listen=127.0.0.1:8081
    +
    +

    Recommended minimal setup for production instances:

    +
       
    +go build -ldflags "-X main.buildCommit=$(git rev-parse --short HEAD)" -o snowchive .
    +./snowchive -root=/path/to/snowchive-data/ -listen=127.0.0.1:8081 -base=https://archive.example.com
    +
    +

    System requirements: at least 300 MB of RAM (1 GB or more + recommended for production).

    +

    The maintainers of this project do not endorse any public Snowchive + instance other than the official snowchive.org instance (unless you + are a reputable organization that wants to host a public + instance).

    +

    See the + snowchive-server README + for the full documentation.

    +
    + +
    +

    Contact

    +

    If the archive is missing a document, or some metadata is wrong, + please submit an issue on the project repository: + https://git.disroot.org/libroot/snowchive-meta/issues

    +

    You can also report issues and errors or give feedback directly to + the project maintainers via email: + https://libroot.org/about

    +

    Please submit server-related issues to the Snowchive server + repository: + https://git.disroot.org/libroot/snowchive-server/issues

    +
    + +
    +

    Contributing

    +

    We welcome anyone who wants to contribute to the project, for + example by fixing errors in the metadata or in the extracted texts. + Please see the Contact section above.

    +

    Running this project doesn't cost much, but if you wish, you + can send a donation to:

    +
    +
    XMR
    +
    85gTBgYuLB4Mvo1ZCGo62126hBQQ678UeESTuSi3L9SECHXr6S7mXE28yAKvURgdJneGKqbPDcBydFfTkK7H5bvh2WXE4oT
    +
    BTC
    +
    bc1q5f5mc4pytajrn2xrp3lsftx64fw6hsdxxvs3ha
    +
    +
    + +
    +

    Thank you

    +

    We want to thank

    + +
    + +
    +{{ end }} +{{ template "base.html" . }} diff --git a/templates/article.html b/templates/article.html new file mode 100644 index 0000000..13a6de7 --- /dev/null +++ b/templates/article.html @@ -0,0 +1,25 @@ +{{ define "title" }}{{ with .Article.Title }}{{ . }}{{ else }}{{ if .Article.Book }}Book{{ else }}Article{{ end }} at {{ .Article.Host }}{{ end }} - Snowden Archive{{ end }} +{{ define "content" }} +
    +

    {{ if .Article.Book }}Book{{ else }}Article{{ end }}

    +

    {{ if .Article.Title }}{{ .Article.Title }}{{ else }}[untitled] {{ .Article.Host }}{{ end }}

    + +

    + {{ if .Article.Book }}About the book at{{ else }}Read the article at{{ end }} {{ .Article.Host }} ↗ +

    +
    + +

    Documents published with this {{ if .Article.Book }}book{{ else }}article{{ end }} {{ .Total }}

    +{{ if .Cards.Docs }} +{{ template "doc-cards" .Cards }} +{{ else }} +

    No documents found for this {{ if .Article.Book }}book{{ else }}article{{ end }}.

    +{{ end }} +{{ end }} +{{ template "base.html" . }} diff --git a/templates/articles.html b/templates/articles.html new file mode 100644 index 0000000..b79929d --- /dev/null +++ b/templates/articles.html @@ -0,0 +1,39 @@ +{{ define "title" }}Articles - Snowden Archive{{ end }} +{{ define "content" }} +
    +

    Articles + {{ .Total }} articles{{ with .Books }} and {{ . }} book{{ if ne . 1 }}s{{ end }}{{ end }} +

    +
    +

    + The news stories, and the few books, these documents were published with, newest first. + A document covered by several stories appears under each. +

    + +{{ if .Groups }} +{{ if .GroupsHTML }}{{ .GroupsHTML }}{{ else }}{{ template "article-groups" . }}{{ end }}{{ else }} +

    No articles.

    +{{ end }} +{{ end }} +{{ template "base.html" . }} + +{{ define "article-groups" }}{{ range .Groups }} +
    +

    {{ .Year }} {{ len .Articles }}

    +
      + {{ range .Articles }} +
    • + + + {{ if .Title }}{{ .Title }}{{ else }}[untitled] {{ .Host }}{{ end }} + + +
    • + {{ end }} +
    +
    +{{ end }} +{{ end }} diff --git a/templates/assemblies.html b/templates/assemblies.html new file mode 100644 index 0000000..5374430 --- /dev/null +++ b/templates/assemblies.html @@ -0,0 +1,32 @@ +{{ define "title" }}Assemblies - Snowden Archive{{ end }} +{{ define "content" }} +
    +

    ← Explore

    +

    Assemblies + {{ len .Assemblies }} +

    +
    +

    + Sets of pages this archive put together from documents and copies published + separately, so they can be read in one place. An assembly is our own + arrangement, not a document that was leaked or published in this form. +

    + +{{ if .Assemblies }} +
      + {{ range .Assemblies }} +
    • + {{ if not $.TextOnly }}{{ end }} +
      + {{ .Title }} +
      {{ len .Slides }} slide{{ if ne (len .Slides) 1 }}s{{ end }} from {{ .Copies }} cop{{ if eq .Copies 1 }}y{{ else }}ies{{ end }}
      + {{ with .Note }}

      {{ . }}

      {{ end }} +
      +
    • + {{ end }} +
    +{{ else }} +

    No assemblies yet.

    +{{ end }} +{{ end }} +{{ template "base.html" . }} diff --git a/templates/assembly.html b/templates/assembly.html new file mode 100644 index 0000000..8bc94dc --- /dev/null +++ b/templates/assembly.html @@ -0,0 +1,60 @@ +{{ define "title" }}{{ .Assembly.Title }} - Snowden Archive{{ end }} + +{{ define "assembly-credit" }}{{ with .Source }}{{ . }} · {{ end }}{{ .DocTitle }}{{ with .Version }} (copy: {{ . }}){{ end }}, p. {{ .Page }}{{ end }} + +{{ define "content" }} +{{ $a := .Assembly }}{{ $n := len $a.Slides }} +
    +

    ← Assemblies

    +

    {{ $a.Title }} {{ $n }} slide{{ if ne $n 1 }}s{{ end }}

    +

    Put together by this archive from {{ $a.Copies }} separately published cop{{ if eq $a.Copies 1 }}y{{ else }}ies{{ end }}. This is our arrangement, not a document in its own right.

    + {{ with $a.Note }}

    {{ . }}

    {{ end }} +
    + +{{ if .TextOnly }} +

    Text-only mode is on - page images are not loaded. Preferences

    + +{{ else }} + + +{{ range $i, $s := $a.Slides }} + +{{ range $k, $alt := $s.Alts }} + +{{ end }} +{{ end }} +{{ end }} +{{ end }} + +{{ template "base.html" . }} diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..fa87396 --- /dev/null +++ b/templates/base.html @@ -0,0 +1,35 @@ + + + + + + {{ block "title" . }}Snowden Archive{{ end }} + + + {{ block "head" . }}{{ end }} + + + + + {{ if .ShowCatBoxes }}{{ template "cat-boxes" . }}{{ end }} +
    + {{ block "content" . }}{{ end }} +
    +
    + {{ $b := buildInfo }} +

    Snowden Archive · changelog + · rev {{ $b.Commit }} +

    +
    + + diff --git a/templates/browse.html b/templates/browse.html new file mode 100644 index 0000000..0f4d09a --- /dev/null +++ b/templates/browse.html @@ -0,0 +1,155 @@ +{{ define "title" }}{{ if eq .Collection "foia" }}FOIA{{ else if eq .Collection "sidtoday-archive" }}SIDtoday Archive{{ else }}Reported{{ end }} documents - Snowden Archive{{ end }} +{{ define "content" }} + +
    + + + +{{ if eq .Collection "sidtoday-archive" }} + +
    +

    SIDtoday Archive ({{ .Total }} issues)

    +

    + All NSA SIDtoday newsletters in chronological order. This section + includes The Intercept's bulk release plus issues that were individually + reported by media (marked reported; those + also appear in the Reported collection). +

    + {{ with .SeriesList }} +
    + Series {{ len . }} - recurring columns and multi-part runs + +

    Full series table - sortable, with per-collection counts.

    +
    + {{ end }} + +
    + +{{ if eq .View "card" }} +
    {{ template "sort-bar" (dict "SortURL" .BaseURL "SortBy" .Sort "Order" .Order "ShowRelevance" false "DefaultSort" .DefaultSort) }}
    +

    Documents

    +{{ template "doc-cards" .Cards }} +{{ template "pagination" . }} +{{ else if eq .View "table" }} +{{ template "doc-table" . }} +{{ else }} +{{ if .ArchiveYears }} +{{ if .TimelineHTML }}{{ .TimelineHTML }}{{ else }}{{ template "archive-timeline" . }}{{ end }}{{ else }} +

    No documents in this collection yet.

    +{{ end }} +{{ end }} + +{{ else }} +
    +

    + {{ if eq .Collection "foia" }}FOIA Documents{{ else }}Reported Documents{{ end }} + ({{ .Total }} total) +

    + + {{ $cardURL := .SortURL }} + {{ $tableURL := print .SortURL "view=table" }} + {{ if .Sort }} + {{ $cardURL = print $cardURL "sort=" .Sort }} + {{ $tableURL = print $tableURL "&sort=" .Sort }} + {{ if .Order }} + {{ $cardURL = print $cardURL "&order=" .Order }} + {{ $tableURL = print $tableURL "&order=" .Order }} + {{ end }} + {{ end }} +
    + +
    +
    + +

    + {{ if eq .Collection "foia" }} + Documents released by government agencies under Freedom of Information Act + requests, rather than through the Snowden disclosures - obtained by + journalists, researchers, and transparency organizations and added here + for context. + {{ else }} + Documents published alongside news reporting on the Snowden disclosures. + {{ end }} +

    + +{{ if ne .View "table" }} +{{ template "sort-bar" (dict "SortURL" .SortURL "SortBy" .Sort "Order" .Order "ShowRelevance" false "DefaultSort" .DefaultSort) }} +{{ end }} + +{{ if eq .Collection "news" }} +{{ template "type-filter" (dict + "Action" "/browse" + "Params" .Params + "AllTypes" .AllTypes + "SelectedTypes" .SelectedTypes +) }} +{{ end }} + +{{ if eq .View "table" }} + {{ template "doc-table" . }} +{{ else }} +

    Documents

    + {{ template "doc-cards" .Cards }} + {{ template "pagination" . }} +{{ end }} + +{{ end }} + +{{ end }} +{{ template "base.html" . }} + +{{ define "archive-timeline" }} + +{{ range .ArchiveYears }} +
    +

    {{ .Year }} ({{ .Count }} issue{{ if ne .Count 1 }}s{{ end }})

    + {{ range .Months }} +
    + {{ with .Name }}

    {{ . }}

    {{ end }} +
      + {{ range .Docs }} +
    • + {{ with .DocMeta.DateCreated }}{{ . }}{{ else }}-{{ end }} + {{ .Title }} + [pdf] + {{ if .ReportedIssue }}reported{{ end }} +
    • + {{ end }} +
    +
    + {{ end }} +
    +{{ end }} +{{ end }} diff --git a/templates/category.html b/templates/category.html new file mode 100644 index 0000000..3e13a84 --- /dev/null +++ b/templates/category.html @@ -0,0 +1,59 @@ +{{ define "title" }}Browse by {{ .Title }} - Snowden Archive{{ end }} +{{ define "content" }} + +

    Browse by {{ .Title }}

    + +{{ with .Blurb }}

    {{ . }}

    {{ end }} + +{{ if .SeeAlso }} +

    See also: {{ range $i, $l := .SeeAlso }}{{ if $i }} · {{ end }}{{ $l.Text }}{{ end }}.

    +{{ end }} + +{{ if .ShowAZ }} + +{{ end }} + +{{ if .CollCols }} + + +{{ end }} +
    + + + {{ range .Heads }}{{ template "sort-th" . }}{{ end }} + + +{{ if .RowsHTML }}{{ .RowsHTML }}{{ else }}{{ template "category-rows" . }}{{ end }} +
    +
    + +{{ end }} +{{ template "base.html" . }} + +{{ define "category-row" }} + + + + {{ .Name }} + + {{- with .Note }} {{ . }}{{ end }} + + {{ .Count }} + {{ if .News }}{{ .News }}{{ else }}-{{ end }} + {{ if .Archive }}{{ .Archive }}{{ else }}-{{ end }} + {{- if not .NoFOIA }} + {{ if .FOIA }}{{ .FOIA }}{{ else }}-{{ end }} + {{- end }} + + {{ end }} +{{ define "category-az-row" }} + {{ . }} + {{ end }} +{{ define "category-rows" }}{{ $prevL := "" }}{{ range .Entries }}{{ if $.ShowAZ }}{{ if ne .FirstLetter $prevL }}{{ template "category-az-row" .FirstLetter }}{{ $prevL = .FirstLetter }}{{ end }}{{ end }}{{ template "category-row" . }}{{ end }}{{ end }} diff --git a/templates/changelog.html b/templates/changelog.html new file mode 100644 index 0000000..d5f859d --- /dev/null +++ b/templates/changelog.html @@ -0,0 +1,25 @@ +{{ define "title" }}Changelog - Snowden Archive{{ end }} +{{ define "content" }} +

    Changelog + {{ if .Total }}{{ .Total }} entries{{ end }} +

    + +{{ if .Entries }} +
      + {{ range .Entries }} +
    • + {{ if ge (len .Date) 10 }}{{ slice .Date 0 10 }}{{ else }}{{ .Date }}{{ end }} + {{ .Type }} + {{ if and .ID (ne .Type "removed") }} + {{ .Headline }} + {{ else }} + {{ .Headline }} + {{ end }} +
    • + {{ end }} +
    +{{ else }} +

    No changes recorded.

    +{{ end }} +{{ end }} +{{ template "base.html" . }} diff --git a/templates/classpatterns.html b/templates/classpatterns.html new file mode 100644 index 0000000..bc5d5ae --- /dev/null +++ b/templates/classpatterns.html @@ -0,0 +1,50 @@ +{{ define "title" }}Classification patterns - Snowden Archive{{ end }} + +{{ define "content" }} +
    +

    ← Explore

    +

    Classification patterns

    +
    + +

    Two views of the marking metadata. Every number counts the +documents carrying that exact classification component. A document carrying +several markings counts once per marking.

    + +

    Mix by document year

    +
    + {{ range $i, $v := .Levels }}{{ $v }}{{ end }} +
    +{{ range .Strips }} +
    + {{ .Year }} + + {{ range .Segs }} + + {{ end }} + + {{ .Total }} +
    +{{ end }} +{{ with .Sparse }}

    Not shown, with fewer than {{ $.MinMarks }} markings each: + {{ range $i, $y := . }}{{ if $i }}, {{ end }}{{ $y.Year }} ({{ $y.Total }}){{ end }}.

    {{ end }} +

    Agency × classification

    +
    + + + {{ range .Cols }}{{ end }} + + + {{ range .HeatRows }} + + + {{ range .Cells }} + + {{ end }} + + {{ end }} + +
    Agency{{ .Name }}
    {{ .Name }}{{ if .URL }}{{ .Count }}{{ else }}-{{ end }}
    +
    +{{ end }} + +{{ template "base.html" . }} diff --git a/templates/codewords.html b/templates/codewords.html new file mode 100644 index 0000000..efd4ed2 --- /dev/null +++ b/templates/codewords.html @@ -0,0 +1,38 @@ +{{ define "title" }}Acronyms & codewords - Snowden Archive{{ end }} + +{{ define "content" }} +
    +

    ← Explore

    +

    Acronyms & codewords

    +
    + +

    ALL-CAPS tokens (3+ letters) extracted automatically from titles and +transcripts - acronyms, program names, compartments, org designators. This is a +text statistic, not curated metadata. A token links to a search for the word in any +letter case, across the metadata too, so the search can find more documents than the +capitals counted here.

    + +
    + + + {{ range .Heads }}{{ template "sort-th" . }}{{ end }} + + +{{ if .RowsHTML }}{{ .RowsHTML }}{{ else }}{{ template "codeword-rows" . }}{{ end }} +
    +
    +{{ end }} + +{{ template "base.html" . }} + +{{ define "codeword-rows" }} {{ range .Rows }} + + {{ .Token }}{{ if .ProgURL }} program →{{ end }} + {{ .Count }} + {{ .Docs }} + {{ if .First }}{{ .First }}{{ else }}-{{ end }} + {{ if .Last }}{{ .Last }}{{ else }}-{{ end }} + {{ if .Gloss }}defined{{ else }}-{{ end }} + + {{ end }} +{{ end }} diff --git a/templates/doc.html b/templates/doc.html new file mode 100644 index 0000000..aa52af4 --- /dev/null +++ b/templates/doc.html @@ -0,0 +1,272 @@ +{{ define "title" }}{{ .Doc.Title }} - Snowden Archive{{ end }} + +{{ define "head" }} + + + + {{ if .Doc.DocMeta.Creator }}{{ end }} + {{ if .Doc.DocMeta.DateCreated }}{{ end }} + {{ if .Doc.Description }}{{ end }} + {{ range .Doc.DocMeta.Programs }}{{ end }} + {{ range .Doc.Tags }}{{ end }} + + + + + + + + + {{ if .Doc.Description }}{{ end }} + + {{ with .OGImageURL }}{{ end }} + + {{ if .APIEnabled }}{{ end }} + {{ .Embeds.JSONLDTag }} +{{ end }} + +{{ define "content" }} + +{{ $sidebar := .Prefs.InfoSidebar }} +
    +{{ if $sidebar }} + +{{ end }} + +
    +
    +
    +

    {{ .Doc.Title }}

    + Open PDF + {{ with .Doc.Versions }}+{{ len . }} other cop{{ if eq (len .) 1 }}y{{ else }}ies{{ end }}{{ end }} +
    + {{ $back := print "/doc/" .Doc.ID }} + + +
    + {{ if or .SerialPrev .SerialNext }} + + {{ end }} + +
    + Creating agency: {{ with .Doc.DocMeta.Creator }}{{ . }}{{ with agencyName . }} {{ . }}{{ end }}{{ else }}-{{ end }}
    + Document date: {{ with .Doc.DocMeta.DateCreated }}{{ . }}{{ else }}-{{ end }}
    + {{ if or (eq .Doc.Type "FOIA") (eq .Doc.Type "SIDTODAY") }}Type: {{ template "type-badge" .Doc.Type }}
    {{ end }} + {{ if .Doc.Pages }}Pages: {{ .Doc.Pages }}
    {{ end }} + {{ with .Doc.DocMeta.Genre }}Genre: {{ . }}
    {{ end }} + {{ with .Doc.DocMeta.Banner }}Banner:
    {{ end }} + Classification: + {{- facetLinks (dict "Vals" .Doc.DocMeta.Classification "Cat" "classification" "Sep" "space" "Class" "tag classification" "QS" $.BrowseQS) -}} +
    + + Dissemination: + {{- facetLinks (dict "Vals" .Doc.DocMeta.Dissemination "Cat" "dissemination" "Class" "doc-banner-link dissemination" "QS" $.BrowseQS) -}} +
    + + {{ with .SeriesGroup }} +
    + Series: + {{ .Name }} +
      + {{ range .Docs }} + + {{ if eq .ID $.Doc.ID }}{{ .Title }} + {{ else }}{{ .Title }}{{ end }} + + {{ end }} +
    +
    + {{ end }} + + {{ with .Assemblies }} +
    + Part of {{ if gt (len .) 1 }}assemblies{{ else }}an assembly{{ end }}: + {{ range $i, $a := . }}{{ if $i }}, {{ end }}{{ $a.Title }}{{ end }} +
    + {{ end }} + +
    + {{- range $i, $pub := .Doc.Publication }} +
    + {{- if or $pub.Link $pub.Publisher $pub.DatePublished }} +
    + {{- if $pub.Link }}{{ with $pub.Publisher }}{{ . }}{{ else }}Read the original{{ end }} ↗ + {{- else }}{{ with $pub.Publisher }}{{ . }}{{ end }}{{ end -}} + {{- with $pub.DatePublished }}{{ . }}{{ end -}} +
    + {{- end -}} + {{- if $pub.Title -}} + {{ $ak := articleKey $pub.Link }} +
    {{ if $ak }}{{ $pub.Title }}{{ else }}{{ $pub.Title }}{{ end }}
    + {{- end -}} + {{- if or $pub.Journalists $pub.Contributors }} +
    + {{- range $j, $name := $pub.Journalists -}} + {{- if $j }} · {{ end -}} + {{ $name }} + {{- end -}} + {{- range $j, $name := $pub.Contributors -}} + {{- if or $j $pub.Journalists }} · {{ end -}} + {{ $name }} + {{- end -}} +
    + {{- end -}} + {{- if $pub.ReportingOnly }} +
    Reporting partner - published this story, not the document
    + {{- end -}} + {{- with $pub.Note }} +
    {{ . }}
    + {{- end -}} + {{- $copy := $.Doc.VersionFor $pub.Version -}} + {{- $shown := and (not $pub.ReportingOnly) (not $pub.Version) $.Doc.Versions -}} + {{- if and (not $pub.ReportingOnly) (or $copy $pub.DocSource $pub.DocURL $shown) }} +
    + {{- if $shown }}the copy shown on this page{{ end -}} + {{- with $copy }}view this copy →{{ if .ChangedPages }}changed pages: {{ template "changed-page-links" . }}{{ end }}{{ end -}} + {{- with $pub.DocSource }}document page ↗{{ end -}} + {{- with $pub.DocURL }}original PDF ↗{{ end -}} +
    + {{- end -}} +
    + {{- end }} + {{- with .Doc.UnlinkedVersions }} +
    +
    Other copies
    +
    + {{- range . }}{{ .Slug }} →{{ if .ChangedPages }} (changed pages: {{ template "changed-page-links" . }}){{ end }}{{ end -}} +
    +
    + {{- end }} +
    + +
    +

    {{ .Doc.Description }}

    + +
    + + + +
    +
    +
    + Long Description: +
    + {{- with .Doc.LongDescription }}{{ . }}{{ else }}No extended description available.{{ end -}} +
    +
    + +
    + Programs:
    + {{- facetLinks (dict "Vals" .Doc.DocMeta.Programs "Cat" "program" "Class" "tag" "AsList" .Prefs.ProgramsAsList "QS" $.BrowseQS) -}} + {{- if not (and .Doc.DocMeta.Programs .Prefs.ProgramsAsList) }}
    {{ end -}} +
    + + People mentioned:
    + {{- facetLinks (dict "Vals" .Doc.DocMeta.People "Cat" "people" "Class" "meta-link" "QS" $.BrowseQS) -}} +

    + + Countries:
    + {{- facetLinks (dict "Vals" .Doc.DocMeta.Countries "Cat" "country" "Class" "meta-link" "QS" $.BrowseQS) -}} +

    + + Tags:
    + {{- facetLinks (dict "Vals" .Doc.Tags "Cat" "tag" "Class" "meta-link" "AsList" .Prefs.TagsAsList "QS" $.BrowseQS) -}} + +

    + Identifier:
    + {{ .Doc.ID }} + {{ with .Doc.AkaID }} +
    Also known as:
    + {{ range $i, $aka := . }}{{ if $i }}
    {{ end }}{{ $aka.ID }}{{ with $aka.DocSource }} page ↗{{ end }}{{ with $aka.DocURL }} PDF ↗{{ end }}{{ end }} + {{ end }} + {{ with .Doc.PDFSha256 }} +

    + PDF SHA-256:
    + {{ . }} + {{ end }} +
    +
    +
    +
    + + +
    +
    +
    +
    + +{{ if $sidebar }} + +{{ end }} + +
    + + {{ template "doc-viewport" . }} +
    +
    {{/* /.doc-layout */}} + +{{ if and .Images (not .Prefs.TextOnly) }} +{{ template "lightbox-overlays" . }} + +{{ end }} + +
    + {{ with .Doc.ArchiveComment }} +
    + Archive note: {{ . }} +
    + {{ end }} + + +
    + +{{ if or .PublishedTogether .RelatedDocs }} + +{{ end }} + +{{ end }} + +{{ template "base.html" . }} diff --git a/templates/explore.html b/templates/explore.html new file mode 100644 index 0000000..d67faa5 --- /dev/null +++ b/templates/explore.html @@ -0,0 +1,50 @@ +{{ define "title" }}Explore - Snowden Archive{{ end }} +{{ define "content" }} + +

    Explore the archive

    +

    + Ranked views and lists across the whole archive, which the + /browse doesn't include. +

    + +
      + {{ range .Metrics }} +
    • + {{ .Title }} + - {{ .Blurb }} +
    • + {{ end }} +
    • + Portion markings + - The marking on each classified portion, read out of the OCRs (levels, compartments, and how often). +
    • +
    • + Classification patterns + - The marking mix across document years, and an agency x classification heat table. +
    • +
    • + Publishers by year + - When each publisher released documents, and how much (i.e. documents and stories per publisher and year). +
    • +
    • + Technical IDs + - Every document id, PDF filename, size and SHA-256 checksum. +
    • +
    • + Genres + - How many documents of each kind the archive holds (slides, reports, wikis, manuals, guides, newsletters). +
    • + {{ if $.HasAssemblies }} +
    • + Assemblies + - Sets of pages this archive put together from documents and copies published separately, so they can be read in one place. +
    • + {{ end }} +
    • + Acronyms & codewords + - Every ALL-CAPS token across the corpus, auto-extracted (e.g. acronyms, program names, compartments, org designators). +
    • +
    + +{{ end }} +{{ template "base.html" . }} diff --git a/templates/explore_table.html b/templates/explore_table.html new file mode 100644 index 0000000..ba873c3 --- /dev/null +++ b/templates/explore_table.html @@ -0,0 +1,35 @@ +{{ define "title" }}{{ .Metric.Title }} - Snowden Archive{{ end }} +{{ define "content" }} + +

    ← Explore

    +

    {{ .Metric.Title }}

    +

    {{ .Metric.Blurb }}

    +

    Top {{ .Shown }} of {{ .Total }} documents.

    + +
    + + + {{ range .Heads }}{{ template "sort-th" . }}{{ end }} + + +{{ if .RowsHTML }}{{ .RowsHTML }}{{ else }}{{ template "explore-rows" . }}{{ end }} +
    +
    + +{{ end }} +{{ template "base.html" . }} + +{{ define "explore-rows" }} {{ range .Rows }} + + {{ .Rank }} + + {{ .Doc.Title }} + {{ template "type-badge" .Doc.Type }} + + {{ with .Doc.DocMeta.DateCreated }}{{ . }}{{ else }}-{{ end }} + {{ range .Vals }} + {{ . }} + {{ end }} + + {{ end }} +{{ end }} diff --git a/templates/glossary.html b/templates/glossary.html new file mode 100644 index 0000000..a6a9270 --- /dev/null +++ b/templates/glossary.html @@ -0,0 +1,36 @@ +{{ define "title" }}Glossary - Snowden Archive{{ end }} +{{ define "content" }} +

    Glossary {{ if .Total }}({{ .Total }} terms){{ end }}

    + +{{ if .Groups }} + + +{{ if .GroupsHTML }}{{ .GroupsHTML }}{{ else }}{{ template "glossary-groups" . }}{{ end }}{{ else }} +

    No glossary entries.

    +{{ end }} +{{ end }} +{{ template "base.html" . }} + +{{ define "glossary-groups" }}{{ range .Groups }} +
    +

    {{ .Letter }}

    +
    + {{ range .Entries }} +
    + {{ .Term }} + {{ if .Aka }}(a.k.a. {{ range $i, $a := .Aka }}{{ if $i }}, {{ end }}{{ $a }}{{ end }}){{ end }} + search → +
    +
    + {{ .DefinitionHTML }} + {{ if .SeeLinks }}
    See also: + {{ range $i, $s := .SeeLinks }}{{ if $i }}, {{ end }}{{ if $s.Term }}{{ $s.Name }}{{ else }}{{ $s.Name }}{{ end }}{{ end }} +
    {{ end }} +
    + {{ end }} +
    +
    +{{ end }} +{{ end }} diff --git a/templates/ids.html b/templates/ids.html new file mode 100644 index 0000000..73d9692 --- /dev/null +++ b/templates/ids.html @@ -0,0 +1,63 @@ +{{ define "title" }}Technical IDs - Snowden Archive{{ end }} + +{{ define "content" }} +
    +

    ← Explore

    +

    Technical IDs {{ .Shown }} document{{ if ne .Shown 1 }}s{{ end }}

    +
    + +

    + Checksums as plain text + (sha256sum -c in the corpus root) +

    + +
    + + + + + + + + + + + + + + + + + {{ if .RowsHTML }}{{ .RowsHTML }}{{ else }}{{ template "id-rows" . }}{{ end }} + +
    DocumentCollectionPagesPDFSizeSHA-256
    +
    +{{ end }} + +{{ template "base.html" . }} + +{{ define "id-rows" }}{{ range .Rows }} + + {{ .ID }} + {{- range .Aka }}
    also published as {{ .ID }}{{ end -}} + + {{ .Collection }} + {{ if .Pages }}{{ .Pages }}{{ end }} + {{ .PDFFile }} + {{ .Size }} + {{ with .SHA }}{{ . }}{{ else }}-{{ end }} + + {{ $id := .ID }} + {{ range .Versions }} + + ↳ versions/{{ .Slug }} + copy + {{ if .Pages }}{{ .Pages }}{{ end }} + {{ .PDFFile }} + {{ .Size }} + {{ if .SHA }}{{ .SHA }}{{ else }}-{{ end }} + + {{ end }} + {{ else }} + No documents. + {{ end }}{{ end }} diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..105d940 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,150 @@ +{{ define "content" }} +

    Snowden Archive

    + +

    + This archive holds {{ .SnowdenTotal }} published Snowden documents. {{ .ReportedTotal }} documents are + published with news reporting, and {{ .ArchiveTotal }} documents are the NSA newsletter SIDtoday + issues, from The Intercept's bulk release.{{ if .FOIATotal }} This archive also holds {{ .FOIATotal }} + FOIA documents related to the Snowden disclosures (FOIA litigation, FISC and ODNI declassifications), + which {{ if .IncludeFOIA }}you have included via Preferences{{ else }}are left out by default but can be included via Preferences{{ end }}.{{ end }} +

    + + + + + +{{ if and .TrackRecent .RecentDocs }} +
    +

    Recently viewed + +

    +
    + + +
    +
      + {{ range .RecentDocs }} +
    • {{ .Title }} {{ .DocMeta.Creator }}
    • + {{ end }} +
    +
    +{{ end }} + +

    {{ if .WithSIDtoday }}The statistics below include SIDtoday archive documents (exclude them).{{ else }}The statistics below exclude SIDtoday archive documents (include them).{{ end }}

    + +{{ if .DocHist }} +
    +

    When the documents were written

    +
    + {{ range .DocHist }} + + {{ .Count }} + + {{ .Year }} + + {{ end }} +
    +
    +{{ end }} + +{{ if .PubHist }} +
    +

    When they were published

    +
    + {{ range .PubHist }} + + {{ .Count }} + + {{ .Year }} + + {{ end }} +
    +
    +{{ end }} + +{{ if .TopPrograms }} +
    +

    Top programs all →

    +
    + {{ range .TopPrograms }} + {{ .Name }} {{ .Count }} + {{ end }} +
    +
    +{{ end }} + +
    + {{ if .TopAgencies }} +
    +

    Agencies all →

    +
      + {{ range .TopAgencies }} +
    • {{ .Name }} {{ .Count }}
    • + {{ end }} +
    +
    + {{ end }} + + {{ if .TopCountries }} +
    +

    Countries all →

    +
      + {{ range .TopCountries }} +
    • {{ .Name }} {{ .Count }}
    • + {{ end }} +
    +
    + {{ end }} + + {{ if .TopPeople }} +
    +

    People all →

    +
      + {{ range .TopPeople }} +
    • {{ .Name }} {{ .Count }}
    • + {{ end }} +
    +
    + {{ end }} + + {{ if .TopJournalists }} +
    +

    Journalists all →

    +
      + {{ range .TopJournalists }} +
    • {{ .Name }} {{ .Count }}
    • + {{ end }} +
    +
    + {{ end }} + + {{ if .TopPublishers }} +
    +

    Publishers all →

    +
      + {{ range .TopPublishers }} +
    • {{ .Name }} {{ .Count }}
    • + {{ end }} +
    +
    + {{ end }} +
    + +{{ if .TopTags }} +
    +

    Top tags all →

    +
    + {{ range .TopTags }} + {{ .Name }} {{ .Count }} + {{ end }} +
    +
    +{{ end }} + +
    +

    {{ with .CatLabel }}{{ . }}{{ else }}Category{{ end }}: {{ .Title }} + {{- if eq .ActiveCategory "agency" }}{{ with agencyName .Title }} {{ . }}{{ end }}{{ end }} + {{ .Total }} document{{ if ne .Total 1 }}s{{ end }} +

    + {{ if .SearchLink }} + Search documents matching “{{ .Title }}” → + {{ end }} + {{ with .Blurb }}

    {{ . }}

    {{ end }} + {{ with .YearScope }} +

    Showing the documents {{ $.Title }} itself published in {{ . }} - + all years.

    + {{ end }} +
    + +{{ if .MonthNav }} + +{{ end }} + +{{ with .GlossaryEntry }} +
    + {{ .Term }} + {{ if .Aka }}(a.k.a. {{ range $i, $a := .Aka }}{{ if $i }}, {{ end }}{{ $a }}{{ end }}){{ end }} + - {{ .DefinitionHTML }} + glossary → +
    +{{ end }} + +{{ if or .Timeline .PubTimeline .Breakdowns }} +
    + Graphs + {{ if .Timeline }}{{ template "timeline-hist" (dict "Bars" .Timeline "Label" "By document date" "Verb" "dated") }}{{ end }} + {{ if .PubTimeline }}{{ template "timeline-hist" (dict "Bars" .PubTimeline "Label" "By published date" "Verb" "published") }}{{ end }} + {{ range .Breakdowns }} +
    +

    By {{ .Label }}

    +
    + {{ range .Values }} + + {{ .Value }} + + {{ .Count }} + + {{ end }} +
    +
    + {{ end }} +
    +{{ end }} + +{{ if .Refinements }} +
    + Refine (combine with another filter, via search) +
    + {{ range .Refinements }} +
    + {{ .Label }} + {{ range .Values }}{{ .Name }}{{ .Count }}{{ end }} +
    + {{ end }} +
    +
    +{{ end }} + +
    + {{ if ne .View "table" }} + {{ template "sort-bar" (dict "SortURL" .SortURL "SortBy" .Sort "Order" .Order "ShowRelevance" false "DefaultSort" .DefaultSort) }} + {{ end }} + {{ template "type-filter" (dict + "Action" (print "/browse/" .ActiveCategory "/" (pathEscape .Title)) + "Params" .Params + "AllTypes" .AllTypes + "SelectedTypes" .SelectedTypes + ) }} +
    + +{{ if eq .View "table" }} + {{ template "doc-table" . }} +{{ else }} +

    Documents

    + {{ template "doc-cards" .Cards }} + {{ template "pagination" . }} +{{ end }} +{{ end }} + + +{{ template "base.html" . }} diff --git a/templates/outlets.html b/templates/outlets.html new file mode 100644 index 0000000..b9fe263 --- /dev/null +++ b/templates/outlets.html @@ -0,0 +1,54 @@ +{{ define "title" }}Publishers by year - Snowden Archive{{ end }} + +{{ define "content" }} +
    +

    ← Explore

    +

    Publishers by year

    +
    + +

    When each publisher released documents, and how much. The number of +distinct stories are after the slash.

    + +

    Showing: + {{ if .NewsOnly }}all collections · reported only + {{ else }}all collections · reported only{{ end }} +

    + +{{ with .Table }} +{{ if .Rows }} +
    + + + + + {{ range .YearLinks }}{{ end }} + + + + + {{ range .Rows }} + + + {{ range .Cells }} + + {{ end }} + + + {{ end }} + + + + + {{ range .Totals }}{{ end }} + + + +
    Outlet{{ .Name }}Total
    {{ .Name }}{{ if .Docs }}{{ .Docs }}{{ if .Articles }}/{{ .Articles }}{{ end }}{{ else }}-{{ end }}{{ .Total }}
    All outlets{{ if . }}{{ . }}{{ else }}-{{ end }}
    +
    +{{ else }} +

    No dated publications in the corpus yet.

    +{{ end }} +{{ end }} +{{ end }} + +{{ template "base.html" . }} diff --git a/templates/pages.html b/templates/pages.html new file mode 100644 index 0000000..872ce4f --- /dev/null +++ b/templates/pages.html @@ -0,0 +1,40 @@ +{{ define "title" }}Pages - {{ .Doc.Title }}{{ with .Version }} ({{ .Slug }}){{ end }} - Snowden Archive{{ end }} + +{{ define "content" }} +
    +

    {{ with .Version }}← {{ $.Doc.Title }} (copy: {{ .Slug }}){{ else }}← {{ .Doc.Title }}{{ end }}

    +

    All pages {{ len .Images }} page{{ if ne (len .Images) 1 }}s{{ end }}

    + {{ with .Version }}{{ if .ChangedPages }}

    Pages that differ from the main document: {{ template "changed-page-links" . }}

    {{ end }}{{ end }} +
    + +{{ if and .Images .TextOnly }} +

    Text-only mode is on - page images are not loaded. Preferences

    + +{{ else if .Images }} + + + +{{ template "lightbox-overlays" . }} + +{{ else }} +

    This document has no page images.

    +{{ end }} +{{ end }} + +{{ template "base.html" . }} diff --git a/templates/partials.html b/templates/partials.html new file mode 100644 index 0000000..225098e --- /dev/null +++ b/templates/partials.html @@ -0,0 +1,412 @@ +{{ define "cat-boxes" }} +{{ $a := .ActiveCategory }}{{ if not $a }}{{ $a = "" }}{{ end }} + +{{ end }} + +{{/* pub-publisher is a publication entry's outlet: a link to the story where + the entry has one, else to the outlet's page - which a partner that only + reported has none of. */}} +{{ define "pub-publisher" -}} +{{ if .Pub.Link }}{{ .Pub.Publisher }} ↗ +{{- else if .Pub.ReportingOnly }}{{ .Pub.Publisher }} +{{- else }}{{ .Pub.Publisher }}{{ end }} +{{- end }} + +{{ define "sort-th" }}{{ if .URL }}{{ .Label }} {{ if .Active }}{{ if .Desc }}▼{{ else }}▲{{ end }}{{ else }}↕{{ end }}{{ else }}{{ .Label }}{{ end }}{{ end }} + +{{ define "doc-table" }} +
    + + + {{ range .TableHeads }}{{ template "sort-th" . }}{{ end }} + + +{{ if .RowsHTML }}{{ .RowsHTML }}{{ else }}{{ range .TableRows }}{{ template "doc-table-row" . }}{{ end }}{{ end }} +
    +
    +{{ end }} + +{{/* The rows are rendered once for every reader, so a FOIA release's links + always name every collection: they must list the release itself. */}} +{{ define "doc-table-row" }}{{ $qs := "" }}{{ if eq .Collection "foia" }}{{ $qs = wholeArchiveQS }}{{ end }} + + + {{ .Title }} + [pdf] + + + {{ if .DocMeta.Creator }}{{ .DocMeta.Creator }}{{ else }}-{{ end }} + + + {{ with .DocMeta.DateCreated }}{{ . }}{{ else }}-{{ end }} + + + {{- $first := true -}} + {{- range .Publication -}} + {{- if .DatePublished -}} + {{- if not $first -}}, {{ end -}} + {{ .DatePublished }} + {{- $first = false -}} + {{- end -}} + {{- end -}} + {{- if $first -}}-{{- end -}} + + {{ facetLinks (dict "Vals" .DocMeta.Classification "Cat" "classification" "QS" $qs) }} + {{ facetLinks (dict "Vals" .DocMeta.Dissemination "Cat" "dissemination" "QS" $qs) }} + {{ facetLinks (dict "Vals" .DocMeta.Programs "Cat" "program" "QS" $qs) }} + {{ facetLinks (dict "Vals" .DocMeta.Countries "Cat" "country" "QS" $qs) }} + {{ facetLinks (dict "Vals" .DocMeta.People "Cat" "people" "QS" $qs) }} + {{ facetLinks (dict "Vals" .JournalistsDeduped "Cat" "journalist" "QS" $qs) }} + + {{- $first = true -}} + {{- range .Publication -}} + {{- if .Publisher -}} + {{- if not $first -}}, {{ end -}} + {{ template "pub-publisher" (dict "Pub" . "QS" $qs) }} + {{- $first = false -}} + {{- end -}} + {{- end -}} + {{- if $first -}}-{{- end -}} + + + {{ template "type-badge" .Type }} + {{ if and .MarkReported .ReportedIssue }}reported{{ end }} + + +{{ end }} + +{{ define "doc-cards" }} +
      + {{ range .Docs }} +
    • + + {{ if $.ShowThumbs }} + + {{ end }} + +
      + {{ $cat := $.ActiveCategory }}{{ $val := $.ActiveValue }} + {{/* A FOIA release's links name every collection for a reader who + leaves FOIA out: the pages they open must list the release. */}} + {{ $qs := "" }}{{ if eq .Collection "foia" }}{{ $qs = $.FOIAQS }}{{ end }} + +

      + {{ if .Title }}{{ .Title }}{{ else }}(untitled){{ end }} + {{ template "type-badge" .Type }} + {{ if and $.MarkReported .ReportedIssue }}reported{{ end }} + [pdf] + {{ if $.ShowUnpin }} + + {{ end }} +

      + {{ if $.ShowUnpin }} +
      + +
      + {{ end }} + +
      + +
      + {{ if .DocMeta.Creator }} + {{ .DocMeta.Creator }} + {{ end }} + {{ if and .DocMeta.Creator .DocMeta.DateCreated }} + · + {{ end }} + {{ if .DocMeta.DateCreated }} + {{ if ge (len .DocMeta.DateCreated) 4 }} + {{ $year := slice .DocMeta.DateCreated 0 4 }} + {{ .DocMeta.DateCreated }} + {{ else }} + {{ .DocMeta.DateCreated }} + {{ end }} + {{ end }} + {{ if .Pages }} + {{ if or .DocMeta.Creator .DocMeta.DateCreated }} + · + {{ end }} + {{ .Pages }} p. + {{ end }} +
      + + {{ if and .Description (not $.HideSummary) }} +

      {{ .Description }}

      + {{ end }} + + {{ if and (or .DocMeta.Classification .DocMeta.Dissemination) (not $.HideClassification) }} +
      + Marking: + {{ with .DocMeta.Classification }}{{ facetLinks (dict "Vals" . "Cat" "classification" "Sep" "dot" "Active" (and (eq $cat "classification") $val) "QS" $qs) }}{{ end }} + {{ if and .DocMeta.Classification .DocMeta.Dissemination }}·{{ end }} + {{ with .DocMeta.Dissemination }}{{ facetLinks (dict "Vals" . "Cat" "dissemination" "Sep" "dot" "Active" (and (eq $cat "dissemination") $val) "QS" $qs) }}{{ end }} +
      + {{ end }} + + {{ if and .DocMeta.Programs (not $.HidePrograms) }} +
      + Programs: + {{ facetLinks (dict "Vals" .DocMeta.Programs "Cat" "program" "Sep" "dot" "Active" (and (eq $cat "program") $val) "QS" $qs) }} +
      + {{ end }} + + {{ if and .DocMeta.People (not $.HidePeople) }} +
      + People: + {{ facetLinks (dict "Vals" .DocMeta.People "Cat" "people" "Sep" "dot" "Active" (and (eq $cat "people") $val) "QS" $qs) }} +
      + {{ end }} + +
      + {{ $shown := false }} + {{ range $pub := .Publication }} + {{ if or $pub.Title $pub.Link $pub.Publisher $pub.DatePublished }} + {{ if $shown }}·{{ end }}{{ $shown = true }} + {{ if $pub.Title }} + {{ $ak := articleKey $pub.Link }} + {{ if $ak }}“{{ $pub.Title }}”{{ else }}“{{ $pub.Title }}”{{ end }} + - + {{ end }} + {{ if or $pub.Link $pub.Publisher }} + {{ template "pub-publisher" (dict "Pub" $pub "QS" $qs "LinkClass" "pub-link" "PageClass" "pub-source" "On" (and (eq $cat "publisher") (eqFold $pub.Publisher $val))) }} + {{ end }} + {{ if $pub.DatePublished }} + ({{ $pub.DatePublished }}) + {{ end }} + {{ end }} + {{ end }} + + {{ $journos := .JournalistsDeduped }} + {{ if and $journos (not $.HideJournalists) }} + · + {{ range $i, $j := $journos }} + {{ if $i }}·{{ end }} + {{ $j }} + {{ end }} + {{ end }} +
      + +
      +
      +
    • + {{ end }} +
    +{{ end }} + +{{ define "type-filter" }} +
    + {{ range $k, $vals := .Params }} + {{ if and (ne $k "type") (ne $k "p") (ne $k "action") }} + {{ range $vals }} + + {{ end }} + {{ end }} + {{ end }} + + Show: + + {{ range $type, $count := .AllTypes }} + + {{ end }} + + +
    +{{ end }} + + +{{ define "pagination" }} +{{ if gt .TotalPages 1 }} +{{ $sortParam := "" }}{{ if .Sort }}{{ $sortParam = print "&sort=" .Sort }}{{ end }} +{{ $orderParam := "" }}{{ if .Order }}{{ $orderParam = print "&order=" .Order }}{{ end }} + +{{ end }} +{{ end }} + +{{ define "timeline-hist" }} +
    +

    {{ .Label }} ({{ len .Bars }} years)

    +
    + {{ range .Bars }} + {{ if .URL }} + + {{ .Count }} + + {{ .Year }} + + {{ else }} + + {{ .Count }} + + {{ .Year }} + + {{ end }} + {{ end }} +
    +
    +{{ end }} + +{{ define "sort-bar" }} +{{ $base := .SortURL }} +{{ $order := .Order }} +{{ $showRelevance := .ShowRelevance }} + +{{ $cur := .SortBy }} +{{ if eq $cur "" }}{{ $cur = .DefaultSort }}{{ end }} + +{{ $activeLabel := sortLabel $cur }} +{{ $directed := ne $cur "relevance" }} +{{ $orderLabel := "↑" }}{{ if eq $order "desc" }}{{ $orderLabel = "↓" }}{{ end }} + +
    + + Sort: {{ $activeLabel }} + {{ if $directed }}{{ end }} + + +
    +{{ end }} + +{{ define "type-badge" }} +{{ if eq . "FOIA" }} +FOIA +{{ else if eq . "SIDTODAY" }} +SIDtoday +{{ end }} +{{ end }} + +{{ define "lightbox-overlays" }} +{{ $lbBase := .ImageBase }}{{ $lbN := len .Images }} +{{ range $i, $img := .Images }} + +{{ end }} +{{ end }} + +{{ define "changed-page-links" }} +{{- $v := . -}} +{{- range $k, $p := .ChangedPages -}} +{{- if $k }}, {{ end -}} +{{- if $p.Pos }}{{ $p.Page }}{{ else }}{{ $p.Page }}{{ end -}} +{{- end -}} +{{ end }} + +{{ define "doc-viewport" }} + + + + +
    + + + +
    + +
    +
    + {{ if gt (len .Images) 1 }} + + {{ end }} +
    + {{ if .Prefs.TextOnly }} +

    Text-only mode is on - page images are not loaded. Preferences

    + + {{ else }} + {{ range $i, $img := .Images }} + + Page {{ add $i 1 }} + {{- if and $.Version ($.Version.PageChanged (add $i 1)) }}differs{{ end }} + + {{ end }} + {{ end }} +
    +
    + +
    +
    + {{ .Content }} +
    +
    +
    +{{ end }} diff --git a/templates/pinned.html b/templates/pinned.html new file mode 100644 index 0000000..423d6fd --- /dev/null +++ b/templates/pinned.html @@ -0,0 +1,46 @@ +{{ define "title" }}Pinned - Snowden Archive{{ end }} +{{ define "content" }} +
    +

    Pinned documents + {{ .Total }} pinned + {{ if .Total }}export JSON{{ end }} +

    +
    +

    Pins are stored only in this browser's cookies.

    +{{ if .Full }} +

    Your pinned list is full. A browser cookie holds only so much, so that pin was not added. Unpin a document or a search below to make room.

    +{{ end }} + +{{ if .Cards.Docs }} +

    Documents

    +{{ template "doc-cards" .Cards }} +{{ else }} +

    Nothing pinned yet. Open a document and press ☆ Pin to start a reading list.

    +{{ end }} + +{{ if .Searches }} +

    Pinned searches

    +
      + {{ range .Searches }} +
    • + {{ .Label }} +
      + + + +
      +
    • + {{ end }} +
    +{{ end }} + + + +{{ end }} +{{ template "base.html" . }} diff --git a/templates/portions.html b/templates/portions.html new file mode 100644 index 0000000..fa0dcd6 --- /dev/null +++ b/templates/portions.html @@ -0,0 +1,82 @@ +{{ define "title" }}Portion markings - Snowden Archive{{ end }} + +{{ define "content" }} +
    +

    ← Explore

    +

    Portion markings

    +
    + +

    The marking each classified portion of a document carries at the +start of its own line (for example "(U//FOUO)", "(TS//SI//NF)").

    + +

    By level

    +
    + + + + {{ range .Stats.Levels }} + + + + + + + {{ end }} + +
    LevelMarkingsDocuments
    {{ .Name }}{{ .Label }}{{ .Marks }}{{ .Docs }}
    +
    + +

    Markings

    +

    Every distinct marking, most used first. +{{ if gt .Stats.Distinct .RowLimit }}Top {{ .RowLimit }} of {{ .Stats.Distinct }}.{{ end }}

    +
    + + + + {{ range .Stats.Markings }} + + + + + + {{ end }} + +
    MarkingMarkingsDocuments
    {{ .Name }}{{ .Marks }}{{ .Docs }}
    +
    + +

    Control markings

    +

    The compartments and dissemination controls appearing inside +those markings.

    +
    + + + + {{ range .Stats.Controls }} + + + + + + {{ end }} + +
    ControlMarkingsDocuments
    {{ .Name }}{{ .Marks }}{{ .Docs }}
    +
    + +{{ if .Stats.UnrecogEg }} +

    Not recognised

    +

    {{ .Stats.Unrecog }} markings open a line with a classification +level but carry a part this server has no rule for.

    +
    + + + + {{ range .Stats.UnrecogEg }} + + {{ end }} + +
    As the transcript writes itOccurrences
    ({{ .Name }}){{ .Marks }}
    +
    +{{ end }} +{{ end }} + +{{ template "base.html" . }} diff --git a/templates/preferences.html b/templates/preferences.html new file mode 100644 index 0000000..fe0057c --- /dev/null +++ b/templates/preferences.html @@ -0,0 +1,97 @@ +{{ define "title" }}Preferences - Snowden Archive{{ end }} +{{ define "content" }} +

    Preferences

    +

    Preferences are saved in browser cookies.

    + +{{ if .Saved }}

    Settings saved.

    {{ end }} +{{ if .Reset }}

    Preferences reset to defaults.

    {{ end }} + +
    +
    + Theme + + + + +
    + +
    + Browse lists + + + + + + + + + +
    + +
    + FOIA releases + +
    + +
    + Document layout + + + +
    + +
    + Document details + + + + + +
    + +
    + History (off by default - nothing is recorded unless you opt in; saved in this browser's cookies; turning this off deletes the history) + + +
    + +
    + + +
    +
    + + + +{{ end }} +{{ template "base.html" . }} diff --git a/templates/search.html b/templates/search.html new file mode 100644 index 0000000..c8711ab --- /dev/null +++ b/templates/search.html @@ -0,0 +1,280 @@ +{{ define "title" }}{{ with .QueryLabel }}{{ . }} - {{ end }}Search - Snowden Archive{{ end }} +{{ define "search-result-item" }} +{{ $r := .Result }}{{ $d := $r.Doc }} +{{ $qs := "" }}{{ if eq $d.Collection "foia" }}{{ $qs = .FOIAQS }}{{ end }} +
  • + {{ if .ShowThumbs }} + + {{ end }} +
    +

    + {{ $r.Title }} + [pdf] +

    + {{ template "type-badge" $d.Type }} + + {{ if $d.Publication }} + {{/* A year links to its publication-year page only when that page + lists the document: the year of its FIRST publication. */}} +
    + {{- $shown := false -}} + {{- range $pub := $d.Publication -}} + {{- if $pub.Publisher -}} + {{- if $shown }} · {{ end }}{{ $shown = true -}} + {{ template "pub-publisher" (dict "Pub" $pub "QS" $qs) }} + {{- if ge (len $pub.DatePublished) 4 }}{{ $y := slice $pub.DatePublished 0 4 }} {{ if eq $y $d.FirstPubYear }}{{ $y }}{{ else }}{{ $y }}{{ end }}{{ end -}} + {{- range $j, $r := $pub.Journalists }}{{ if $j }}, {{ else }} - {{ end }}{{ $r }}{{ end -}} + {{- end -}} + {{- end }} +
    + {{ end }} + +
    + {{ if $d.DocMeta.Creator }} + {{ $d.DocMeta.Creator }} + {{ end }} + {{ if and $d.DocMeta.Creator $d.DocMeta.DateCreated }} + · + {{ end }} + {{ if $d.DocMeta.DateCreated }} + {{ if ge (len $d.DocMeta.DateCreated) 4 }} + {{ $d.DocMeta.DateCreated }} + {{ else }} + {{ $d.DocMeta.DateCreated }} + {{ end }} + {{ end }} + + {{ if $d.DocMeta.Programs }} + · + {{ $progs := $d.DocMeta.Programs }} + {{ $shown := $progs }}{{ if gt (len $progs) 8 }}{{ $shown = slice $progs 0 8 }}{{ end }} + {{ facetLinks (dict "Vals" $shown "Cat" "program" "Sep" "dot" "Class" "meta-link-soft" "QS" $qs) }} + {{ if gt (len $progs) 8 }}+{{ add (len $progs) -8 }} more{{ end }} + {{ end }} +
    + +
    {{ $r.Desc }}
    + + {{ range $r.Snippets }} +
    {{ . }}
    + {{ end }} + {{ range $r.VersionSnippets }} +
    Copy: {{ .Label }} {{ .HTML }}
    + {{ end }} + {{ with $r.MatchesIn }} +
    Matches in: + {{- range $i, $v := . }}{{ if $i }},{{ end }} {{ $v.Label }}{{ end }} +
    + {{ end }} +
    +
  • +{{ end }} + +{{ define "content" }} +

    Search

    + +
    + Search syntax help +

    + Ways to write a value, in any row or after any operator: +

      +
    • cable tapping finds both words anywhere, in their other forms too (tapping also finds tapped);
    • +
    • "apping" finds that text anywhere, even inside a word, like grep -i (it finds for example mapping and tapping);
    • +
    • =tap or ="cable tapping" finds exactly those whole words, so =tap skips taps and strap, and on a metadata field it matches one whole value, so agency:=NSA finds documents whose agency is NSA but not NSA, GCHQ;
    • +
    • * wildcard finds part of a word: tap* finds words starting with tap, *land words ending in land and *keys* words containing keys. A leading * needs at least three letters after it.
    • +
    • "upstream collection"~10 finds the words within 10 words of each other (in any order, common stopwords ignored).
    • +
    • A leading - excludes within its row: xkeyscore -tempora finds xkeyscore without tempora, and the row keeps that meaning under OR and NOT.
    • +
    +

    +

    + The simple search box (on home page, on 404 page, and via URL ?q=) understands field operators: +

      +
    • program:XKEYSCORE country:Germany cable or journalist:="Poitras, Laura".
    • +
    • A leading - negates (-tag:=junk, -draft);
    • +
    • date:2013, date:2013..2015, and published:2014-06 filter by date.
    • +
    • Operators: text, title, content, description, agency, journalist, publisher, classification, program, dissemination, banner, genre, series, country, people, tag, type.
    • +
    • Bare words search all fields.
    • +
    +

    +
    + +
    + + + +
    + {{ range $i, $val := .Queries }} +
    +
    + {{ if eq $i 0 }} + Find + + {{ else }} + + {{ end }} +
    + +
    + in + {{ $fi := index $.Fields $i }} + +
    + {{ if ne $i 0 }} + + {{ else }} + + {{ end }} +
    + {{ end }} + +
    +
    + {{ if lt (len .Queries) .ClauseLimit }} + + {{ else }} + {{ .ClauseLimit }} rows is the most a search can have + {{ end }} +
    +
    + + {{ $hasDates := or .DateDocStart .DateDocEnd .DatePubStart .DatePubEnd }} + {{ $hasFilters := or $hasDates .CollFiltered }} +
    + Filters{{ if $hasFilters }} (active){{ end }} +
    + Collections + + + +
    +
    +
    + Doc date + + to + +
    +
    + Published + + to + +
    +
    +
    + +
    + + Reset +
    +
    + +{{ if .RecentSearches }} +
    + Recent searches ({{ len .RecentSearches }}) +
    + {{ range .RecentSearches }}{{ .Label }}{{ end }} +
    + + + +
    +
    +
    +{{ end }} + +
    + +{{ if .OverClauses }} +

    Search not run: it has {{ .OverClauses }} parts - rows, or the +field:, “quoted” and -excluded terms of a typed query - +and a search can have at most {{ .ClauseLimit }}. The form above keeps the first {{ .ClauseLimit }}; +the rest, including any exclusions at the end, were left out. Check the rows and search again.

    +{{ else if .OverWords }} +

    Search not run: it has {{ .OverWords }} words, counting the words of +“quoted” phrases and -excluded terms, and a search can have at most +{{ .WordLimit }}. Shorten it and search again.

    +{{ end }} + +{{ if .Searched }} +{{ if gt (len .SearchTypeCounts) 1 }}{{ template "type-filter" (dict "Action" "/search" "Params" .FormParams "AllTypes" .SearchTypeCounts "SelectedTypes" .SelTypeSet) }}{{ end }} +
    +

    {{ .Total }} result{{ if ne .Total 1 }}s{{ end }}

    + {{ template "sort-bar" (dict "SortURL" .SearchBaseURL "SortBy" .Sort "Order" .Order "ShowRelevance" true "DefaultSort" "relevance") }} + {{ if .Savable }} +
    + + + +
    + {{ end }} +
    + +{{ if gt (len .DateHist) 1 }} +
    + {{ range .DateHist }} + {{ if .URL }} + + {{ .Count }} + + {{ .Year }} + + {{ else }} + + {{ .Count }} + + {{ .Year }} + + {{ end }} + {{ end }} +
    +{{ end }} + +{{ if .Results }} +{{ if .Facets }} +
    + Refine results +
    + {{ range .Facets }} +
    + {{ .Label }} + {{ range .Values }}{{ .Name }}{{ .Count }}{{ end }} +
    + {{ end }} +
    +
    +{{ end }} +
      +{{ $showThumbs := .ShowThumbs }} +{{ range .Results }}{{ template "search-result-item" (dict "Result" . "ShowThumbs" $showThumbs "CardPagePreview" $.CardPagePreview "FOIAQS" $.FOIAQS) }}{{ end }} +
    +{{ if .SnippetsCapped }} +

    Text snippets are shown for the first {{ .SnippetLimit }} results on this page - +choose a page size in preferences, or narrow the search, to see +snippets for the rest.

    +{{ end }} +{{ template "pagination" (dict "Page" .Page "TotalPages" .TotalPages "BaseURL" .SearchBaseURL "Sort" .Sort "Order" .Order) }} +{{ else }} +

    No matches found.

    +{{ end }} + +{{ end }} + +{{ end }} + +{{ template "base.html" . }} diff --git a/templates/version.html b/templates/version.html new file mode 100644 index 0000000..464312f --- /dev/null +++ b/templates/version.html @@ -0,0 +1,59 @@ +{{ define "title" }}{{ .Doc.Title }} ({{ .Version.Slug }}) - Snowden Archive{{ end }} + +{{ define "content" }} +
    +

    ← {{ .Doc.Title }}

    +

    {{ .Doc.Title }} copy: {{ .Version.Slug }}

    +

    Another published copy of this document that differs from the one on the + document page - for example less redacted, with extra pages, or a different scan.

    +
    + +
    + Download this copy (PDF) + {{ with .Version.Images }}All pages ({{ len . }}){{ end }} +
    +{{ if .Version.ChangedPages }} +

    Pages that differ from the main document: {{ template "changed-page-links" .Version }}

    +{{ end }} + +
    + {{- range $i, $pub := .Publications }} +
    +
    + {{- with $pub.Publisher }}{{ . }}{{ else }}Unknown publisher{{ end -}} + {{- with $pub.DatePublished }} {{ . }}{{ end -}} +
    + {{- if $pub.Title }} +
    {{ $pub.Title }}{{ with $pub.Link }} article ↗{{ end }}
    + {{- else if $pub.Link }} + + {{- end }} + {{- with $pub.Note }} +
    {{ . }}
    + {{- end }} + {{- if or $pub.DocSource $pub.DocURL }} +
    + {{- with $pub.DocSource }}document page ↗{{ end -}} + {{- with $pub.DocURL }}original PDF ↗{{ end -}} +
    + {{- end }} +
    + {{- else }} +

    No publication record points at this copy.

    + {{- end }} +
    + +{{ if or .Images .Version.HTMLContent }} +
    + {{ template "doc-viewport" . }} +
    +{{ else }} +

    This copy has no transcript and no page images - only the PDF above.

    +{{ end }} + +{{ if and .Images (not .Prefs.TextOnly) }} +{{ template "lightbox-overlays" . }} +{{ end }} +{{ end }} + +{{ template "base.html" . }} diff --git a/tests/api_test.go b/tests/api_test.go new file mode 100644 index 0000000..8ed76a6 --- /dev/null +++ b/tests/api_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "testing" +) + +func TestAPIPageSize(t *testing.T) { + for raw, want := range map[string]int{ + "": apiPerPageDefault, + "25": 25, + "500": 500, + "1000": apiMaxPerPage, + "99999999": apiMaxPerPage, + "0": apiPerPageDefault, // not a page size; fall back + "-5": apiPerPageDefault, + "lots": apiPerPageDefault, + "all": 0, + } { + v := url.Values{} + if raw != "" { + v.Set("per_page", raw) + } + if got := apiPageSize(v); got != want { + t.Errorf("apiPageSize(per_page=%q) = %d, want %d", raw, got, want) + } + } +} + +func TestAPIRespPerPage(t *testing.T) { + for _, c := range []struct{ perPage, total, want int }{ + {50, 3106, 50}, + {0, 3106, 3106}, + {0, 0, 1}, + } { + if got := apiRespPerPage(c.perPage, c.total); got != c.want { + t.Errorf("apiRespPerPage(%d, %d) = %d, want %d", c.perPage, c.total, got, c.want) + } + } +} + +func TestAPIVersionsShape(t *testing.T) { + root := t.TempDir() + writeTestFile(t, filepath.Join(root, "doc-a", "metadata.json"), `{ + "title": "A", "type": "REPORTED", + "publication": [{"publisher": "Le Monde", "version": "entier"}], + "versions": {"entier": {"changed_pages": [1]}} + }`) + writeTestFile(t, filepath.Join(root, "doc-a", "a.pdf"), "%PDF-1.4") + writeTestFile(t, filepath.Join(root, "doc-a", "versions", "entier", "entier.pdf"), "%PDF-1.4") + withCorpus(t, root) + + type versionsObj struct { + Versions map[string]VersionInfo `json:"versions"` + } + var detail struct { + versionsObj + VersionCopies []apiVersion `json:"version_copies"` + } + decode := func(path string, v interface{}) { + t.Helper() + w := get(t, path) + if w.Code != 200 { + t.Fatalf("%s: %d", path, w.Code) + } + if err := json.Unmarshal(w.Body.Bytes(), v); err != nil { + t.Fatalf("%s: %v", path, err) + } + } + decode("/api/doc/doc-a", &detail) + if len(detail.Versions["entier"].ChangedPages) != 1 { + t.Errorf("/api/doc versions = %+v, want the metadata object", detail.Versions) + } + if len(detail.VersionCopies) != 1 || detail.VersionCopies[0].Slug != "entier" { + t.Errorf("/api/doc version_copies = %+v, want the entier copy", detail.VersionCopies) + } + var list struct{ Docs []versionsObj } + decode("/api/docs", &list) + var search struct{ Results []versionsObj } + decode("/api/search?q=A", &search) + for name, docs := range map[string][]versionsObj{"/api/docs": list.Docs, "/api/search": search.Results} { + if len(docs) != 1 || len(docs[0].Versions["entier"].ChangedPages) != 1 { + t.Errorf("%s versions = %+v, want the metadata object", name, docs) + } + } +} + +func TestAPIFacetsRevalidates(t *testing.T) { + w := get(t, "/api/facets") + tag := w.Header().Get("ETag") + if w.Code != 200 || tag == "" || w.Header().Get("Access-Control-Allow-Origin") != "*" { + t.Fatalf("/api/facets: %d, ETag %q, CORS %q", w.Code, tag, w.Header().Get("Access-Control-Allow-Origin")) + } + mux, _ := buildFixture(t) + r := httptest.NewRequest("GET", "/api/facets", nil) + r.Header.Set("If-None-Match", tag) + if w := record(mux, r); w.Code != http.StatusNotModified || w.Body.Len() != 0 { + t.Errorf("revalidation: %d with %d bytes, want 304 and none", w.Code, w.Body.Len()) + } +} + +// /api/docs takes coll= as /api/search and the pages do. +func TestAPIDocsCollections(t *testing.T) { + total := func(query string) int { + var body struct{ Total int } + if err := json.Unmarshal(get(t, "/api/docs?"+query).Body.Bytes(), &body); err != nil { + t.Fatalf("%s: %v", query, err) + } + return body.Total + } + all, news, foia := total(""), total("coll=news"), total("coll=foia") + if news == 0 || foia == 0 || news+foia >= all { + t.Fatalf("fixture totals: %d documents, %d news, %d foia", all, news, foia) + } + if got := total("coll=news&coll=foia"); got != news+foia { + t.Errorf("coll=news&coll=foia lists %d, want %d", got, news+foia) + } +} diff --git a/tests/assemblies_test.go b/tests/assemblies_test.go new file mode 100644 index 0000000..58f37b9 --- /dev/null +++ b/tests/assemblies_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "encoding/json" + "fmt" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" +) + +func writeAssemblyDoc(t *testing.T, root, id string, meta map[string]interface{}, pages []string, versions map[string][]string) { + t.Helper() + dir := filepath.Join(root, id) + b, err := json.Marshal(meta) + if err != nil { + t.Fatal(err) + } + writeFixtureFile(t, filepath.Join(dir, "metadata.json"), string(b)) + writeFixtureFile(t, filepath.Join(dir, id+".pdf"), "%PDF-1.4\n") + for _, p := range pages { + writeFixtureFile(t, filepath.Join(dir, "images", p), fixtureJPEG) + } + for slug, imgs := range versions { + v := filepath.Join(dir, "versions", slug) + writeFixtureFile(t, filepath.Join(v, slug+".pdf"), "%PDF-1.4\n") + for _, p := range imgs { + writeFixtureFile(t, filepath.Join(v, "images", p), fixtureJPEG) + } + } +} + +func assemblyCorpus(t *testing.T, manifest string) string { + t.Helper() + root := t.TempDir() + writeAssemblyDoc(t, root, "deck", map[string]interface{}{ + "title": "The Deck", + "publication": []map[string]interface{}{ + {"publisher": "Outlet A"}, + {"publisher": "Partner", "reporting_only": true}, + {"publisher": "Outlet B", "version": "fuller"}, + }, + }, []string{"page-1.jpg", "page-2.jpg", "page-3.jpg"}, map[string][]string{"fuller": {"page-1.jpg", "page-2.jpg"}}) + writeAssemblyDoc(t, root, "single", map[string]interface{}{ + "title": "One Slide", + "publication": []map[string]interface{}{{"publisher": "Outlet C"}}, + }, []string{"page-1.jpg"}, nil) + writeAssemblyDoc(t, root, "bystander", map[string]interface{}{"title": "Bystander"}, []string{"page-1.jpg"}, nil) + if manifest != "" { + writeFixtureFile(t, filepath.Join(root, "assemblies.json"), manifest) + } + return root +} + +const assemblyManifest = `[{ + "id": "set", "title": "A Set", "note": "Order is a guess.", + "slides": [ + {"doc": "deck", "pages": "1-2", "alt": [{"doc": "deck", "version": "fuller", "pages": "1-2"}]}, + {"doc": "deck", "version": "fuller", "pages": "2", "note": "Only here."}, + {"doc": "single", "pages": "1"} + ] +}]` + +func TestParsePageList(t *testing.T) { + for in, want := range map[string][]int{ + "8": {8}, "1-3": {1, 2, 3}, "1,3,5-6": {1, 3, 5, 6}, " 2 , 4 ": {2, 4}, "9,1": {9, 1}, + } { + got, err := parsePageList(in) + if err != nil || !reflect.DeepEqual(got, want) { + t.Errorf("parsePageList(%q) = %v, %v; want %v", in, got, err, want) + } + } + for _, in := range []string{"", "a", "0", "3-1", "1-", "1,,2"} { + if got, err := parsePageList(in); err == nil { + t.Errorf("parsePageList(%q) = %v, want an error", in, got) + } + } +} + +func TestAssemblyResolves(t *testing.T) { + withCorpus(t, assemblyCorpus(t, assemblyManifest)) + a := corpus().assemblyIndex["set"] + if a == nil { + t.Fatal("assembly not loaded") + } + if len(a.Slides) != 4 { + t.Fatalf("%d slides, want 4", len(a.Slides)) + } + for i, s := range a.Slides { + if s.Pos != i+1 { + t.Errorf("slide %d has Pos %d", i+1, s.Pos) + } + } + first := a.Slides[0] + if first.Image != "/files/deck/images/page-1.jpg" || first.Source != "Outlet A" { + t.Errorf("first slide %+v", first) + } + if len(first.Alts) != 1 || first.Alts[0].Image != "/files/deck/versions/fuller/images/page-1.jpg" || first.Alts[0].Source != "Outlet B" { + t.Errorf("first slide alts %+v", first.Alts) + } + if a.Slides[2].Note != "Only here." || a.Slides[2].Path != "/doc/deck/versions/fuller" { + t.Errorf("third slide %+v", a.Slides[2]) + } + if fmt.Sprint(a.DocList) != "[deck single]" { + t.Errorf("DocList %v", a.DocList) + } + if a.Copies != 3 { + t.Errorf("Copies %d, want 3", a.Copies) + } + if len(corpus().assembliesByDoc["single"]) != 1 || len(corpus().assembliesByDoc["bystander"]) != 0 { + t.Errorf("assembliesByDoc %v", corpus().assembliesByDoc) + } +} + +func TestAssemblyPages(t *testing.T) { + withCorpus(t, assemblyCorpus(t, assemblyManifest)) + h := withBaseHeaders(newMux()) + body := func(path string, code int) string { + t.Helper() + w := record(h, httptest_NewGet(path)) + if w.Code != code { + t.Fatalf("%s: %d, want %d", path, w.Code, code) + } + return w.Body.String() + } + list := body("/explore/assemblies", 200) + if !strings.Contains(list, `href="/explore/assemblies/set"`) || !strings.Contains(list, "4 slides from 3 copies") { + t.Errorf("/explore/assemblies:\n%s", list) + } + page := body("/explore/assemblies/set", 200) + for _, want := range []string{ + "A Set - Snowden Archive", "from 3 separately published copies.", "Order is a guess.", "not a document in its own right", + `id="lb-4"`, `href="#lb-a1-1"`, `id="lb-a1-1"`, "Other copy: Outlet B", "Back to slide 1", "(copy: fuller)", "Outlet C", "· Only here.", + } { + if !strings.Contains(page, want) { + t.Errorf("/explore/assemblies/set lacks %q", want) + } + } + if strings.Contains(page, `id="lb-5"`) || strings.Contains(page, "Partner") { + t.Errorf("/explore/assemblies/set has a fifth overlay or a reporting-only credit") + } + if !strings.Contains(body("/doc/single", 200), `href="/explore/assemblies/set"`) { + t.Errorf("/doc/single has no assembly link") + } + if strings.Contains(body("/doc/bystander", 200), "/explore/assemblies/") { + t.Errorf("/doc/bystander links an assembly") + } + // Assemblies live under /explore only: listed there, not on /browse. + if !strings.Contains(body("/explore", 200), `href="/explore/assemblies"`) { + t.Errorf("/explore has no assemblies link") + } + if strings.Contains(body("/browse", 200), "assemblies") { + t.Errorf("/browse still links assemblies") + } + body("/explore/assemblies/nope", 404) + if !slicesContains(corpus().sitemapPaths, "/explore/assemblies/set") { + t.Errorf("sitemap lacks the assembly page") + } +} + +func TestAssemblyProblemsWarn(t *testing.T) { + cases := map[string]struct { + manifest string + slides int + loaded bool + }{ + "missing doc": {`[{"id":"x","title":"X","slides":[{"doc":"ghost","pages":"1"},{"doc":"single","pages":"1"}]}]`, 1, true}, + "missing page": {`[{"id":"x","title":"X","slides":[{"doc":"deck","pages":"3-4"}]}]`, 1, true}, + "bad version": {`[{"id":"x","title":"X","slides":[{"doc":"deck","version":"no","pages":"1"},{"doc":"single","pages":"1"}]}]`, 1, true}, + "bad pages": {`[{"id":"x","title":"X","slides":[{"doc":"deck","pages":"two"},{"doc":"single","pages":"1"}]}]`, 1, true}, + "alt mismatch": {`[{"id":"x","title":"X","slides":[{"doc":"deck","pages":"1-2","alt":[{"doc":"single","pages":"1"}]}]}]`, 2, true}, + "nothing left": {`[{"id":"x","title":"X","slides":[{"doc":"ghost","pages":"1"}]}]`, 0, false}, + "unsafe id": {`[{"id":"a/b","title":"X","slides":[{"doc":"single","pages":"1"}]}]`, 0, false}, + "malformed": {`[{`, 0, false}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + withCorpus(t, assemblyCorpus(t, c.manifest)) + if loadWarnings == 0 { + t.Errorf("no warning") + } + a := corpus().assemblyIndex["x"] + if (a != nil) != c.loaded { + t.Fatalf("loaded = %v, want %v", a != nil, c.loaded) + } + if a != nil && len(a.Slides) != c.slides { + t.Errorf("%d slides, want %d", len(a.Slides), c.slides) + } + }) + } +} + +func TestNoAssembliesFile(t *testing.T) { + withCorpus(t, assemblyCorpus(t, "")) + if len(corpus().assemblies) != 0 { + t.Fatalf("assemblies %v", corpus().assemblies) + } + h := withBaseHeaders(newMux()) + if w := record(h, httptest_NewGet("/explore/assemblies")); w.Code != 200 { + t.Errorf("/explore/assemblies: %d", w.Code) + } + if w := record(h, httptest_NewGet("/explore")); strings.Contains(w.Body.String(), `href="/explore/assemblies"`) { + t.Errorf("/explore links assemblies with none loaded") + } + if slicesContains(corpus().sitemapPaths, "/explore/assemblies") { + t.Errorf("sitemap lists assemblies with none loaded") + } +} + +func slicesContains(s []string, v string) bool { return slices.Contains(s, v) } diff --git a/tests/bench_test.go b/tests/bench_test.go new file mode 100644 index 0000000..6e47db5 --- /dev/null +++ b/tests/bench_test.go @@ -0,0 +1,312 @@ +package main + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "runtime" + "sort" + "strconv" + "sync" + "testing" + "time" +) + +// Page benchmarks against a REAL corpus - the fixture is four documents, so +// it cannot show what 3,000 do. Point $SNOWCHIVE_BENCH_ROOT at an archive +// root (the directory holding data/ and project/) and run: +// tests/run.sh -run XXX -bench . -benchtime 40x +// +// What they are for: every page but /search is built from data indexed at +// load, so a request should cost what its OUTPUT costs (template execution) +// and nothing more. A page whose time or allocations grow with the CORPUS +// rather than with the page has picked up a scan. +var ( + benchOnce sync.Once + benchMux http.Handler +) + +func benchCorpus(tb testing.TB) http.Handler { + root := os.Getenv("SNOWCHIVE_BENCH_ROOT") + if root == "" { + tb.Skip("set SNOWCHIVE_BENCH_ROOT to an archive root to run the page benchmarks") + } + benchOnce.Do(func() { + setRoot(root) + devMode = false + apiEnabled = true + initAssets() + if err := parseTemplates(); err != nil { + tb.Fatal(err) + } + loadData() + benchMux = withBaseHeaders(newMux()) + }) + return benchMux +} + +func benchPage(b *testing.B, path string) { + h := benchCorpus(b) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + r := httptest.NewRequest("GET", path, nil) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != 200 { + b.Fatalf("%s: %d", path, w.Code) + } + } +} + +func BenchmarkPageHome(b *testing.B) { benchPage(b, "/") } +func BenchmarkPageBrowse(b *testing.B) { benchPage(b, "/browse") } +func BenchmarkPageArchiveTab(b *testing.B) { benchPage(b, "/browse?collection=sidtoday-archive") } +func BenchmarkPageCategoryList(b *testing.B) { benchPage(b, "/browse/program") } +func BenchmarkPageCategoryValue(b *testing.B) { benchPage(b, "/browse/agency/NSA") } +func BenchmarkPageDoc(b *testing.B) { + h := benchCorpus(b) + if len(corpus().archive.Docs) == 0 { + b.Skip("empty corpus") + } + benchPageWith(b, h, "/doc/"+corpus().archive.Docs[0].ID) +} +func BenchmarkPageExploreIDs(b *testing.B) { benchPage(b, "/explore/ids") } +func BenchmarkPageGlossary(b *testing.B) { benchPage(b, "/glossary") } +func BenchmarkPageSitemap(b *testing.B) { benchPage(b, "/sitemap.xml") } +func BenchmarkPageFacetsAPI(b *testing.B) { benchPage(b, "/api/facets") } +func BenchmarkPageSearch(b *testing.B) { benchPage(b, "/search?q=nsa&do_search=1") } + +func benchPageWith(b *testing.B, h http.Handler, path string) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + r := httptest.NewRequest("GET", path, nil) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + if w.Code != 200 { + b.Fatalf("%s: %d", path, w.Code) + } + } +} + +func BenchmarkPageCategoryTag(b *testing.B) { benchPage(b, "/browse/tag") } +func BenchmarkPageCategoryPeople(b *testing.B) { benchPage(b, "/browse/people") } +func BenchmarkPageCategoryTagSorted(b *testing.B) { benchPage(b, "/browse/tag?sort=count") } +func BenchmarkPageExploreMetric(b *testing.B) { benchPage(b, "/explore/pages") } +func BenchmarkPageCodewords(b *testing.B) { benchPage(b, "/explore/terms") } +func BenchmarkPageOutlets(b *testing.B) { benchPage(b, "/explore/outlets") } +func BenchmarkPageClassPatterns(b *testing.B) { benchPage(b, "/explore/classification") } +func BenchmarkPageArticles(b *testing.B) { benchPage(b, "/articles") } +func BenchmarkPageChangelog(b *testing.B) { benchPage(b, "/changelog") } +func BenchmarkPageAbout(b *testing.B) { benchPage(b, "/about") } +func BenchmarkPageAPIDocs(b *testing.B) { benchPage(b, "/api/docs") } +func BenchmarkPageExploreIndex(b *testing.B) { benchPage(b, "/explore") } +func BenchmarkPageContactSheet(b *testing.B) { + h := benchCorpus(b) + id := "" + for i := range corpus().archive.Docs { + if len(corpus().archive.Docs[i].Images) > 40 { + id = corpus().archive.Docs[i].ID + break + } + } + if id == "" { + b.Skip("no document with many pages") + } + benchPageWith(b, h, "/doc/"+id+"/pages") +} + +// Pathological documents: the biggest thing in the corpus is what breaks +// first. What IS the biggest thing? +func TestProbeExtremes(t *testing.T) { + if os.Getenv("SNOWCHIVE_BENCH_ROOT") == "" { + t.Skip("needs a real corpus") + } + benchCorpus(t) + var maxImgs, maxText, maxPubs, maxVersions *Document + for i := range corpus().archive.Docs { + d := &corpus().archive.Docs[i] + if maxImgs == nil || len(d.Images) > len(maxImgs.Images) { + maxImgs = d + } + if maxText == nil || len(d.FullText) > len(maxText.FullText) { + maxText = d + } + if maxPubs == nil || len(d.Publication) > len(maxPubs.Publication) { + maxPubs = d + } + if maxVersions == nil || len(d.Versions) > len(maxVersions.Versions) { + maxVersions = d + } + } + h := benchCorpus(t) + timeIt := func(p string) time.Duration { + start := time.Now() + r := httptest.NewRequest("GET", p, nil) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + return time.Since(start) + } + fmt.Printf("most pages: %d %s doc %s, contact sheet %s\n", len(maxImgs.Images), trunc30(maxImgs.ID), + timeIt("/doc/"+maxImgs.ID).Round(time.Millisecond), timeIt("/doc/"+maxImgs.ID+"/pages").Round(time.Millisecond)) + fmt.Printf("most text: %d KB %s doc page %s\n", len(maxText.FullText)/1024, trunc30(maxText.ID), + timeIt("/doc/"+maxText.ID).Round(time.Millisecond)) + fmt.Printf("most pubs: %d %s\n", len(maxPubs.Publication), trunc30(maxPubs.ID)) + fmt.Printf("most copies: %d %s\n", len(maxVersions.Versions), trunc30(maxVersions.ID)) +} + +// Distribution, not the median: which document pages are slow, and why? +func TestProbeDocPageDistribution(t *testing.T) { + if os.Getenv("SNOWCHIVE_BENCH_ROOT") == "" { + t.Skip("needs a real corpus") + } + h := benchCorpus(t) + totalImgs := 0 + for i := range corpus().archive.Docs { + totalImgs += len(corpus().archive.Docs[i].Images) + } + var ds []time.Duration + for i := 0; i < len(corpus().archive.Docs); i += 7 { // a sample, every 7th + d := &corpus().archive.Docs[i] + start := time.Now() + r := httptest.NewRequest("GET", "/doc/"+d.ID, nil) + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + ds = append(ds, time.Since(start)) + } + sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] }) + p := func(q float64) time.Duration { return ds[int(float64(len(ds)-1)*q)] } + fmt.Printf("%d page images in the corpus (%d docs sampled)\n", totalImgs, len(ds)) + fmt.Printf("doc page: p50 %s p90 %s p99 %s max %s\n", + p(0.5).Round(time.Microsecond), p(0.9).Round(time.Microsecond), + p(0.99).Round(time.Millisecond), ds[len(ds)-1].Round(time.Millisecond)) +} + +func TestProbeWorstDocSplit(t *testing.T) { + if os.Getenv("SNOWCHIVE_BENCH_ROOT") == "" { + t.Skip("needs a real corpus") + } + h := benchCorpus(t) + var worst *Document + for i := range corpus().archive.Docs { + if worst == nil || len(corpus().archive.Docs[i].Images) > len(worst.Images) { + worst = &corpus().archive.Docs[i] + } + } + run := func(path string, cookies ...*http.Cookie) (time.Duration, int) { + var best time.Duration + var size int + for i := 0; i < 5; i++ { + r := httptest.NewRequest("GET", path, nil) + for _, c := range cookies { + r.AddCookie(c) + } + w := httptest.NewRecorder() + start := time.Now() + h.ServeHTTP(w, r) + d := time.Since(start) + if i == 0 || d < best { + best = d + } + size = w.Body.Len() + } + return best, size + } + full, fullSize := run("/doc/" + worst.ID) + textOnly, textSize := run("/doc/"+worst.ID, &http.Cookie{Name: "pref_text_only", Value: "1"}) + sheet, sheetSize := run("/doc/" + worst.ID + "/pages") + fmt.Printf("worst doc: %d images, %d KB transcript\n", len(worst.Images), len(worst.FullText)/1024) + fmt.Printf(" full page %6s %d KB\n", full.Round(time.Microsecond), fullSize/1024) + fmt.Printf(" text-only %6s %d KB (no image grid, no lightbox)\n", textOnly.Round(time.Microsecond), textSize/1024) + fmt.Printf(" contact sheet %6s %d KB\n", sheet.Round(time.Microsecond), sheetSize/1024) +} + +func trunc30(s string) string { + if len(s) > 30 { + return s[:30] + } + return s +} + +// TestSoak hammers the server for a while and watches its memory. The claim +// it checks: nothing accumulates between requests - no cache, no growing +// map, no goroutine left behind. Run it against a real corpus: +// +// SNOWCHIVE_BENCH_ROOT=/path/to/archive tests/run.sh -run TestSoak -timeout 30m +// +// SNOWCHIVE_SOAK_REQUESTS sets the count (default 20,000). +func TestSoak(t *testing.T) { + h := benchCorpus(t) + n := 20000 + if v := os.Getenv("SNOWCHIVE_SOAK_REQUESTS"); v != "" { + if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 { + n = parsed + } + } + paths := []string{"/", "/browse", "/browse?collection=foia", "/browse/program", + "/browse/tag?sort=count", "/browse/agency/NSA", "/search?q=nsa&do_search=1", + "/search?q=%22upstream+collection%22&do_search=1", "/explore/ids", "/glossary", + "/articles", "/api/facets", "/api/search?q=prism", "/sitemap.xml"} + cookies := []*http.Cookie{ + {Name: "pref_theme", Value: "dark"}, + {Name: "pref_per_page", Value: "100"}, + {Name: "pref_text_only", Value: "1"}, + } + + read := func() (heap, sys uint64, goroutines int) { + runtime.GC() + var m runtime.MemStats + runtime.ReadMemStats(&m) + return m.HeapAlloc, m.Sys, runtime.NumGoroutine() + } + // Warm up first: the pre-rendered bodies and the first template + // executions allocate once and stay. + for i := 0; i < 200; i++ { + r := httptest.NewRequest("GET", paths[i%len(paths)], nil) + h.ServeHTTP(httptest.NewRecorder(), r) + } + heap0, sys0, gor0 := read() + + start := time.Now() + var wg sync.WaitGroup + const workers = 8 + for w := 0; w < workers; w++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + for i := id; i < n; i += workers { + r := httptest.NewRequest("GET", paths[i%len(paths)], nil) + if i%4 == 0 { + r.AddCookie(cookies[i%len(cookies)]) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, r) + if rec.Code != 200 { + t.Errorf("%s: %d", paths[i%len(paths)], rec.Code) + return + } + } + }(w) + } + wg.Wait() + elapsed := time.Since(start) + heap1, sys1, gor1 := read() + + mb := func(b uint64) float64 { return float64(b) / (1 << 20) } + fmt.Printf("%d requests in %s (%.0f req/s, %d workers)\n", + n, elapsed.Round(time.Millisecond), float64(n)/elapsed.Seconds(), workers) + fmt.Printf("heap %.1f MB -> %.1f MB (%+.1f)\n", mb(heap0), mb(heap1), mb(heap1)-mb(heap0)) + fmt.Printf("sys %.1f MB -> %.1f MB (%+.1f)\n", mb(sys0), mb(sys1), mb(sys1)-mb(sys0)) + fmt.Printf("goroutines %d -> %d\n", gor0, gor1) + + if gor1 > gor0+2 { + t.Errorf("goroutines grew from %d to %d", gor0, gor1) + } + // The heap after a GC should be the corpus and the pre-rendered pages, + // not a function of how many requests have been served. + if grown := mb(heap1) - mb(heap0); grown > 32 { + t.Errorf("heap grew by %.1f MB over %d requests - something is accumulating", grown, n) + } +} diff --git a/tests/bugs_test.go b/tests/bugs_test.go new file mode 100644 index 0000000..607025f --- /dev/null +++ b/tests/bugs_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/url" + "strings" + "testing" +) + +// A hyphenated term must match its words in sequence, not just the first one. +func TestHyphenatedTermMatchesInSequence(t *testing.T) { + hit := Document{ID: "hit", FullText: "A cyber attack on the power grid."} + miss := Document{ID: "miss", FullText: "Cyber policy, and an attack helicopter."} + for _, d := range []*Document{&hit, &miss} { + setBodySearchText(d) + d.StemmedText, d.SearchText = d.StemmedFull, d.FoldedFull + } + + for _, field := range []string{"FT", "TE"} { + if !compileQuery([]string{"cyber-attack"}, []string{field}, []string{"and"}).matches(&hit) { + t.Errorf("%s: cyber-attack should match text containing the words in sequence", field) + } + if compileQuery([]string{"cyber-attack"}, []string{field}, []string{"and"}).matches(&miss) { + t.Errorf("%s: cyber-attack matched a document where the words are apart", field) + } + } + // The words on their own still match both, as before. + if !compileQuery([]string{"cyber"}, []string{"FT"}, []string{"and"}).matches(&miss) { + t.Error("a plain word should still match") + } +} + +// A value page's publication-year bars count every year a document was +// published in, because each bar links to a search that matches any of them. +func TestPubYearsOf(t *testing.T) { + var d Document + if err := json.Unmarshal([]byte(`{"publication": [ + {"publisher": "The Guardian", "date_published": "2013-06-06"}, + {"publisher": "Le Monde", "date_published": "2014-10-21"}, + {"publisher": "Der Spiegel", "date_published": "2013-12-30"}, + {"publisher": "No date"}, + {"publisher": "Bad date", "date_published": "201"} + ]}`), &d); err != nil { + t.Fatal(err) + } + if got := fmt.Sprint(pubYearsOf(&d)); got != "[2013 2014]" { + t.Errorf("pubYearsOf = %s, want [2013 2014]", got) + } + if got := fmt.Sprint(pubYearsOf(&Document{})); got != "[]" { + t.Errorf("pubYearsOf(no publications) = %s, want []", got) + } +} + +// A saved search's label must name the type filter, or two saved searches +// differing only by Reported/SIDtoday are indistinguishable. +func TestSearchLabelIncludesTypeFilter(t *testing.T) { + entry := func(v url.Values) string { return url.QueryEscape(v.Encode()) } + base := url.Values{"fqv": {"xkeyscore"}, "fqf": {"TE"}, "fqc": {"and"}} + typed := url.Values{"fqv": {"xkeyscore"}, "fqf": {"TE"}, "fqc": {"and"}, "type": {"REPORTED"}} + both := url.Values{"fqv": {"xkeyscore"}, "fqf": {"TE"}, "fqc": {"and"}, + "type": {"SIDTODAY"}, "coll": {"news"}} + + got := corpus().parseSearchEntries([]string{entry(typed), entry(base), entry(both)}) + if len(got) != 3 { + t.Fatalf("parsed %d entries, want 3", len(got)) + } + if !strings.Contains(got[0].Label, "REPORTED") { + t.Errorf("label %q should name the type filter", got[0].Label) + } + if got[0].Label == got[1].Label { + t.Errorf("searches differing only by type share the label %q", got[0].Label) + } + // Collection and type share one bracket rather than two. + if !strings.Contains(got[2].Label, "(news, SIDTODAY)") { + t.Errorf("label %q should list collection and type together", got[2].Label) + } +} diff --git a/tests/consistency_test.go b/tests/consistency_test.go new file mode 100644 index 0000000..e001fc3 --- /dev/null +++ b/tests/consistency_test.go @@ -0,0 +1,538 @@ +package main + +import ( + "encoding/json" + "html" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "reflect" + "regexp" + "slices" + "sort" + "strings" + "testing" +) + +// A corpus built to exercise the places where one fact used to be computed +// twice: search rows and their negations, the page and the API, counts and +// the pages they link to. Each document's words are chosen so the expected +// result sets can be written down by hand. +// +// n1 news alpha gamma ZEBRA 2010 The Intercept 2014 + the intercept 2016 +// n2 news alpha ZEBRA 2011 The Intercept 2014 + Der Spiegel 2014 +// n3 news beta gamma 2012 Der Spiegel 2015 +// n4 news beta 2012 Der Spiegel 2015 +// f1 foia alpha ZEBRA 2010 (court filing: "(c)" list labels) +// s1 sidtoday ZEBRA 2011 +// a1 news "aka_id": null +// d1 news date_created "ODNI", date_published "15 May 2015" +func writeConsistencyCorpus(t *testing.T) string { + t.Helper() + root := t.TempDir() + doc := func(dir, id string, meta map[string]interface{}, text string) { + b, err := json.Marshal(meta) + if err != nil { + t.Fatal(err) + } + d := filepath.Join(root, dir, id) + writeFixtureFile(t, filepath.Join(d, "metadata.json"), string(b)) + writeFixtureFile(t, filepath.Join(d, id+".pdf"), "%PDF-1.4\n") + if text != "" { + writeFixtureFile(t, filepath.Join(d, id+".md"), text) + } + } + pub := func(publisher, date, link string, reporters ...string) map[string]interface{} { + return map[string]interface{}{"publisher": publisher, "date_published": date, + "link": link, "reporters": reporters} + } + meta := func(title, typ, created string, programs []string, pubs ...map[string]interface{}) map[string]interface{} { + return map[string]interface{}{"title": title, "type": typ, + "document": map[string]interface{}{"creator": "NSA", "date_created": created, "programs": programs}, + "publication": pubs} + } + zebra := []string{"ZEBRA"} + doc("", "n1", meta("Report one", "REPORTED", "2010-03-01", zebra, + pub("The Intercept", "2014-01-01", "https://example.org/a", "Glenn Greenwald"), + pub("the intercept", "2016-01-01", "https://example.org/b", "glenn greenwald")), + "alpha gamma\n\n(S//SI) a secret line\n(S/SI) the same marking, spelled by OCR\n(U) plain\n") + doc("", "n2", meta("Report two", "REPORTED", "2011-03-01", zebra, + pub("The Intercept", "2014-02-01", "https://example.org/c"), + pub("Der Spiegel", "2014-02-02", "https://example.org/f")), + "alpha\n\n(TS//SI//NF) top line\n") + doc("", "n3", meta("Report three", "REPORTED", "2012-03-01", nil, + pub("Der Spiegel", "2015-01-01", "https://example.org/d")), "beta gamma\n") + doc("", "n4", meta("Report four", "REPORTED", "2012-04-01", nil, + pub("Der Spiegel", "2015-01-01", "https://example.org/d")), "beta\n") + doc("FOIA", "f1", meta("Court filing", "FOIA", "2010-05-01", zebra), + "alpha\n\n(c) The Court shall issue an order.\n(s) another subsection\n") + doc("SIDtoday-archive", "s1", meta("Issue", "SIDTODAY", "2011-05-01", zebra), "(S) issue line\n") + a1 := meta("Null aka", "REPORTED", "", nil) + a1["aka_id"] = nil + doc("", "a1", a1, "") + // A date of no shape the archive reads: served, undated. + doc("", "d1", meta("Odd date", "REPORTED", "ODNI", nil, pub("Der Spiegel", "15 May 2015", "https://example.org/e")), "") + writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[]`) + return root +} + +// matchingIDs runs the search engine and returns the matching ids, sorted. +func matchingIDs(queries, fields, conjs []string) []string { + q, f, c := normalizeClauses(queries, fields, conjs) + var ids []string + scored, _ := corpus().executeSearch(searchParams{Queries: q, Fields: f, Conjs: c}) + for _, s := range scored { + ids = append(ids, s.Doc.ID) + } + sort.Strings(ids) + return ids +} + +// A row is one unit: its -terms are exclusions within it, whatever joins +// it to the rows before. And an empty row starts nothing - a blank first +// row with "OR x" under it is a search for x. +func TestSearchRowsAndNegations(t *testing.T) { + withCorpus(t, writeConsistencyCorpus(t)) + te2 := []string{"TE", "TE"} + for _, c := range []struct { + name string + queries, conjs []string + want []string + }{ + {"inline negation", []string{"alpha -gamma"}, []string{"and"}, []string{"f1", "n2"}}, + // alpha AND NOT (beta AND NOT gamma): no alpha document is lost + // for mentioning gamma. + {"negation inside NOT", []string{"alpha", "beta -gamma"}, []string{"and", "not"}, []string{"f1", "n1", "n2"}}, + // alpha OR (beta AND NOT gamma): the exclusion stays inside its row. + {"negation inside OR", []string{"alpha", "beta -gamma"}, []string{"and", "or"}, []string{"f1", "n1", "n2", "n4"}}, + {"blank first row, OR", []string{"", "alpha"}, []string{"and", "or"}, []string{"f1", "n1", "n2"}}, + {"spaces first row, OR", []string{" ", "alpha"}, []string{"and", "or"}, []string{"f1", "n1", "n2"}}, + {"blank first row, OR nothing", []string{"", "zzzzunmatched"}, []string{"and", "or"}, nil}, + {"blank first row, NOT", []string{"", "gamma"}, []string{"and", "not"}, []string{"a1", "d1", "f1", "n2", "n4", "s1"}}, + {"only exclusions", []string{"-gamma -alpha"}, []string{"and"}, []string{"a1", "d1", "n4", "s1"}}, + } { + if got := matchingIDs(c.queries, te2[:len(c.queries)], c.conjs); !reflect.DeepEqual(got, c.want) { + t.Errorf("%s: %q %v matched %v, want %v", c.name, c.queries, c.conjs, got, c.want) + } + } + // The scorer and the highlighter read the same rows: an excluded term + // is neither rewarded nor marked. + if got := positiveTerms([]string{"alpha -gamma", "", "beta"}, []string{"and", "and", "not"}); !reflect.DeepEqual(got, []string{"alpha"}) { + t.Errorf("positiveTerms = %q, want [alpha]", got) + } + if term, _ := snippetAnchor([]string{"-gamma", "alpha -beta"}, te2, []string{"and", "and"}); term != "alpha" { + t.Errorf("snippetAnchor = %q, want alpha", term) + } +} + +// "+ Add row" and "remove" re-run the search: on the same rows, meaning the +// same thing, as the Search button would. +func TestRowEditsSearchTheSameRows(t *testing.T) { + withCorpus(t, writeConsistencyCorpus(t)) + rows := "do_search=1&fqv=alpha+-gamma&fqf=TE&fqc=and" + want := resultTotal(t, "/search?action=search&"+rows, foiaOn) + if want != 2 { + t.Fatalf("search for alpha -gamma found %d, want 2", want) + } + for _, action := range []string{"add", "remove-1"} { + path := "/search?action=" + action + "&" + rows + "&fqv=beta&fqf=TE&fqc=or" + if action == "add" { + path = "/search?action=add&" + rows + } + if got := resultTotal(t, path, foiaOn); got != want { + t.Errorf("action=%s: %d results, the same rows searched find %d", action, got, want) + } + } +} + +// The type filter resubmits the search the page shows, not the request: a +// removed row stays removed, and the chosen sort is kept. +func TestTypeFilterFormCarriesTheEffectiveSearch(t *testing.T) { + withCorpus(t, writeConsistencyCorpus(t)) + body := get(t, "/search?do_search=1&action=remove-1&sort=title"+ + "&fqv=alpha&fqf=TE&fqc=and&fqv=removedrow&fqf=TE&fqc=or", foiaOn).Body.String() + if !strings.Contains(body, `class="type-filter"`) { + t.Fatal("no type filter on a result set holding two types") + } + if strings.Contains(body, "removedrow") { + t.Error("the removed row is still in the page - the type filter would bring it back") + } + if !strings.Contains(body, `name="sort" value="title"`) { + t.Error("the type filter drops the chosen sort") + } +} + +// /api/search runs the search page's pipeline, type filter included. The +// API has no preference: it matches the page of a reader who includes FOIA +// releases. +func TestAPISearchMatchesThePage(t *testing.T) { + withCorpus(t, writeConsistencyCorpus(t)) + for qs, want := range map[string]int{ + "q=alpha": 3, + "q=alpha&type=REPORTED": 2, + "q=alpha&type=FOIA&type=REPORTED": 3, + "q=alpha&type=reported": 2, + "q=zebra&coll=news": 2, + "q=zebra&coll=foia&type=FOIA": 1, + "do_search=1&fqv=alpha&fqf=TE&fqc=and&fqv=beta+-gamma&fqf=TE&fqc=or": 4, + } { + page := resultTotal(t, "/search?"+qs, foiaOn) + var api struct{ Total int } + if err := json.Unmarshal(get(t, "/api/search?"+qs).Body.Bytes(), &api); err != nil { + t.Fatalf("%s: %v", qs, err) + } + if page != want || api.Total != want { + t.Errorf("%s: the page finds %d, the API %d, want %d", qs, page, api.Total, want) + } + } +} + +// Every count in this corpus is the number its page or search returns. +func TestConsistencyCorpusCounts(t *testing.T) { + withCorpus(t, writeConsistencyCorpus(t)) + + // Facets, with journalist and publisher spellings that differ by case + // across one document's publications. + for ci := range categories { + c := &categories[ci] + for val, want := range c.Facet(corpus()) { + if got := len(corpus().categoryMatches(c.Slug, val)); got != want { + t.Errorf("%s=%q: facet says %d, the page lists %d", c.Slug, val, want, got) + } + } + } + if js := corpus().archive.Facets["journalist"]; len(js) != 1 || js["Glenn Greenwald"] != 1 { + t.Errorf("journalists = %v, want Glenn Greenwald once", js) + } + if n := corpus().archive.Facets["publisher"]["The Intercept"]; n != 2 { + t.Errorf("The Intercept: %d documents, want 2 (%v)", n, corpus().archive.Facets["publisher"]) + } + + // The value-page timelines and refine links, scoped to a collection: + // f1 (FOIA) shares 2010 with n1 and s1 (SIDtoday) shares 2011 with n2, + // so an unscoped bar link finds more than its bar counts. + for _, p := range []string{"/browse/program/ZEBRA?coll=news", "/browse/program/ZEBRA"} { + if n := followLinks(t, p, reBarLink, "timeline bar"); n < 2 { + t.Errorf("%s: %d timeline bars checked, want the doc and pub timelines", p, n) + } + } + + // /explore/outlets: rows, cells and the per-year footer. + for coll, table := range corpus().outletTables { + for _, row := range table.Rows { + if got := resultTotal(t, row.URL); got != row.Total { + t.Errorf("outlets[%q] %s: total %d, its page lists %d", coll, row.Name, row.Total, got) + } + for _, cell := range row.Cells { + if cell.URL != "" && resultTotal(t, cell.URL) != cell.Docs { + t.Errorf("outlets[%q] %s: a cell says %d, its search finds %d", coll, row.Name, cell.Docs, resultTotal(t, cell.URL)) + } + } + } + // The footer adds up its column: a document several outlets + // published counts once for each of them. + for i, y := range table.Years { + sum := 0 + for _, row := range table.Rows { + sum += row.Cells[i].Docs + } + if table.Totals[i] != sum { + t.Errorf("outlets[%q] %s: footer %d, its column adds up to %d", coll, y, table.Totals[i], sum) + } + } + } + cell := func(outlet, year string) (docs, footer int) { + table := corpus().outletTables[""] + for i, y := range table.Years { + if y != year { + continue + } + footer = table.Totals[i] + for _, row := range table.Rows { + if row.Name == outlet { + docs = row.Cells[i].Docs + } + } + } + return docs, footer + } + // n2: The Intercept and Der Spiegel both published it in 2014, so it + // counts for both outlets and twice in the year's footer (n1 is the + // third: The Intercept, 2014). + if spiegel, footer := cell("Der Spiegel", "2014"); spiegel != 1 || footer != 3 { + t.Errorf("2014: Der Spiegel cell %d, footer %d - want 1 and 3 (n2 once per outlet)", spiegel, footer) + } + if intercept, _ := cell("The Intercept", "2014"); intercept != 2 { + t.Errorf("2014: The Intercept cell %d, want 2", intercept) + } + intercept := 0 + for _, row := range corpus().outletTables[""].Rows { + if row.Name == "The Intercept" { + intercept = row.Total + } + } + if intercept != 2 { + t.Errorf("The Intercept's outlet total = %d, want 2 documents (n1 counts once for its two years)", intercept) + } + + // /explore/portions: lower-case list labels are not markings, and every + // row's document count is what its search finds. + for _, l := range corpus().portionStats.Levels { + if l.Name == "C" { + t.Errorf("a CONFIDENTIAL row from the court filing's (c) labels: %+v", l) + } + } + rows := append(append(append([]portionRow(nil), corpus().portionStats.Levels...), corpus().portionStats.Markings...), corpus().portionStats.Controls...) + if len(rows) < 6 { + t.Fatalf("only %d portion rows", len(rows)) + } + for _, r := range rows { + if got := resultTotal(t, string(r.SearchURL)); got != r.Docs { + t.Errorf("portion row %s: %d documents, its search finds %d (%s)", r.Name, r.Docs, got, r.SearchURL) + } + } + if got := corpus().docIndex["n1"].PortionMarkings; !reflect.DeepEqual(got, []string{"S//SI", "U"}) { + t.Errorf("n1 markings = %v, want [S//SI U]", got) + } + + // A malformed date is no date: no "ODNI" year that no year page opens. + if d := corpus().docIndex["d1"]; d == nil || d.DocMeta.DateCreated != "" || d.Publication[0].DatePublished != "" { + t.Errorf("d1's malformed dates were kept: %+v", d) + } + if _, ok := corpus().archive.Facets["docdate"]["ODNI"]; ok { + t.Error(`"ODNI" counted as a document year`) + } + + // aka_id: null is no akas, not a reason to drop the document. + if corpus().docIndex["a1"] == nil { + t.Error(`a document with "aka_id": null was not loaded`) + } +} + +// A corpus for the counts that turn on WHICH publication entry carries a +// date or a byline, and on the reader's collection scope. Every document +// says "alpha". +// +// p1 news 2010-03 Outlet X 2013 (Able), Outlet Y 2014 (Baker) +// p2 news 2010-06 Outlet X 2014 (Able) people: Clerk +// p3 news 2011-01 Outlet X 2014 (Baker) +// p4 news 2012-03 Outlet Y 2014 +// p5 news 2012-09 Outlet Y 2014 +// p6 news 2013-02 Outlet Y 2014, Partner Z 2014 (its own story; reporting only) +// f1 foia 2009-07 Outlet Y 2014 (a story of its own) people: Docket +func writeEntryCorpus(t *testing.T) string { + t.Helper() + root := t.TempDir() + pub := func(publisher, date string, reporters ...string) map[string]interface{} { + return map[string]interface{}{"publisher": publisher, "date_published": date, "reporters": reporters} + } + doc := func(dir, id, typ, created string, people []string, pubs ...map[string]interface{}) { + b, err := json.Marshal(map[string]interface{}{"title": "Document " + id, "type": typ, + "document": map[string]interface{}{"creator": "NSA", "date_created": created, "people": people}, + "publication": pubs}) + if err != nil { + t.Fatal(err) + } + d := filepath.Join(root, dir, id) + writeFixtureFile(t, filepath.Join(d, "metadata.json"), string(b)) + writeFixtureFile(t, filepath.Join(d, id+".pdf"), "%PDF-1.4\n") + writeFixtureFile(t, filepath.Join(d, id+".md"), "alpha\n") + } + doc("", "p1", "REPORTED", "2010-03-01", nil, + pub("Outlet X", "2013-05-01", "Able, Ann"), pub("Outlet Y", "2014-02-01", "Baker, Bob")) + doc("", "p2", "REPORTED", "2010-06-01", []string{"Clerk, Carol"}, pub("Outlet X", "2014-03-01", "Able, Ann")) + doc("", "p3", "REPORTED", "2011-01-01", nil, pub("Outlet X", "2014-04-01", "Baker, Bob")) + doc("", "p4", "REPORTED", "2012-03-01", nil, pub("Outlet Y", "2014-05-01")) + doc("", "p5", "REPORTED", "2012-09-01", nil, pub("Outlet Y", "2014-06-01")) + story := func(p map[string]interface{}, title, link string) map[string]interface{} { + p["title"], p["link"] = title, link + return p + } + partner := story(pub("Partner Z", "2014-07-02"), "Partner story", "https://example.org/partner") + partner["reporting_only"] = true + doc("", "p6", "REPORTED", "2013-02-01", nil, pub("Outlet Y", "2014-07-01"), partner) + doc("FOIA", "f1", "FOIA", "2009-07-01", []string{"Docket, Dan"}, + story(pub("Outlet Y", "2014-08-01"), "FOIA story", "https://example.org/foia-story")) + writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[]`) + return root +} + +// Counts that turn on a publication entry's date or byline, or on the +// reader's scope, are still the numbers their links return. +func TestCountsTiedToEntriesAndScope(t *testing.T) { + withCorpus(t, writeEntryCorpus(t)) + + // A publication-year page files a document under its first publication, + // which no search asks for: p1, first published in 2013, would join the + // 2010 bar's search for 2014. So those bars carry no link. The + // document-year page's publication bars do, and they are exact. + body := get(t, "/browse/published/2014").Body.String() + if !strings.Contains(body, "By document date") || strings.Contains(body, `[^<]*(\d+)`) + +// checkCategoryBoxes holds the number on each category box in the browse +// nav to the rows of the list its click opens, for the reader with these +// cookies. +func checkCategoryBoxes(t *testing.T, cookies ...*http.Cookie) int { + t.Helper() + boxes := reCategoryBox.FindAllStringSubmatch(get(t, "/browse", cookies...).Body.String(), -1) + for _, m := range boxes { + slug, shown := m[1], mustAtoi(t, m[2]) + list := get(t, "/browse/"+slug, cookies...).Body.String() + if rows := strings.Count(list, `href="/browse/`+slug+`/`); rows != shown { + t.Errorf("the %s box says %d, its list has %d rows (%d cookies)", slug, shown, rows, len(cookies)) + } + } + return len(boxes) +} + +// -check reads the load counters, so the glossary's problems must still be +// in them when it does. +func TestGlossaryWarningsReachCheck(t *testing.T) { + root := writeConsistencyCorpus(t) + withCorpus(t, root) + clean := loadWarnings + writeFixtureFile(t, filepath.Join(root, "glossary.json"), `{not json`) + loadData() + if loadWarnings != clean+1 { + t.Errorf("malformed glossary.json: %d warnings, want %d", loadWarnings, clean+1) + } +} + +// A 404 under /files must not carry the files' public cache policy. +func TestMissingFileIsNotCached(t *testing.T) { + w := get(t, "/files/prism-slides/no-such-file.pdf") + if w.Code != http.StatusNotFound { + t.Fatalf("status %d, want 404", w.Code) + } + if cc := w.Header().Get("Cache-Control"); strings.Contains(cc, "public") { + t.Errorf("404 carries Cache-Control %q", cc) + } +} + +// -dev serves the on-disk assets so edits show without a restart: no +// startup ETag may pin the old content. +func TestDevModeStaticRevalidatesEdits(t *testing.T) { + buildFixture(t) + devMode = true + initAssets() + t.Cleanup(func() { + devMode = false + initAssets() + }) + if staticHashes != nil { + t.Errorf("dev mode built %d static hashes", len(staticHashes)) + } + w := httptest.NewRecorder() + handleStatic(w, httptest.NewRequest("GET", "/static/style.css", nil)) + if w.Code != http.StatusOK || w.Header().Get("ETag") != "" || w.Header().Get("Last-Modified") == "" { + t.Errorf("dev static: status %d, ETag %q, Last-Modified %q - want 200, none, the file's", + w.Code, w.Header().Get("ETag"), w.Header().Get("Last-Modified")) + } +} + +// What the search page offers to pin is what the pinned page can list. +func TestPinnableSearchesAreListed(t *testing.T) { + h, _ := buildFixture(t) + if label, ok := corpus().searchLabel(mustQuery(t, "/search?do_search=1&fqv=&fqf=TE&fqc=and&date_doc_start=2013&date_doc_end=2013")); !ok || label != "all documents · (doc 2013)" { + t.Errorf("date-only search label = %q, %v", label, ok) + } + if strings.Contains(get(t, "/search?do_search=1&fqv=&fqf=TE&fqc=and").Body.String(), "pin-search-btn") { + t.Error("a search with no terms and no filters offers a pin") + } + if !strings.Contains(get(t, "/search?q=date:2013").Body.String(), "pin-search-btn") { + t.Error("a date-only search offers no pin") + } + + pin := func(qs string) *httptest.ResponseRecorder { + r := httptest.NewRequest("POST", "/pinsearch", strings.NewReader(url.Values{"qs": {qs}, "back": {"/pinned"}}.Encode())) + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + return record(h, r) + } + if w := pin("do_search=1&fqc=and&fqf=TE&fqv="); w.Code != http.StatusBadRequest { + t.Errorf("pinning an empty search: status %d, want 400", w.Code) + } + w := pin("date_doc_end=2013&date_doc_start=2013&do_search=1&fqc=and&fqf=TE&fqv=") + if w.Code != http.StatusSeeOther { + t.Fatalf("pinning a date-only search: status %d", w.Code) + } + body := get(t, "/pinned", w.Result().Cookies()...).Body.String() + if !strings.Contains(html.UnescapeString(body), "all documents · (doc 2013)") { + t.Error("the pinned date-only search is not listed on /pinned") + } +} diff --git a/tests/corpus_test.go b/tests/corpus_test.go new file mode 100644 index 0000000..6bd36d0 --- /dev/null +++ b/tests/corpus_test.go @@ -0,0 +1,199 @@ +package main + +import ( + "html" + "net/http" + "os" + "reflect" + "regexp" + "slices" + "strings" + "testing" +) + +// Checks against a REAL corpus. Point $SNOWCHIVE_BENCH_DATA at one, as for +// the page benchmarks, and run: go test -run TestCorpus -v +// +// The fixtures are a handful of documents, each built to show one thing. A +// count and its link are computed by different code, and whether the two +// agree turns on what the documents happen to hold: a second publication in +// a later year, a byline on only one of them, a value found only in FOIA +// releases. The disagreements these tests walk the archive for were all +// found on the full corpus and on no fixture. +func TestCorpus(t *testing.T) { + root := os.Getenv("SNOWCHIVE_BENCH_DATA") + if root == "" { + t.Skip("set SNOWCHIVE_BENCH_DATA to a corpus directory to check a real corpus") + } + withCorpus(t, root) + t.Run("counts", corpusCounts) + t.Run("FOIA links", corpusFOIALinks) + t.Run("inflections", corpusInflections) + t.Run("snippets", corpusSnippets) + t.Run("index", corpusIndex) +} + +// corpusCounts follows the counted links of the loaded corpus, as a reader +// who leaves FOIA releases out and as one who includes them. +func corpusCounts(t *testing.T) { + // The pages whose bars and refine links are followed: the year pages of + // both date categories, the three commonest values of every other + // category, each cell of the outlets table (a publisher's page for one + // publication year) and searches under each kind of date range. + var pages []string + for ci := range categories { + c := &categories[ci] + n := 3 + if c.isDate() { + n = -1 + } + for _, v := range rankCounts(c.Facet(corpus()), n) { + pages = append(pages, categoryValueURL(c.Slug, v)) + } + } + for _, row := range corpus().outletTables[""].Rows { + for _, cell := range row.Cells { + if cell.URL != "" { + pages = append(pages, cell.URL) + } + } + } + pages = append(pages, + "/search?q=nsa", + "/search?q=nsa+date%3A2012-06..2013-06", + "/search?q=nsa+published%3A2016", + "/search?q=published%3A2014", + "/search?do_search=1&fqv=nsa&fqf=TE&fqc=and&date_pub_start=2015-01-01&date_pub_end=2016-06-30", + "/search?do_search=1&fqv=nsa&fqf=TE&fqc=and&fqv=gchq&fqf=TE&fqc=or&date_pub_start=2016&date_pub_end=2016", + ) + + for name, reader := range map[string][]*http.Cookie{"FOIA left out": nil, "FOIA included": {foiaOn}} { + t.Run(name, func(t *testing.T) { + bars, refines := 0, 0 + for _, p := range pages { + bars += followLinks(t, p, reBarLink, "bar", reader...) + refines += followLinks(t, p, reFacetLink, "refine link", reader...) + } + if bars == 0 || refines == 0 { + t.Errorf("nothing exercised: %d bars, %d refine links", bars, refines) + } + boxes := checkCategoryBoxes(t, reader...) + t.Logf("%d pages: %d bars, %d refine links and %d category boxes checked", len(pages), bars, refines, boxes) + }) + } +} + +// corpusFOIALinks opens the page of every tenth FOIA release as a reader +// who leaves FOIA out, and follows each category value it links: the page +// behind the link has to list the release. +func corpusFOIALinks(t *testing.T) { + reValueLink := regexp.MustCompile(`href="(/browse/[a-z]+/[^"]+)"`) + links := 0 + for i, d := range corpus().collectionDocs[collectionFOIA] { + if i%10 != 0 { + continue + } + // The release's own metadata: the cards under it are other documents. + page, _, _ := strings.Cut(get(t, docPath(d.ID)).Body.String(), `class="related-section"`) + for _, m := range reValueLink.FindAllStringSubmatch(page, -1) { + link := html.UnescapeString(m[1]) + links++ + if !strings.Contains(get(t, withQuery(link, "view=table")).Body.String(), `href="`+docPath(d.ID)+`"`) { + t.Errorf("%s links %s, which does not list it", d.ID, link) + } + } + } + if links == 0 { + t.Error("no FOIA release with a value link to follow") + } + t.Logf("%d value links followed", links) +} + +// A word in -ies or -ied stems as the word in -y it is a form of, whatever +// rules that word goes through - checked over every such pair the corpus +// holds, since a suffix rule that reads one of the two forms and not the +// other splits them silently ("capability" from "capabilities"). +func corpusInflections(t *testing.T) { + vocab := map[string]bool{} + for i := range corpus().archive.Docs { + for _, w := range splitWords(corpus().archive.Docs[i].SearchText) { + vocab[w] = true + } + } + pairs := 0 + for w := range vocab { + if len(w) < 5 || !(strings.HasSuffix(w, "ies") || strings.HasSuffix(w, "ied")) { + continue + } + base := w[:len(w)-3] + "y" + if !vocab[base] { + continue + } + pairs++ + if stem(w) != stem(base) { + t.Errorf("stem(%q) = %q, stem(%q) = %q; want equal", w, stem(w), base, stem(base)) + } + } + if pairs == 0 { + t.Error("the corpus holds no -ies/-ied word next to its -y form") + } + t.Logf("%d pairs checked in a vocabulary of %d words", pairs, len(vocab)) +} + +// corpusSnippets holds every document's snippets off the fold kept at load +// to the ones folding its transcript again gives - the versions' lines sit +// behind that fold in the same string - and a page of "all" cards to its +// bound. +func corpusSnippets(t *testing.T) { + for _, q := range []string{"nsa", "cable tapping", `"upstream collection"`, "tap*", "the"} { + for i := range corpus().archive.Docs { + d := &corpus().archive.Docs[i] + if got, want := d.bodySnippets(q, 3), getSnippets(d.FullText, q, 3); !reflect.DeepEqual(got, want) { + t.Fatalf("%s, %q: %q from the loaded fold, %q from the text", d.ID, q, got, want) + } + } + } + all := &http.Cookie{Name: "pref_per_page", Value: "all"} + for _, c := range categories { + for _, v := range rankCounts(c.Facet(corpus()), 1) { + if c.Facet(corpus())[v] <= maxCardsPerPage { + continue + } + body := get(t, categoryValueURL(c.Slug, v), all, foiaOn).Body.String() + if n := strings.Count(body, `class="doc-card"`); n != maxCardsPerPage { + t.Errorf("%s=%s: %d cards on a page of all, want %d", c.Slug, v, n, maxCardsPerPage) + } + } + } +} + +// corpusIndex holds the shortcut a one-word all-fields clause takes - it +// asks the document's entry in the index instead of walking its text - to +// the walk, for a spread of the vocabulary over every document. +func corpusIndex(t *testing.T) { + var vocab []string + for w := range corpus().tfidf.DocFreq { + vocab = append(vocab, w) + } + slices.Sort(vocab) + checked := 0 + for i := 0; i < len(vocab); i += 131 { + w := vocab[i] + docs := 0 + for di := range corpus().archive.Docs { + d := &corpus().archive.Docs[di] + _, indexed := d.terms[w] + if indexed != containsTokens(d.StemmedText, w) { + t.Fatalf("%q in %s: the index says %v, the all-fields stream the opposite", w, d.ID, indexed) + } + if indexed { + docs++ + } + } + if docs != corpus().tfidf.DocFreq[w] { + t.Errorf("%q: in %d documents' entries, with a document frequency of %d", w, docs, corpus().tfidf.DocFreq[w]) + } + checked++ + } + t.Logf("%d terms checked over %d documents", checked, len(corpus().archive.Docs)) +} diff --git a/tests/data_test.go b/tests/data_test.go new file mode 100644 index 0000000..5b09ddf --- /dev/null +++ b/tests/data_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestAkaIDShape(t *testing.T) { + for _, s := range []string{ + `{"title": "no aka_id key"}`, + `{"aka_id": []}`, + // null is "no akas": not a reason to drop the document. + `{"aka_id": null}`, + `{"aka_id": [{"id": "a"}, {"id": "b", "doc_source": "https://e.org/d", "doc_url": "https://e.org/b.pdf"}]}`, + } { + var d Document + if err := json.Unmarshal([]byte(s), &d); err != nil { + t.Errorf("%s: unexpected error: %v", s, err) + } + } + + for _, s := range []string{`{"aka_id": "a"}`, `{"aka_id": {"id": "a"}}`, `{"aka_id": ["a", "b"]}`, `{"aka_id": [{"id": "a"}, 3]}`} { + var d Document + if err := json.Unmarshal([]byte(s), &d); err == nil { + t.Errorf("%s: decoded without an error", s) + } + } +} + +func TestNormalizeAka(t *testing.T) { + d := Document{ID: "x", AkaID: []AkaEntry{ + {ID: " first ", DocSource: " https://e.org/p "}, + {ID: " "}, + {ID: "FIRST"}, + {ID: "second", DocURL: "https://e.org/s.pdf"}, + }} + got := normalizeAka(&d) + if len(got) != 2 || got[0].ID != "first" || got[0].DocSource != "https://e.org/p" || got[1].ID != "second" { + t.Errorf("normalizeAka = %+v, want first (trimmed) and second, without the blank id and the repeat", got) + } +} + +func writeTestFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestLoadVersions(t *testing.T) { + dir := t.TempDir() + v := filepath.Join(dir, "versions") + writeTestFile(t, filepath.Join(v, "lemonde-2013", "copy.pdf"), "%PDF-1.4") + writeTestFile(t, filepath.Join(v, "lemonde-2013", "copy.md"), "# Slide 12\n\nOnly in Le Monde's copy.") + writeTestFile(t, filepath.Join(v, "lemonde-2013", "images", "page-10.jpg"), "") + writeTestFile(t, filepath.Join(v, "lemonde-2013", "images", "page-2.jpg"), "") + writeTestFile(t, filepath.Join(v, "guardian", "copy.pdf"), "%PDF-1.4") // no transcript or images: still a version + writeTestFile(t, filepath.Join(v, "no-pdf", "copy.md"), "text only") // skipped: no PDF + writeTestFile(t, filepath.Join(v, "bad name", "copy.pdf"), "%PDF-1.4") // skipped: name + writeTestFile(t, filepath.Join(v, "loose.pdf"), "%PDF-1.4") // skipped: not a folder + + var d Document + if err := json.Unmarshal([]byte(`{"publication": [ + {"publisher": "Le Monde", "version": "lemonde-2013"}, + {"publisher": "The Guardian"}, + {"publisher": "Dagbladet", "version": " lemonde-2013 "}, + {"publisher": "Nobody", "version": "missing"} + ]}`), &d); err != nil { + t.Fatal(err) + } + d.ID, d.Dir = "prism", dir + corpus().loadVersions(&d) + + var slugs []string + for _, v := range d.Versions { + slugs = append(slugs, v.Slug) + } + if got := strings.Join(slugs, ","); got != "guardian,lemonde-2013" { + t.Fatalf("versions %q, want guardian,lemonde-2013", got) + } + + lm := d.VersionFor("lemonde-2013") + if lm.PDFFile != "copy.pdf" || !strings.Contains(lm.FullText, "Only in Le Monde") || lm.HTMLContent == "" { + t.Errorf("lemonde-2013 not fully loaded: pdf %q, text %q, html %q", lm.PDFFile, lm.FullText, lm.HTMLContent) + } + if got := strings.Join(lm.Images, ","); got != "page-2.jpg,page-10.jpg" { + t.Errorf("images %q, want page-2.jpg,page-10.jpg (page order)", got) + } + if got := fmt.Sprint(lm.PubIndex); got != "[0 2]" { + t.Errorf("PubIndex %s, want [0 2]", got) + } + if got := d.Publication[2].Version; got != "lemonde-2013" { + t.Errorf("publication version not trimmed: %q", got) + } + if u := d.UnlinkedVersions(); len(u) != 1 || u[0].Slug != "guardian" { + t.Errorf("unlinked versions %+v, want only guardian", u) + } + if d.VersionFor("missing") != nil || d.VersionFor("") != nil { + t.Error("VersionFor returned a version that doesn't exist") + } +} + +// TestBodyFormsAreTheCombinedTails: loadData slices FoldedFull and +// StemmedFull out of the ends of SearchText and StemmedText to save memory; +// they must still be exactly what setBodySearchText builds. +func TestBodyFormsAreTheCombinedTails(t *testing.T) { + buildFixture(t) + for i := range corpus().archive.Docs { + d := &corpus().archive.Docs[i] + c := *d + setBodySearchText(&c) + if c.FoldedFull != d.FoldedFull || c.StemmedFull != d.StemmedFull { + t.Errorf("%s: body forms differ from a fresh build", d.ID) + } + } +} diff --git a/tests/fixture_test.go b/tests/fixture_test.go new file mode 100644 index 0000000..1b39265 --- /dev/null +++ b/tests/fixture_test.go @@ -0,0 +1,295 @@ +package main + +import ( + "bytes" + "encoding/json" + "image" + "image/jpeg" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +// A small corpus on disk plus a running mux: the harness every handler test +// uses. The fixture covers the shapes the handlers branch on - one document +// per collection, a SIDtoday issue with a series, another published copy in +// versions/, a joint investigation with a reporting-only partner, and a +// directory with no PDF (which the loader must skip). +// +// Built once for the package: loadData writes package globals, so tests +// share one corpus rather than racing to replace it. + +var ( + fixtureOnce sync.Once + fixtureMux http.Handler + fixtureDir string +) + +type fixtureDoc struct { + id, collection string + meta map[string]interface{} + text string + pages []string + versions map[string][]string // slug -> page images + noPDF bool +} + +// fixtureJPEG is a real 4x3 JPEG, so the loader can read each page image's size. +var fixtureJPEG = func() string { + var b bytes.Buffer + if err := jpeg.Encode(&b, image.NewGray(image.Rect(0, 0, 4, 3)), nil); err != nil { + panic(err) + } + return b.String() +}() + +func writeFixtureFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +// TestMain removes the shared fixture corpus once every test has run. +func TestMain(m *testing.M) { + code := m.Run() + if fixtureDir != "" { + os.RemoveAll(fixtureDir) + } + os.Exit(code) +} + +func buildFixture(t *testing.T) (http.Handler, string) { + t.Helper() + fixtureOnce.Do(func() { + root, err := os.MkdirTemp("", "snowchive-fixture") + if err != nil { + t.Fatal(err) + } + for _, d := range fixtureDocs() { + dir := root + switch d.collection { + case collectionFOIA: + dir = filepath.Join(root, "FOIA") + case collectionArchive: + dir = filepath.Join(root, "SIDtoday-archive") + } + dir = filepath.Join(dir, d.id) + b, err := json.MarshalIndent(d.meta, "", " ") + if err != nil { + t.Fatal(err) + } + writeFixtureFile(t, filepath.Join(dir, "metadata.json"), string(b)+"\n") + if !d.noPDF { + writeFixtureFile(t, filepath.Join(dir, d.id+".pdf"), "%PDF-1.4\nfixture\n") + } + if d.text != "" { + writeFixtureFile(t, filepath.Join(dir, d.id+".md"), d.text) + } + for _, p := range d.pages { + writeFixtureFile(t, filepath.Join(dir, "images", p), fixtureJPEG) + } + for slug, imgs := range d.versions { + vdir := filepath.Join(dir, "versions", slug) + writeFixtureFile(t, filepath.Join(vdir, slug+".pdf"), "%PDF-1.4\n") + writeFixtureFile(t, filepath.Join(vdir, slug+".md"), + "# Copy\n\nA line only this copy has: unredacted FAIRVIEW detail.\n") + for _, p := range imgs { + writeFixtureFile(t, filepath.Join(vdir, "images", p), fixtureJPEG) + } + } + } + writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[ + {"term": "XKEYSCORE", "definition": "A search system.", "aka": ["XKS"]} + ]`) + + dataDir, projectDir = root, root + devMode = false + apiEnabled = true + initAssets() + if err := parseTemplates(); err != nil { + t.Fatalf("templates: %v", err) + } + loadData() + fixtureDir = root + fixtureMux = withBaseHeaders(newMux()) + }) + if fixtureMux == nil { + t.Fatal("fixture not built") + } + return fixtureMux, fixtureDir +} + +func fixtureDocs() []fixtureDoc { + return []fixtureDoc{ + { + id: "prism-slides", + collection: collectionNews, + text: "# PRISM\n\nCollection directly from the servers. XKEYSCORE ingests it.\n" + + "Upstream collection feeds the corpus, and analysts were collecting metadata\n" + + "from RT-RG dashboards. Müller-Maguhn reviewed the São Paulo cable tapping.\n", + pages: []string{"page-1.jpg", "page-2.jpg"}, + versions: map[string][]string{"lemonde": {"page-1.jpg"}}, + meta: map[string]interface{}{ + "title": "PRISM Collection Slides", + "type": "REPORTED", + "description": "Slides describing PRISM collection.", + "tags": []string{"PRISM", "collection", "cable tapping"}, + "pdf_sha256": strings.Repeat("a", 64), + "aka_id": []map[string]string{{"id": "prism-slides-guardian"}}, + "document": map[string]interface{}{ + "creator": "NSA", "date_created": "2013-04-01", + "programs": []string{"PRISM", "XKEYSCORE"}, + "classification": []string{"TOP SECRET"}, "dissemination": []string{"NOFORN"}, + "countries": []string{"United States", "Brazil"}, + "people": []string{"Snowden, Edward", "Müller-Maguhn, Andy"}, + "banner": "TOP SECRET//SI//NOFORN", "genre": "slides", + }, + "publication": []map[string]interface{}{ + {"publisher": "The Guardian", "title": "NSA Prism program", + "link": "https://example.org/prism", "date_published": "2013-06-06", + "reporters": []string{"Greenwald, Glenn"}}, + {"publisher": "Le Monde", "link": "https://example.org/prism-fr", + "date_published": "2013-06-08", "version": "lemonde"}, + {"publisher": "NRK", "link": "https://example.org/prism", + "date_published": "2013-06-06", "reporting_only": true}, + }, + }, + }, + { + // Shares the Guardian link with prism-slides, which is what + // makes the "published in the same report" section exist. + id: "prism-companion", collection: collectionNews, + text: "# Companion\n\nA second document from the same story.\n", + pages: []string{"page-1.jpg"}, + meta: map[string]interface{}{ + "title": "PRISM Companion Slide", "type": "REPORTED", + "description": "A second slide from the same story.", + "tags": []string{"PRISM"}, + "document": map[string]interface{}{ + "creator": "NSA", "date_created": "2013-04-02", + "programs": []string{"PRISM"}, + }, + "publication": []map[string]interface{}{ + {"publisher": "The Guardian", "title": "NSA Prism program", + "link": "https://example.org/prism", "date_published": "2013-06-06", + "reporters": []string{"Greenwald, Glenn"}}, + }, + }, + }, + { + id: "2003-05-29_SIDToday_-_Fixture_Issue", + collection: collectionArchive, + text: "# SIDtoday\n\nThe analyst cockpit and other matters. Upstream tapping was\n" + + "discussed at length by the directorate, and the collection of that traffic\n" + + "began only the following year.\n", + pages: []string{"page-1.jpg"}, + meta: map[string]interface{}{ + "title": "A Day in the Life of a Fixture", "type": "SIDTODAY", + "description": "An issue of SIDtoday.", + "tags": []string{"analysis"}, + "document": map[string]interface{}{ + "creator": "NSA", "date_created": "2003-05-29", + "classification": []string{"UNCLASSIFIED"}, "distribution": []string{}, + "series": map[string]interface{}{"name": "(U) A Day in the Life of...", "part": 1}, + }, + "publication": []map[string]interface{}{ + {"publisher": "The Intercept", "link": "https://example.org/sidtoday/1", + "date_published": "2016-05-16"}, + }, + }, + }, + { + id: "FISC_-_fixture_order", + collection: collectionFOIA, + text: "# Order\n\nThe Court finds the application satisfies the statute.\n", + meta: map[string]interface{}{ + "title": "Fixture FISC Order", "type": "FOIA", + "description": "A court order released under FOIA.", + "document": map[string]interface{}{ + "creator": "FISC", "date_created": "2013-10-11", + "classification": []string{"SECRET"}, + }, + "publication": []map[string]interface{}{{"publisher": "ODNI", "link": "https://example.org/foia/1"}}, + }, + }, + { + id: "undated-fixture", + collection: collectionNews, + text: "# Undated\n\nNo document date at all.\n", + meta: map[string]interface{}{ + "title": "Undated Fixture", "type": "REPORTED", + "description": "A document with no date.", + "document": map[string]interface{}{"creator": "GCHQ"}, + "publication": []map[string]interface{}{{"publisher": "The Intercept", "link": "https://example.org/undated"}}, + }, + }, + { + id: "broken-no-pdf", + collection: collectionNews, + noPDF: true, + meta: map[string]interface{}{"title": "Broken", "type": "REPORTED"}, + }, + } +} + +// get runs one request through the real routing table and header wrapper. +func get(t *testing.T, path string, cookies ...*http.Cookie) *httptest.ResponseRecorder { + t.Helper() + mux, _ := buildFixture(t) + r := httptest.NewRequest("GET", path, nil) + for _, c := range cookies { + r.AddCookie(c) + } + w := httptest.NewRecorder() + mux.ServeHTTP(w, r) + return w +} + +// newPost builds a form POST for the state-changing endpoints. +func newPost(path string) *http.Request { + r := httptest.NewRequest("POST", path, strings.NewReader("back=/browse")) + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + return r +} + +func record(h http.Handler, r *http.Request) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + return w +} + +// withCorpus loads another corpus for one test and puts the fixture back +// afterwards. Only possible because loadData is idempotent - every build* +// starts from empty (TestLoadDataIsIdempotent). +func withCorpus(t *testing.T, root string) { + t.Helper() + withDirs(t, root, root) +} + +// withDirs is withCorpus with the corpus and the project files apart, as +// -root lays them out. +func withDirs(t *testing.T, data, project string) { + t.Helper() + buildFixture(t) // make sure templates and the fixture path exist first + prev := dataDir + t.Cleanup(func() { + dataDir, projectDir = prev, prev + loadData() + }) + dataDir, projectDir = data, project + loadData() +} + +// httptest_NewGet is a plain GET request for the tests that drive a mux +// directly rather than through get(). +func httptest_NewGet(path string) *http.Request { + return httptest.NewRequest("GET", path, nil) +} diff --git a/tests/flags_test.go b/tests/flags_test.go new file mode 100644 index 0000000..647fcbd --- /dev/null +++ b/tests/flags_test.go @@ -0,0 +1,167 @@ +package main + +// flags_test.go - what each command-line flag actually changes. The flags +// are parsed in main(), which no test can call, so these exercise the state +// the flags set and the behaviour that depends on it: the routes an +// instance serves, the URLs it writes, the warnings it prints. + +import ( + "encoding/json" + "net/http" + "path/filepath" + "strings" + "testing" +) + +// -no-api: every /api/ path answers a JSON 403, and the UI stops offering +// the API. The kill switch is one registration, so a future endpoint +// inherits it - that is the property worth testing. +func TestFlagNoAPI(t *testing.T) { + buildFixture(t) + prev := apiEnabled + t.Cleanup(func() { apiEnabled = prev; fixtureMux = withBaseHeaders(newMux()) }) + + apiEnabled = false + mux := withBaseHeaders(newMux()) + for _, p := range []string{"/api/docs", "/api/doc/prism-slides", "/api/search?q=x", + "/api/facets", "/api/something-added-next-year"} { + w := record(mux, httptest_NewGet(p)) + if w.Code != http.StatusForbidden { + t.Errorf("%s with -no-api: status %d, want 403", p, w.Code) + } + var body map[string]string + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil || body["error"] == "" { + t.Errorf("%s: body %q is not a JSON error", p, w.Body.String()) + } + if ct := w.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/json") { + t.Errorf("%s: Content-Type %q", p, ct) + } + } + // The visitor's own export is NOT part of the API and stays available. + if w := record(mux, httptest_NewGet("/pinned/export")); w.Code != 200 { + t.Errorf("/pinned/export with -no-api: %d", w.Code) + } +} + +// -base: the absolute URLs in robots.txt and the sitemap follow +// the canonical host instead of the request's Host header. +func TestFlagBase(t *testing.T) { + buildFixture(t) + prev, prevMap := canonicalBase, corpus().canonicalSitemap + t.Cleanup(func() { + canonicalBase, corpus().canonicalSitemap = prev, prevMap + corpus().buildCanonicalSitemap() + }) + canonicalBase = "https://archive.example.org" + corpus().buildCanonicalSitemap() + + for _, c := range []struct{ path, want string }{ + {"/robots.txt", "Sitemap: https://archive.example.org/sitemap.xml"}, + {"/sitemap.xml", "https://archive.example.org/doc/prism-slides"}, + } { + if body := get(t, c.path).Body.String(); !strings.Contains(body, c.want) { + t.Errorf("%s does not carry %q", c.path, c.want) + } + } + // A hostile Host header cannot reach a response once -base is set. + r := httptest_NewGet("/robots.txt") + r.Host = "evil.example.net" + if body := record(fixtureMux, r).Body.String(); strings.Contains(body, "evil") { + t.Errorf("the Host header leaked into robots.txt: %s", body) + } +} + +// -base is validated, not trusted: anything but scheme://host is refused, +// so the flag cannot produce URLs that do not resolve. +func TestFlagBaseValidation(t *testing.T) { + for _, ok := range []string{"https://a.org", "http://a.org", "https://a.org:8443", "https://a.org/"} { + if _, err := parseCanonicalBase(ok); err != nil { + t.Errorf("%s rejected: %v", ok, err) + } + } + for _, bad := range []string{"", "a.org", "ftp://a.org", "https://", "https://a.org/path", + "https://a.org?q=1", "https://a.org#f", "https://user@a.org", "://x"} { + if _, err := parseCanonicalBase(bad); err == nil { + t.Errorf("%q accepted", bad) + } + } +} + +// -check counts what it reports: the fixture holds one document the loader +// refuses (no PDF) and one directory that looks like a document. +func TestFlagCheckCounts(t *testing.T) { + buildFixture(t) + if loadErrors == 0 { + t.Error("the fixture's PDF-less document should be an error") + } + if loadErrors+loadWarnings == 0 { + t.Error("-check would exit 0 on a corpus with problems") + } +} + +// -listen warns for anything but loopback, because binding elsewhere puts +// the archive in front of the proxy instead of behind it. +func TestFlagListenWarning(t *testing.T) { + before := loadWarnings + for _, quiet := range []string{"127.0.0.1:8080", "localhost:8080", "[::1]:8080", "127.0.0.53:80"} { + warnNonLoopback(quiet) + if loadWarnings != before { + t.Errorf("%s warned", quiet) + before = loadWarnings + } + } + for _, loud := range []string{"0.0.0.0:8080", ":8080", "192.168.1.10:8080", "[::]:8080"} { + warnNonLoopback(loud) + if loadWarnings == before { + t.Errorf("%s did not warn", loud) + } + before = loadWarnings + } +} + +// -dev: templates come from disk and nothing is pre-rendered, so every page +// must still render - the fallback path the pre-rendered blocks rely on. +func TestFlagDevMode(t *testing.T) { + buildFixture(t) + prevDev, prevPre, prevTmpl := devMode, corpus().prerendered, tmplSet + t.Cleanup(func() { + devMode, corpus().prerendered, tmplSet = prevDev, prevPre, prevTmpl + initAssets() + }) + devMode = true + initAssets() // templates/ and static/ from the working directory + corpus().prerenderPages() + if len(corpus().prerendered) != 0 { + t.Error("dev mode pre-rendered pages; it must render them live") + } + for _, p := range []string{"/", "/browse", "/browse/program", "/explore/ids", "/glossary", + "/articles", "/explore/terms", "/doc/prism-slides"} { + if w := get(t, p); w.Code != 200 { + t.Errorf("%s in dev mode: %d", p, w.Code) + } + } +} + +// -root: the documents are read from its data/ and the files about the +// archive from its project/. +func TestRootDir(t *testing.T) { + buildFixture(t) + prevData, prevProject := dataDir, projectDir + t.Cleanup(func() { + dataDir, projectDir = prevData, prevProject + loadData() + }) + root := t.TempDir() + writeFixtureFile(t, filepath.Join(root, "data", "doc-in-root", "metadata.json"), `{"title": "In the root"}`) + writeFixtureFile(t, filepath.Join(root, "data", "doc-in-root", "doc-in-root.pdf"), "%PDF-1.4\n") + writeFixtureFile(t, filepath.Join(root, "project", "glossary.json"), `[{"term": "INPROJECT", "definition": "Read from project/."}]`) + setRoot(root) + loadData() + + if corpus().docIndex["doc-in-root"] == nil { + t.Error("the document in data/ was not loaded") + } + if corpus().glossaryLookup("INPROJECT") == nil { + t.Error("glossary.json in project/ was not loaded") + } +} diff --git a/tests/foia_test.go b/tests/foia_test.go new file mode 100644 index 0000000..7e35aab --- /dev/null +++ b/tests/foia_test.go @@ -0,0 +1,272 @@ +package main + +import ( + "html" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "testing" +) + +// foiaOn is the cookie of a reader who opted in to FOIA releases. +var foiaOn = &http.Cookie{Name: "pref_include_foia", Value: "1"} + +// In the consistency corpus "alpha" is in n1, n2 and the FOIA release f1. +func TestSearchLeavesOutFOIAByDefault(t *testing.T) { + withCorpus(t, writeConsistencyCorpus(t)) + + if got := resultTotal(t, "/search?q=alpha"); got != 2 { + t.Errorf("q=alpha, no preference: %d results, want 2 (FOIA left out)", got) + } + body := get(t, "/search?q=alpha").Body.String() + if strings.Contains(body, "left out") { + t.Error("the page counts the FOIA releases it left out") + } + // The page's own links name the collections it searched, so a copied + // link shows the same results to a reader with the other preference. + if !strings.Contains(body, "coll=news&coll=sidtoday-archive") { + t.Error("the page's links do not carry the default collections") + } + if got := resultTotal(t, "/search?q=alpha&coll=news&coll=sidtoday-archive&coll=foia"); got != 3 { + t.Errorf("q=alpha in every collection: %d results, want 3", got) + } + + if got := resultTotal(t, "/search?q=alpha", foiaOn); got != 3 { + t.Errorf("q=alpha with FOIA on: %d results, want 3", got) + } + if strings.Contains(get(t, "/search?q=alpha", foiaOn).Body.String(), "left out") { + t.Error("FOIA on, yet the page says something was left out") + } + // An explicit collection choice wins over the preference. + if got := resultTotal(t, "/search?q=alpha&coll=foia"); got != 1 { + t.Errorf("q=alpha&coll=foia, no preference: %d results, want 1", got) + } + if got := resultTotal(t, "/search?q=alpha&coll=news", foiaOn); got != 2 { + t.Errorf("q=alpha&coll=news with FOIA on: %d results, want 2", got) + } +} + +// ZEBRA is on n1, n2, f1 (FOIA) and s1 (SIDtoday). +func TestCategoryPagesLeaveOutFOIAByDefault(t *testing.T) { + withCorpus(t, writeConsistencyCorpus(t)) + + if got := resultTotal(t, "/browse/program/ZEBRA"); got != 3 { + t.Errorf("ZEBRA, no preference: %d documents, want 3", got) + } + if strings.Contains(get(t, "/browse/program/ZEBRA").Body.String(), "left out") { + t.Error("the page still carries a FOIA notice") + } + if got := resultTotal(t, "/browse/program/ZEBRA", foiaOn); got != 4 { + t.Errorf("ZEBRA with FOIA on: %d documents, want 4", got) + } + if strings.Contains(get(t, "/browse/program/ZEBRA", foiaOn).Body.String(), "left out") { + t.Error("FOIA on, yet the page says FOIA releases are left out") + } +} + +// Every total in a category table is the number its row's link lists, for +// either preference. +func TestCategoryTotalsMatchTheirPages(t *testing.T) { + withCorpus(t, writeConsistencyCorpus(t)) + reRow := regexp.MustCompile(`(?s)\s*([^<]+?)\s*.*?\s*(\d+)`) + for _, cookies := range [][]*http.Cookie{nil, {foiaOn}} { + for _, slug := range []string{"program", "agency", "docdate", "published", "publisher"} { + body := get(t, "/browse/"+slug, cookies...).Body.String() + _, body, _ = strings.Cut(body, "") + body, _, _ = strings.Cut(body, "") + rows := reRow.FindAllStringSubmatch(body, -1) + if len(rows) == 0 { + t.Fatalf("/browse/%s: no rows", slug) + } + for _, row := range rows { + want, _ := strconv.Atoi(row[3]) + if strings.Contains(row[1], "coll=") { + t.Errorf("/browse/%s %s: row links with a collection scope: %s", slug, row[2], row[1]) + } + if got := resultTotal(t, html.UnescapeString(row[1]), cookies...); got != want { + t.Errorf("/browse/%s %s (FOIA on: %v): total %d, its page lists %d", slug, row[2], cookies != nil, want, got) + } + } + } + } + off, on := get(t, "/browse/program").Body.String(), get(t, "/browse/program", foiaOn).Body.String() + if strings.Contains(off, "Totals leave out") || strings.Contains(off, "sort=foia") { + t.Error("FOIA off: the table still has a FOIA notice or column") + } + if !strings.Contains(on, "sort=foia") { + t.Error("FOIA on: the table has no FOIA column") + } +} + +func TestRandomAndRelatedLeaveOutFOIA(t *testing.T) { + withCorpus(t, writeConsistencyCorpus(t)) + + sawFOIA := false + for i := 0; i < 300; i++ { + if d := corpus().randomDoc(false); d == nil || d.Collection == collectionFOIA { + t.Fatalf("randomDoc(false) = %v", d) + } + if corpus().randomDoc(true).Collection == collectionFOIA { + sawFOIA = true + } + } + if !sawFOIA { + t.Error("randomDoc(true) never picked the FOIA release in 300 tries") + } + + corpus().archive.RelatedDocs["n2"] = []string{"f1", "n1"} + corpus().archive.RelatedDocs["f1"] = []string{"n1"} + body := get(t, "/doc/n2").Body.String() + if strings.Contains(body, `href="/doc/f1"`) || !strings.Contains(body, `href="/doc/n1"`) { + t.Error("/doc/n2, no preference: want n1 related and f1 left out") + } + if !strings.Contains(get(t, "/doc/n2", foiaOn).Body.String(), `href="/doc/f1"`) { + t.Error("/doc/n2 with FOIA on: f1 is not related") + } + if !strings.Contains(get(t, "/doc/f1").Body.String(), `href="/doc/n1"`) { + t.Error("/doc/f1: its related documents are gone") + } +} + +// The home page counts the Snowden documents apart from the FOIA releases and +// says how the preference treats them. The corpus: n1-n4, a1, d1 (news), s1 +// (SIDtoday archive), f1 (FOIA). +func TestHomeStatesTheFOIAScope(t *testing.T) { + withCorpus(t, writeConsistencyCorpus(t)) + body := get(t, "/").Body.String() + for _, want := range []string{"holds 7 published Snowden documents. 6 documents are", "1 documents are the NSA newsletter", + "also holds 1", "are left out by default but can be included via"} { + if !strings.Contains(body, want) { + t.Errorf("home page lacks %q", want) + } + } + if !strings.Contains(get(t, "/", foiaOn).Body.String(), "you have included via") { + t.Error("FOIA on: the home page does not say so") + } +} + +func TestFOIAPreferenceIsSaved(t *testing.T) { + withCorpus(t, writeConsistencyCorpus(t)) + if !strings.Contains(get(t, "/preferences").Body.String(), `name="include_foia"`) { + t.Error("/preferences has no FOIA checkbox") + } + if readPrefs(httptest_NewGet("/")).IncludeFOIA { + t.Error("FOIA releases are included by default") + } +} + +// The form shows the scope the search ran with: FOIA left out by default +// is an unchecked box, so submitting the form keeps it out. +func TestSearchFormKeepsTheDefaultScope(t *testing.T) { + withCorpus(t, writeConsistencyCorpus(t)) + + checked := regexp.MustCompile(`value="foia" checked`) + if checked.MatchString(get(t, "/search?q=alpha").Body.String()) { + t.Error("FOIA left out, yet its box is checked") + } + if !checked.MatchString(get(t, "/search?q=alpha", foiaOn).Body.String()) { + t.Error("FOIA on, yet its box is not checked") + } + if got := resultTotal(t, "/search?q=alpha&coll=news&coll=sidtoday-archive&do_search=1"); got != 2 { + t.Errorf("the form's resubmit finds %d, want 2", got) + } +} + +// A FOIA release's metadata links name every collection, so its value +// pages list it under the default scope. +func TestFOIADocLinksIncludeFOIA(t *testing.T) { + buildFixture(t) + // Every value link printed for the release - on its own page, on its + // card and its table row in the FOIA tab, in a search result - opens a + // page that lists the release. + const id = "FISC_-_fixture_order" + reValueLink := regexp.MustCompile(`href="(/browse/[a-z]+/[^"]+)"`) + for _, page := range []string{"/doc/" + id, "/browse?collection=foia", "/browse?collection=foia&view=table", "/search?q=court&coll=foia"} { + links := reValueLink.FindAllStringSubmatch(get(t, page).Body.String(), -1) + if len(links) < 2 { + t.Fatalf("%s: %d value links, want the release's agency and classification at least", page, len(links)) + } + for _, m := range links { + link := html.UnescapeString(m[1]) + if !strings.Contains(get(t, withQuery(link, "view=table")).Body.String(), `href="/doc/`+id+`"`) { + t.Errorf("%s links %s, which does not list the release", page, link) + } + } + } + // /explore/classification counts every collection, and so do its links. + for _, m := range regexp.MustCompile(`href="(/browse/[^"]+)"`).FindAllStringSubmatch(get(t, "/explore/classification").Body.String(), -1) { + if !strings.Contains(m[1], "coll=foia") { + t.Errorf("/explore/classification links %s without its scope", m[1]) + } + } + + body := get(t, "/doc/FISC_-_fixture_order").Body.String() + m := regexp.MustCompile(`href="(/browse/agency/FISC[^"]*)"`).FindStringSubmatch(body) + if m == nil || !strings.Contains(m[1], "coll=foia") { + t.Fatalf("agency link %v does not include FOIA", m) + } + if got := resultTotal(t, html.UnescapeString(m[1])); got != 1 { + t.Errorf("the agency link lists %d documents, want 1", got) + } + if strings.Contains(get(t, "/doc/FISC_-_fixture_order", foiaOn).Body.String(), "/browse/agency/FISC?") { + t.Error("FOIA on: the links need no scope") + } +} + +// On a publisher page scoped to a publication year, the links built from +// its documents carry that year. +func TestPublisherYearLinksCarryTheYear(t *testing.T) { + buildFixture(t) + body := get(t, "/browse/publisher/The%20Intercept?year=2016").Body.String() + if !strings.Contains(body, "date_pub_start=2016") { + t.Error("no link on the year-scoped page carries the publication year") + } +} + +// collScope is the one place a page's collection scope is decided: FOIA +// releases are opt-in when the URL names no collection, an explicit coll= +// wins, and only the search pins a defaulted scope into its links. +func TestCollScope(t *testing.T) { + off, on := Prefs{}, Prefs{IncludeFOIA: true} + none := url.Values{} + for _, c := range []struct { + name string + v url.Values + p Prefs + pin bool + foia, def bool + qs string + }{ + {"default, browse", none, off, false, false, true, ""}, + {"default, search", none, off, true, false, true, "coll=news&coll=sidtoday-archive"}, + {"opted in, browse", none, on, false, true, true, ""}, + {"opted in, search", none, on, true, true, true, "coll=news&coll=sidtoday-archive&coll=foia"}, + {"explicit foia", url.Values{"coll": {"foia"}}, off, false, true, false, "coll=foia"}, + {"explicit news over opt-in", url.Values{"coll": {"news"}}, on, false, false, false, "coll=news"}, + {"explicit everything", url.Values{"coll": {"news", "foia", "sidtoday-archive"}}, off, false, true, false, + "coll=news&coll=sidtoday-archive&coll=foia"}, + } { + s := pageScope(c.v, c.p, c.pin) + foia := s.shows(&Document{Collection: collectionFOIA}) + if foia != c.foia || s.defaulted != c.def || s.qs() != c.qs || s.shown[collectionFOIA] != c.foia { + t.Errorf("%s: shows FOIA %v, defaulted %v, qs %q; want %v, %v, %q", + c.name, foia, s.defaulted, s.qs(), c.foia, c.def, c.qs) + } + } + + // A FOIA release's links widen the default to the whole archive and + // say so; a reader who opted in needs no widening. + if got := pageScope(none, off, false).including(collectionFOIA).suffix(); got != "?coll=news&coll=sidtoday-archive&coll=foia" { + t.Errorf("widened default: %q", got) + } + if got := pageScope(none, on, false).including(collectionFOIA).suffix(); got != "" { + t.Errorf("opted in, widened: %q, want no scope", got) + } + // and adds to the links, not to the documents. + s := pageScope(none, off, false).and("date_pub_start=2014") + if s.link("/search?q=x") != "/search?q=x&date_pub_start=2014" || s.shows(&Document{Collection: collectionFOIA}) { + t.Errorf("and: %q", s.link("/search?q=x")) + } +} diff --git a/tests/go.mod b/tests/go.mod new file mode 100644 index 0000000..712d687 --- /dev/null +++ b/tests/go.mod @@ -0,0 +1,5 @@ +// Keeps this directory out of the snowchive module's ./... patterns. The +// files here are package main's tests; tests/run.sh compiles them with it. +module snowchive/tests + +go 1.22 diff --git a/tests/headings_test.go b/tests/headings_test.go new file mode 100644 index 0000000..325b241 --- /dev/null +++ b/tests/headings_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "regexp" + "strconv" + "strings" + "testing" +) + +var reHeadingTag = regexp.MustCompile(`A

    B

    C

    D

    `, + "### A\n\n### B\n\n#### C": `

    A

    B

    C

    `, + "```\n# not a heading\n```\n\n## A": `

    A

    `, + } { + var got []string + for _, line := range strings.Split(Render(src), "\n") { + if strings.HasPrefix(line, " prev+1 { + t.Errorf("%s: follows ", path, level, prev) + } + prev = level + } + if h1 != 1 { + t.Errorf("%s: %d

    , want 1", path, h1) + } + } +} diff --git a/tests/http_test.go b/tests/http_test.go new file mode 100644 index 0000000..b255c6a --- /dev/null +++ b/tests/http_test.go @@ -0,0 +1,231 @@ +package main + +// http_test.go - protocol behaviour the browser and the proxy rely on: +// HEAD, byte ranges, conditional requests, and method handling. net/http +// implements most of it, but only if the handlers hand it the right thing - +// and a download path that loses Range support breaks resuming a 120 MB PDF +// and the page-by-page fetches a PDF viewer makes. + +import ( + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" +) + +func TestHTTPHead(t *testing.T) { + buildFixture(t) + for _, p := range []string{"/", "/browse", "/doc/prism-slides", + "/files/prism-slides/prism-slides.pdf", "/api/facets", "/static/style.css"} { + r := httptest.NewRequest("HEAD", p, nil) + w := httptest.NewRecorder() + fixtureMux.ServeHTTP(w, r) + if w.Code != 200 { + t.Errorf("HEAD %s: %d", p, w.Code) + continue + } + // net/http drops the body for HEAD at the transport layer, so the + // recorder still sees one; the headers are what matters here. + if w.Header().Get("Content-Type") == "" { + t.Errorf("HEAD %s: no Content-Type", p) + } + } +} + +// Range on a document file. http.ServeFile implements it - this test is +// here so a future change to the download path (a buffered writer, a +// compressing wrapper, a manual io.Copy) cannot quietly remove it. +func TestHTTPRangeOnFiles(t *testing.T) { + buildFixture(t) + const path = "/files/prism-slides/prism-slides.pdf" + full := get(t, path) + body := full.Body.String() + if full.Header().Get("Accept-Ranges") != "bytes" { + t.Fatalf("no Accept-Ranges on a document file: %v", full.Header()) + } + + req := func(rangeHdr string) *httptest.ResponseRecorder { + r := httptest.NewRequest("GET", path, nil) + r.Header.Set("Range", rangeHdr) + w := httptest.NewRecorder() + fixtureMux.ServeHTTP(w, r) + return w + } + + w := req("bytes=0-3") + if w.Code != http.StatusPartialContent { + t.Fatalf("Range bytes=0-3: status %d, want 206", w.Code) + } + if got, want := w.Body.String(), body[:4]; got != want { + t.Errorf("range body %q, want %q", got, want) + } + if cr := w.Header().Get("Content-Range"); cr != fmt.Sprintf("bytes 0-3/%d", len(body)) { + t.Errorf("Content-Range %q", cr) + } + + // A suffix range: what a resumed download asks for. + w = req(fmt.Sprintf("bytes=%d-", len(body)-4)) + if w.Code != http.StatusPartialContent || w.Body.String() != body[len(body)-4:] { + t.Errorf("suffix range: %d %q", w.Code, w.Body.String()) + } + + // Past the end is a 416 with the length, not a 200 of the whole file. + w = req(fmt.Sprintf("bytes=%d-%d", len(body)+10, len(body)+20)) + if w.Code != http.StatusRequestedRangeNotSatisfiable { + t.Errorf("unsatisfiable range: status %d, want 416", w.Code) + } + + // A malformed Range is refused with 416 - net/http's choice, not this + // server's; it is recorded here so a change in behaviour is noticed. + if w = req("bytes=abc"); w.Code != http.StatusRequestedRangeNotSatisfiable { + t.Errorf("malformed Range: status %d, want 416", w.Code) + } + // An unknown range unit is refused the same way. RFC 7233 says to + // ignore it and serve the whole representation; net/http does not, and + // nothing this archive serves sends one. Recorded, not worked around. + if w = req("pages=1-2"); w.Code != http.StatusRequestedRangeNotSatisfiable { + t.Errorf("unknown range unit: status %d, want net/http's 416", w.Code) + } +} + +// Conditional requests: a document file and a static asset both carry +// validators, so a reader's second visit is a 304 instead of a re-download. +func TestHTTPConditional(t *testing.T) { + buildFixture(t) + // A document file comes off disk and carries a modification time. + const doc = "/files/prism-slides/prism-slides.pdf" + lm := get(t, doc).Header().Get("Last-Modified") + if lm == "" { + t.Fatalf("%s: no Last-Modified", doc) + } + r := httptest.NewRequest("GET", doc, nil) + r.Header.Set("If-Modified-Since", lm) + w := httptest.NewRecorder() + fixtureMux.ServeHTTP(w, r) + if w.Code != http.StatusNotModified || w.Body.Len() != 0 { + t.Errorf("%s with If-Modified-Since: status %d, %d bytes", doc, w.Code, w.Body.Len()) + } + + // An embedded asset has no modification time - embed.FS zeroes it - so + // it carries a content ETag instead. Without one, every page view + // re-downloads the stylesheet on an unstamped build. + for _, asset := range []string{"/static/style.css", "/static/main.js"} { + first := get(t, asset) + tag := first.Header().Get("ETag") + if tag == "" { + t.Errorf("%s: no ETag", asset) + continue + } + r := httptest.NewRequest("GET", asset, nil) + r.Header.Set("If-None-Match", tag) + w := httptest.NewRecorder() + fixtureMux.ServeHTTP(w, r) + if w.Code != http.StatusNotModified || w.Body.Len() != 0 { + t.Errorf("%s with If-None-Match: status %d, %d bytes (want an empty 304)", + asset, w.Code, w.Body.Len()) + } + // A different asset's tag must not satisfy it. + r = httptest.NewRequest("GET", asset, nil) + r.Header.Set("If-None-Match", `"0000000000000000"`) + w = httptest.NewRecorder() + fixtureMux.ServeHTTP(w, r) + if w.Code != 200 { + t.Errorf("%s with a stale ETag: status %d, want the asset", asset, w.Code) + } + } +} + +// Method handling: the routes declare GET or POST, so anything else is a +// 405 from the mux rather than a handler running on the wrong verb. +func TestHTTPMethods(t *testing.T) { + buildFixture(t) + for _, c := range []struct { + method, path string + want int + }{ + // Methods the archive never answers stop at the wrapper. + {"PUT", "/api/docs", http.StatusMethodNotAllowed}, + {"DELETE", "/doc/prism-slides", http.StatusMethodNotAllowed}, + {"OPTIONS", "/", http.StatusMethodNotAllowed}, + {"TRACE", "/browse", http.StatusMethodNotAllowed}, + // A supported method on the wrong route is a 404: the catch-all + // route claims every path, which is what keeps unknown paths on the + // archive's own 404 page, and it claims them for POST too. + {"POST", "/browse", http.StatusNotFound}, + {"GET", "/preferences/save", http.StatusNotFound}, + {"GET", "/pin/prism-slides", http.StatusNotFound}, + } { + r := httptest.NewRequest(c.method, c.path, nil) + w := httptest.NewRecorder() + fixtureMux.ServeHTTP(w, r) + if w.Code != c.want { + t.Errorf("%s %s: status %d, want %d", c.method, c.path, w.Code, c.want) + } + } +} + +// Odd but legal requests must not break anything: repeated parameters, +// unknown parameters, a very long path, an empty query. +func TestHTTPOddRequests(t *testing.T) { + buildFixture(t) + for _, c := range []struct { + path string + want int + }{ + {"/browse?sort=title&sort=date", 200}, + {"/browse?unknown=1&another=2", 200}, + {"/search?q=a&q=b&do_search=1", 200}, + {"/browse/program/" + strings.Repeat("x", 2000), 200}, // an empty page, not a crash + {"/doc/" + strings.Repeat("x", 2000), 404}, + {"/browse?", 200}, + {"/search?do_search=1&fqv=&fqv=&fqv=", 200}, + } { + if w := get(t, c.path); w.Code != c.want { + t.Errorf("%.60s: status %d, want %d", c.path, w.Code, c.want) + } + } +} + +// With -accel-files the file is nginx's to send: the handler answers only +// the files it would have served itself, with the redirect and no body. +func TestAccelFiles(t *testing.T) { + buildFixture(t) + accelPrefix = "/_files/" + defer func() { accelPrefix = "" }() + + w := get(t, "/files/prism-slides/prism-slides.pdf") + rel, err := filepath.Rel(dataDir, filepath.Join(corpus().docIndex["prism-slides"].Dir, "prism-slides.pdf")) + if err != nil { + t.Fatal(err) + } + if got, want := w.Header().Get("X-Accel-Redirect"), "/_files/"+filepath.ToSlash(rel); got != want { + t.Errorf("X-Accel-Redirect %q, want %q", got, want) + } + if w.Code != http.StatusOK || w.Body.Len() != 0 { + t.Errorf("accel answer: status %d, %d body bytes, want an empty 200", w.Code, w.Body.Len()) + } + if w.Header().Get("Cache-Control") == "" { + t.Error("accel answer carries no Cache-Control for nginx to pass on") + } + + for _, p := range []string{"/files/prism-slides/zz-stray.pdf", "/files/prism-slides/../glossary.json", "/files/no-such-doc/x.pdf"} { + w := get(t, p) + if w.Header().Get("X-Accel-Redirect") != "" || w.Code == http.StatusOK { + t.Errorf("%s: status %d, X-Accel-Redirect %q - a file the server does not serve was handed to nginx", p, w.Code, w.Header().Get("X-Accel-Redirect")) + } + } +} + +func TestAccelPathEscapes(t *testing.T) { + old, oldPrefix := dataDir, accelPrefix + dataDir, accelPrefix = "/srv/data", "/_files/" + defer func() { dataDir, accelPrefix = old, oldPrefix }() + if got, ok := accelPath("/srv/data/FOIA/a doc/50%?.pdf"); !ok || got != "/_files/FOIA/a%20doc/50%25%3F.pdf" { + t.Errorf("accelPath = %q, %v", got, ok) + } + if got, ok := accelPath("/srv/other/x.pdf"); ok { + t.Errorf("a file outside the data directory mapped to %q", got) + } +} diff --git a/tests/ids_test.go b/tests/ids_test.go new file mode 100644 index 0000000..f00a615 --- /dev/null +++ b/tests/ids_test.go @@ -0,0 +1,99 @@ +package main + +import ( + "bytes" + "log" + "os" + "path/filepath" + "strings" + "testing" +) + +// The /explore/ids table: one row per document in id order, versions nested +// under their document, and a checksum file that sha256sum -c can read from +// the corpus root. +func TestBuildIDRows(t *testing.T) { + oldDocs, oldDir := corpus().archive.Docs, dataDir + t.Cleanup(func() { + // The id tables are package state the routes serve: rebuild them + // from the restored corpus, or the fake rows below leak into + // every later test. + corpus().archive.Docs, dataDir = oldDocs, oldDir + corpus().buildIDRows() + }) + + dataDir = "/corpus" + corpus().archive.Docs = []Document{ + { + ID: "zebra", Dir: filepath.Join(dataDir, "FOIA", "zebra"), Collection: "foia", + PDFFile: "zebra.pdf", PDFSize: 2 << 20, Pages: 3, + }, + { + ID: "alpha", Dir: filepath.Join(dataDir, "alpha"), Collection: "news", + PDFFile: "alpha.pdf", PDFSize: 1500, Pages: 1, + PDFSha256: "abc123", AkaID: []AkaEntry{{ID: "alpha-lemonde"}}, + Versions: []DocVersion{{Slug: "entier", PDFFile: "copy.pdf", PDFSize: 4096, Images: []string{"a", "b"}, SHA: "def456"}}, + }, + } + corpus().buildIDRows() + + if len(corpus().idRows) != 2 || corpus().idRows[0].ID != "alpha" || corpus().idRows[1].ID != "zebra" { + t.Fatalf("rows out of order: %v", corpus().idRows) + } + if got := corpus().idRows[0].Size; got != "1 KB" { + t.Errorf("size %q, want 1 KB", got) + } + if v := corpus().idRows[0].Versions; len(v) != 1 || v[0].Pages != 2 || v[0].Size != "4 KB" || v[0].SHA != "def456" { + t.Errorf("version row %+v", v) + } + if len(corpus().idRows[1].Versions) != 0 { + t.Errorf("zebra has no versions, got %+v", corpus().idRows[1].Versions) + } + + // sha256sum -c format: checksum, two spaces, path from the corpus root, + // a document's versions after it. Files without a checksum are absent. + if corpus().idChecksums != "abc123 alpha/alpha.pdf\ndef456 alpha/versions/entier/copy.pdf\n" { + t.Errorf("checksums %q", corpus().idChecksums) + } +} + +func TestHumanSize(t *testing.T) { + for n, want := range map[int64]string{0: "", -1: "", 900: "900 B", 2048: "2 KB", + 3 << 20: "3.0 MB", 5 << 30: "5.00 GB"} { + if got := humanSize(n); got != want { + t.Errorf("humanSize(%d) = %q, want %q", n, got, want) + } + } + if !strings.HasSuffix(humanSize(1<<20+1<<19), " MB") { + t.Errorf("1.5 MiB should render in MB") + } +} + +// A version's PDF checksum comes from the metadata's +// "versions"//pdf_sha256 only; a version without one is warned about +// and has no line in ids.txt. +func TestVersionChecksums(t *testing.T) { + root := assemblyCorpus(t, "") + writeAssemblyDoc(t, root, "listed", map[string]interface{}{ + "title": "Listed", + "versions": map[string]interface{}{"copy": map[string]interface{}{"pdf_sha256": " ABC999 "}}, + }, []string{"page-1.jpg"}, map[string][]string{"copy": {"page-1.jpg"}}) + var logged bytes.Buffer + log.SetOutput(&logged) + withCorpus(t, root) + log.SetOutput(os.Stderr) + + if v := corpus().docIndex["listed"].VersionFor("copy"); v == nil || v.SHA != "abc999" { + t.Errorf("metadata version checksum: %+v, want abc999 from the metadata", v) + } + if v := corpus().docIndex["deck"].VersionFor("fuller"); v == nil || v.SHA != "" { + t.Errorf("a version without pdf_sha256 in the metadata got a checksum: %+v", v) + } + if !strings.Contains(logged.String(), "deck: versions/fuller has no pdf_sha256") { + t.Errorf("no warning for the version without a checksum:\n%s", logged.String()) + } + if !strings.Contains(corpus().idChecksums, "abc999 listed/versions/copy/copy.pdf\n") || + strings.Contains(corpus().idChecksums, "deck/versions/fuller/") { + t.Errorf("ids.txt version lines:\n%s", corpus().idChecksums) + } +} diff --git a/tests/image_sizes_test.go b/tests/image_sizes_test.go new file mode 100644 index 0000000..a10abe5 --- /dev/null +++ b/tests/image_sizes_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "bytes" + "log" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPageImagesCarryTheirSize(t *testing.T) { + for _, path := range []string{ + "/doc/prism-slides", + "/doc/prism-slides/pages", + "/doc/prism-slides/versions/lemonde", + "/doc/prism-slides/versions/lemonde/pages", + } { + body := get(t, path).Body.String() + if n := strings.Count(body, `page-1.jpg" width="4" height="3"`); n < 2 { + t.Errorf("%s: %d page-1 images with their size, want the page and its full-screen view", path, n) + } + } +} + +func TestAssemblySlidesCarryTheirSize(t *testing.T) { + root := assemblyCorpus(t, `[{"id": "sized", "title": "Sized", "slides": [{"doc": "deck", "pages": "1"}]}]`) + withCorpus(t, root) + for _, path := range []string{"/explore/assemblies", "/explore/assemblies/sized"} { + if body := get(t, path).Body.String(); !strings.Contains(body, `page-1.jpg" width="4" height="3"`) { + t.Errorf("%s: slide image without its size", path) + } + } +} + +func TestUnreadablePageImageWarns(t *testing.T) { + root := assemblyCorpus(t, "") + writeAssemblyDoc(t, root, "broken", map[string]interface{}{"title": "Broken"}, []string{"page-1.jpg"}, nil) + writeFixtureFile(t, filepath.Join(root, "broken", "images", "page-1.jpg"), "not an image") + var logged bytes.Buffer + log.SetOutput(&logged) + withCorpus(t, root) + log.SetOutput(os.Stderr) + + if !strings.Contains(logged.String(), "page image unreadable, served without its size: broken/images/page-1.jpg") { + t.Errorf("no warning for the unreadable image:\n%s", logged.String()) + } + body := get(t, "/doc/broken").Body.String() + if !strings.Contains(body, `.*?(\d+)`) + reBarLink = regexp.MustCompile(`]*>\s*(\d+)`) + reTotal = regexp.MustCompile(`([\d,]+)\s+results?`) + reListTotal = regexp.MustCompile(`([\d,]+)\s+documents?`) +) + +// resultTotal reads the count a /search or a list page reports. +func resultTotal(t *testing.T, path string, cookies ...*http.Cookie) int { + t.Helper() + body := get(t, path, cookies...).Body.String() + re := reTotal + if !strings.HasPrefix(path, "/search") { + re = reListTotal + } + m := re.FindStringSubmatch(body) + if m == nil { + t.Fatalf("%s: no result count in the page", path) + } + n, err := strconv.Atoi(strings.ReplaceAll(m[1], ",", "")) + if err != nil { + t.Fatalf("%s: %v", path, err) + } + return n +} + +// followLinks checks every (link, count) pair a page prints, as the reader +// with these cookies sees it. +func followLinks(t *testing.T, page string, re *regexp.Regexp, what string, cookies ...*http.Cookie) int { + t.Helper() + body := get(t, page, cookies...).Body.String() + pairs := re.FindAllStringSubmatch(body, -1) + for _, p := range pairs { + link := html.UnescapeString(p[1]) + shown, _ := strconv.Atoi(p[2]) + if got := resultTotal(t, link, cookies...); got != shown { + t.Errorf("%s: %s says %d, its link returns %d\n %s", page, what, shown, got, link) + } + } + return len(pairs) +} + +// A category page lists exactly as many documents as its facet counts. +func TestInvariantFacetCountsMatchPages(t *testing.T) { + buildFixture(t) + checked := 0 + for ci := range categories { + c := &categories[ci] + for val, want := range c.Facet(corpus()) { + if got := len(corpus().categoryMatches(c.Slug, val)); got != want { + t.Errorf("%s=%q: facet says %d, the page lists %d", c.Slug, val, want, got) + } + checked++ + } + } + if checked == 0 { + t.Fatal("no facet values in the fixture") + } +} + +// Refine links and histogram bars open searches that must return what they +// promise - the two places where a count and its click are computed by +// different code paths. +func TestInvariantRefineAndBarCounts(t *testing.T) { + pages := []string{ + "/browse/agency/NSA", + "/browse/program/PRISM", + "/browse/classification/TOP%20SECRET", + "/search?q=prism&do_search=1", + "/search?do_search=1&fqv=NSA&fqf=CR&fqc=and", + } + facets, bars := 0, 0 + for _, p := range pages { + facets += followLinks(t, p, reFacetLink, "refine link") + bars += followLinks(t, p, reBarLink, "histogram bar") + } + if facets == 0 || bars == 0 { + t.Fatalf("nothing exercised: %d refine links, %d bars", facets, bars) + } + t.Logf("checked %d refine links and %d histogram bars", facets, bars) +} + +// The home page is scoped to the reported collection while the category +// pages follow the reader's scope, so its bars and lists link to those +// pages with the collection named. Every one of them has to be the number +// its click returns. +func TestInvariantHomeCounts(t *testing.T) { + body := get(t, "/").Body.String() + pairs := regexp.MustCompile(`[^<]*(\d+)`).FindAllStringSubmatch(body, -1) + lists := regexp.MustCompile(`
  • [^<]* (\d+)`).FindAllStringSubmatch(body, -1) + if len(lists) < 5 { + t.Fatalf("only %d counted list links on the home page - the stat lists should link to their browse pages", len(lists)) + } + pairs = append(pairs, lists...) + pairs = append(pairs, regexp.MustCompile(`]*>\s*([A-Z]+)\s*]*>\(?(\d+)\)?`) + for _, m := range re.FindAllStringSubmatch(body, -1) { + link := html.UnescapeString(m[1]) + shown, _ := strconv.Atoi(m[3]) + if got := resultTotal(t, link); got != shown { + t.Errorf("type pill %s says %d, its link returns %d (%s)", m[2], shown, got, link) + } + } +} + +// Everything the sitemap advertises must exist: those URLs are built in Go, +// away from the templates that normally escape ids. +func TestInvariantAdvertisedURLsResolve(t *testing.T) { + for _, src := range []string{"/sitemap.xml"} { + body := get(t, src).Body.String() + locs := regexp.MustCompile(`([^<]+)|]+href="([^"]+)"`).FindAllStringSubmatch(body, -1) + if len(locs) == 0 { + t.Fatalf("%s advertises nothing", src) + } + seen := 0 + for _, m := range locs { + raw := m[1] + if raw == "" { + raw = m[2] + } + u, err := url.Parse(html.UnescapeString(raw)) + if err != nil || u.Path == "" { + continue + } + p := u.Path + if u.RawQuery != "" { + p += "?" + u.RawQuery + } + if w := get(t, p); w.Code != 200 { + t.Errorf("%s advertises %s, which answers %d", src, p, w.Code) + } + seen++ + } + t.Logf("%s: %d URLs resolve", src, seen) + } +} + +// The API's pagination envelope has to describe the page it returned, at +// every page size, and per_page=all is the export mode that skips snippets. +func TestInvariantAPIPagination(t *testing.T) { + type envelope struct { + Total int `json:"total"` + Page int `json:"page"` + PerPage int `json:"per_page"` + TotalPages int `json:"total_pages"` + Omitted bool `json:"snippets_omitted"` + Docs []struct { + ID string `json:"id"` + } `json:"docs"` + Results []struct { + ID string `json:"id"` + Snippet string `json:"snippet"` + } `json:"results"` + } + read := func(path string) envelope { + var e envelope + w := get(t, path) + if w.Code != 200 { + t.Fatalf("%s: status %d", path, w.Code) + } + if err := json.Unmarshal(w.Body.Bytes(), &e); err != nil { + t.Fatalf("%s: %v", path, err) + } + return e + } + + for _, path := range []string{"/api/docs", "/api/docs?per_page=1", "/api/docs?per_page=all", + "/api/search?q=prism", "/api/search?q=prism&per_page=1", "/api/search?q=prism&per_page=all"} { + e := read(path) + n := len(e.Docs) + len(e.Results) + switch { + case e.Page < 1: + t.Errorf("%s: page %d", path, e.Page) + case e.PerPage < 1: + t.Errorf("%s: per_page %d", path, e.PerPage) + case n > e.PerPage: + t.Errorf("%s: returned %d rows for a page of %d", path, n, e.PerPage) + case e.TotalPages < 1: + t.Errorf("%s: total_pages %d", path, e.TotalPages) + } + if want := (e.Total + e.PerPage - 1) / e.PerPage; e.Total > 0 && e.TotalPages != want { + t.Errorf("%s: total_pages %d, want %d for %d/%d", path, e.TotalPages, want, e.Total, e.PerPage) + } + } + + // An out-of-range page clamps rather than 500s or returns junk. + if e := read("/api/docs?p=9999"); e.Page != e.TotalPages { + t.Errorf("p=9999 gave page %d of %d", e.Page, e.TotalPages) + } + // Export mode returns everything and says why there are no snippets. + all := read("/api/search?q=prism&per_page=all") + if !all.Omitted || len(all.Results) != all.Total { + t.Errorf("per_page=all: omitted=%v, %d of %d results", all.Omitted, len(all.Results), all.Total) + } + for _, r := range all.Results { + if r.Snippet != "" { + t.Errorf("per_page=all returned a snippet for %s", r.ID) + } + } + // A normal page does carry them. + paged := read("/api/search?q=prism") + if paged.Omitted { + t.Error("a paged search should not omit snippets") + } +} + +// Snippet work is bounded by maxSnippetDocs, not by the page size, and the +// page says so when the two differ. +func TestInvariantSnippetCap(t *testing.T) { + buildFixture(t) + doc := corpus().docIndex["prism-slides"] + if doc == nil { + t.Fatal("fixture document missing") + } + docs := make([]*Document, maxSnippetDocs+5) + for i := range docs { + docs[i] = doc + } + page, capped := decorateResults(docs, []string{"collection"}, []string{"TE"}, []string{"and"}) + if !capped { + t.Error("a page past the budget should report itself capped") + } + withSnippets := 0 + for i := range page { + if len(page[i].Snippets) > 0 { + withSnippets++ + } + if page[i].Title == "" { + t.Fatalf("row %d lost its title highlighting", i) + } + } + if withSnippets != maxSnippetDocs { + t.Errorf("%d rows got snippets, want %d", withSnippets, maxSnippetDocs) + } +} diff --git a/tests/loader_test.go b/tests/loader_test.go new file mode 100644 index 0000000..e142823 --- /dev/null +++ b/tests/loader_test.go @@ -0,0 +1,166 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// What the loader does with a corpus that is wrong in the ways corpora are +// wrong in practice. Every case must leave the rest of the archive intact - +// a broken document is skipped, never a reason to serve nothing - and say +// so in the tally that -check reads. +func TestLoaderRejectsBadDocuments(t *testing.T) { + root := t.TempDir() + write := func(path, content string) { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + good := func(id string) { + write(filepath.Join(root, id, "metadata.json"), `{"title": "`+id+`", "type": "REPORTED"}`) + write(filepath.Join(root, id, id+".pdf"), "%PDF-1.4") + } + good("keeper-one") + good("keeper-two") + + // No PDF: this is an archive OF the PDFs, so the document is skipped. + write(filepath.Join(root, "no-pdf", "metadata.json"), `{"title": "No PDF"}`) + // Invalid JSON: skipped, everything else still loads. + write(filepath.Join(root, "bad-json", "metadata.json"), `{"title": "oops"`) + write(filepath.Join(root, "bad-json", "x.pdf"), "%PDF-1.4") + // aka_id that is not a list of objects: refused, not silently half-read. + write(filepath.Join(root, "bad-aka", "metadata.json"), `{"title": "Aka", "aka_id": "other-id"}`) + write(filepath.Join(root, "bad-aka", "x.pdf"), "%PDF-1.4") + // A directory that is not a document at all: skipped without a warning. + write(filepath.Join(root, "notes", "README.txt"), "just notes") + // A directory holding a PDF and nothing else: the shape worth warning + // about, since it looks like a document nobody described. + write(filepath.Join(root, "loose", "orphan.pdf"), "%PDF-1.4") + + withCorpus(t, root) + clean := loadWarnings + + // A document in the SIDtoday archive that is not typed SIDTODAY is on + // no browse tab: served, and warned about. + write(filepath.Join(root, "SIDtoday-archive", "mistyped", "metadata.json"), `{"title": "Mistyped", "type": "REPORTED"}`) + write(filepath.Join(root, "SIDtoday-archive", "mistyped", "x.pdf"), "%PDF-1.4") + loadData() + if corpus().docIndex["mistyped"] == nil || loadWarnings != clean+1 { + t.Errorf("a REPORTED document in SIDtoday-archive: loaded %v, %d warnings, want it loaded with %d", corpus().docIndex["mistyped"] != nil, loadWarnings, clean+1) + } + if err := os.RemoveAll(filepath.Join(root, "SIDtoday-archive")); err != nil { + t.Fatal(err) + } + loadData() + + if len(corpus().archive.Docs) != 2 { + ids := make([]string, 0, len(corpus().archive.Docs)) + for _, d := range corpus().archive.Docs { + ids = append(ids, d.ID) + } + t.Fatalf("loaded %v, want the two good documents", ids) + } + if loadErrors < 3 { // no-pdf, bad-aka and bad JSON are errors: not served + t.Errorf("loadErrors = %d, want at least 3", loadErrors) + } + if loadWarnings < 1 { // the loose PDF directory: served nothing, lost nothing + t.Errorf("loadWarnings = %d, want at least 1", loadWarnings) + } + for _, id := range []string{"no-pdf", "bad-json", "bad-aka", "notes", "loose"} { + if corpus().docIndex[id] != nil { + t.Errorf("%s should not have loaded", id) + } + } + // The archive is coherent: facets, search and the sitemap all see the + // same two documents. + if got := corpus().archive.CollectionCounts[collectionNews]; got != 2 { + t.Errorf("news collection count %d, want 2", got) + } + if got, _ := corpus().executeSearch(searchParams{Queries: []string{""}, Fields: []string{"TE"}, Conjs: []string{"and"}}); len(got) != 2 { + t.Errorf("match-all search found %d, want 2", len(got)) + } +} + +// An empty corpus is a corpus: the server loads, every page renders, and +// nothing divides by zero. +func TestLoaderEmptyCorpus(t *testing.T) { + withCorpus(t, t.TempDir()) + if len(corpus().archive.Docs) != 0 { + t.Fatalf("loaded %d documents from an empty directory", len(corpus().archive.Docs)) + } + for _, p := range []string{"/", "/browse", "/search?q=x&do_search=1", "/explore", "/explore/ids", + "/articles", "/api/docs", "/api/facets", "/sitemap.xml"} { + if w := get(t, p); w.Code != 200 { + t.Errorf("%s on an empty corpus: %d", p, w.Code) + } + } +} + +// Values are tidied before anything reads them, and a copy the loader skips +// leaves no trace - not even a claim on its ID. +func TestLoaderTidiesBeforeUse(t *testing.T) { + root := t.TempDir() + write := func(path, content string) { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + // FOIA/ sorts before the flat root, so the PDF-less copy is seen first. + write(filepath.Join(root, "FOIA", "dup-doc", "metadata.json"), `{"title": "Shadow"}`) + write(filepath.Join(root, "dup-doc", "metadata.json"), `{"title": "Real", "type": "REPORTED"}`) + write(filepath.Join(root, "dup-doc", "x.pdf"), "%PDF-1.4") + + write(filepath.Join(root, "padded", "metadata.json"), `{ + "title": "Padded", "type": "REPORTED", + "document": {"creator": " NSA ", "series": {"name": "Series X ", "part": 1}}, + "publication": [{"publisher": "Le Monde", "date_published": "June 2014", "version": "entier"}] + }`) + write(filepath.Join(root, "padded", "x.pdf"), "%PDF-1.4") + write(filepath.Join(root, "padded", "versions", "entier", "entier.pdf"), "%PDF-1.4") + + withCorpus(t, root) + + if d := corpus().docIndex["dup-doc"]; d == nil || d.Title != "Real" { + t.Errorf("dup-doc = %+v, want the copy with a PDF", d) + } + d := corpus().docIndex["padded"] + if d == nil { + t.Fatal("padded not loaded") + } + if d.DocMeta.Creator != "NSA" || d.DocMeta.Series.Name != "Series X" { + t.Errorf("creator %q, series %q: want both trimmed", d.DocMeta.Creator, d.DocMeta.Series.Name) + } + // The malformed date is dropped everywhere, the version label included. + if v := d.VersionFor("entier"); v == nil || v.Label != "Le Monde" { + t.Errorf("version label %+v, want \"Le Monde\" without the invalid date", v) + } +} + +// Two page images with one page number show that page twice; the loader +// says so instead of linking changed pages to whichever sorted first. +func TestLoadPageImagesWarnsOnDuplicatePageNumbers(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "images"), 0o755); err != nil { + t.Fatal(err) + } + for _, name := range []string{"page-1.jpg", "page-2.jpg", "page-02.png"} { + if err := os.WriteFile(filepath.Join(dir, "images", name), nil, 0o644); err != nil { + t.Fatal(err) + } + } + before := loadWarnings + imgs := loadPageImages(dir, "doc") + if len(imgs) != 3 || imgs[0] != "page-1.jpg" { + t.Errorf("images %v, want all three in page order", imgs) + } + if loadWarnings != before+1 { + t.Errorf("%d warnings for one duplicated page number, want 1", loadWarnings-before) + } +} diff --git a/tests/main_test.go b/tests/main_test.go new file mode 100644 index 0000000..365fcbb --- /dev/null +++ b/tests/main_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" +) + +// TestFileWriteBudget: a download slower than the server's WriteTimeout still +// completes, because handleFiles replaces that page-sized budget with +// fileWriteBudget. The control - the same slow download with the budget cut +// short - must be truncated, or the test would not be exercising the deadline +// at all. +func TestFileWriteBudget(t *testing.T) { + const size = 64 << 20 // well past the loopback socket buffers + path := filepath.Join(t.TempDir(), "big.pdf") + if err := os.WriteFile(path, bytes.Repeat([]byte("snowchive"), size/9+1)[:size], 0o644); err != nil { + t.Fatal(err) + } + old := fileWriteBudget + defer func() { fileWriteBudget = old }() + + // The same two lines handleFiles uses, over a server whose page budget is + // far shorter than the download takes. + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = http.NewResponseController(w).SetWriteDeadline(time.Now().Add(fileWriteBudget)) + http.ServeFile(w, r, path) + })) + ts.Config.WriteTimeout = 300 * time.Millisecond + ts.Start() + defer ts.Close() + + // fetch reads 1 MB at a time with a pause after each chunk, returning the + // bytes received before the body ended. + fetch := func(pause time.Duration) (int64, time.Duration) { + start := time.Now() + resp, err := http.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var got int64 + buf := make([]byte, 1<<20) + for { + n, err := io.ReadFull(resp.Body, buf) + got += int64(n) + if err != nil { + return got, time.Since(start) + } + time.Sleep(pause) + } + } + + fileWriteBudget = 30 * time.Second + got, elapsed := fetch(40 * time.Millisecond) + if got != size { + t.Errorf("slow download got %d of %d bytes", got, size) + } + if elapsed < 2*ts.Config.WriteTimeout { + t.Errorf("download took %v: too fast to prove it outlasted the %v WriteTimeout", + elapsed, ts.Config.WriteTimeout) + } + + fileWriteBudget = 200 * time.Millisecond + if got, _ := fetch(40 * time.Millisecond); got >= size { + t.Errorf("control: a %v budget did not cut the download (got all %d bytes)", + fileWriteBudget, got) + } +} diff --git a/tests/mdparser_test.go b/tests/mdparser_test.go new file mode 100644 index 0000000..7e8ed14 --- /dev/null +++ b/tests/mdparser_test.go @@ -0,0 +1,145 @@ +package main + +import "testing" + +// Emphasis follows CommonMark's flanking rules, so the loose asterisks and +// underscores of OCR - footnote marks, flattened bullets, star pages, +// form blanks - stay text, and real emphasis nests properly. +func TestParseInlineEmphasis(t *testing.T) { + for in, want := range map[string]string{ + // Real emphasis. + "*it* and **bold**": "it and bold", + "***bold italic***": "bold italic", + "**2** | **61**": "2 | 61", + "An *n*-clique": "An n-clique", + "*a **b** c*": "a b c", + "**bold *nested* x**": "bold nested x", + "*`code`*": "code", + "_em_ and __init__.py": "em and init.py", + `"On Language,"* a *New York Times*`: ""On Language,"* a New York Times", + // A closer takes the nearest opener, not the first one on the line. + "Inc.*, 321 F.3d; see also *North v. Ashcroft*, 308": "Inc.*, 321 F.3d; see also North v. Ashcroft, 308", + // Not emphasis. + "PINWALE (Sweet* and Sour* partitions)": "PINWALE (Sweet* and Sour* partitions)", + "NRO*, CIA, SCEs* and field sites": "NRO*, CIA, SCEs* and field sites", + "works with: * NCSC * TAO": "works with: * NCSC * TAO", + "*** Currently Inactive ***": "*** Currently Inactive ***", + "a Driving Force ******": "a Driving Force ******", + "2013 WL 5741573, at *3-*9": "2013 WL 5741573, at *3-*9", + "2 * 3 * 4": "2 * 3 * 4", + "snake_case_name": "snake_case_name", + "D/DIR____,IG_____,OGC___": "D/DIR____,IG_____,OGC___", + "SO ORDERED, this _____ day of July": "SO ORDERED, this _____ day of July", + "datastore
    _ target user
    _ time": "datastore
    _ target user
    _ time", + `\*not em\*`: "*not em*", + } { + if got := parseInline(in); got != want { + t.Errorf("parseInline(%q)\n got %q\nwant %q", in, got, want) + } + } +} + +// A list never mixes bullets and numbers: a number kept in a