first commit
This commit is contained in:
commit
0d90363116
6510 changed files with 679146 additions and 0 deletions
|
|
@ -0,0 +1,116 @@
|
|||
# SSO GHOSTMACHINE Analytics
|
||||
|
||||
hadoop
|
||||
[REDACTED]
|
||||
|
||||
SiteStore Infrastructure & Analytics
|
||||
|
||||
TOP SECRET//COMINT//NOFORN
|
||||
|
||||
---
|
||||
|
||||
## Hitting The Wall
|
||||
|
||||
Fascia Limit?
|
||||
|
||||
```
|
||||
Tuskattire Volume Totals
|
||||
5,000M
|
||||
4,000M
|
||||
3,000M
|
||||
2,000M
|
||||
1,000M
|
||||
0M
|
||||
09-DEC-10
|
||||
11-DEC-10
|
||||
13-DEC-10
|
||||
15-DEC-10
|
||||
17-DEC-10
|
||||
19-DEC-10
|
||||
22-DEC-10
|
||||
24-DEC-10
|
||||
26-DEC-10
|
||||
28-DEC-10
|
||||
30-DEC-10
|
||||
01-JAN-11
|
||||
03-JAN-11
|
||||
05-JAN-11
|
||||
|
||||
CERF
|
||||
POPTOP
|
||||
DRT
|
||||
SIGDASYS
|
||||
AST128
|
||||
JUGGERNAUT
|
||||
SEADIVER
|
||||
```
|
||||
|
||||
|
||||
TOP SECRET//COMINT//NOFORN
|
||||
|
||||
---
|
||||
|
||||
## Data at a Larger Scale
|
||||
|
||||
27704057147161 hdfs://r05sv24:9000/in/asdf
|
||||
27827753336125 hdfs://r05sv24:9000/in/fascia
|
||||
|
||||
- Terabytes are available to query in the cloud, not always available in corporate databases
|
||||
- Real value comes in being able to correlate and enrich with multiple types of data
|
||||
|
||||
TOP SECRET//COMINT//NOFORN
|
||||
|
||||
---
|
||||
|
||||
```
|
||||
SS7 Summary
|
||||
Current SIGAD | Search | Info
|
||||
Current SIGADs
|
||||
SIGAD OPC/DPC Pairs
|
||||
US-3127 671
|
||||
US-3145 381
|
||||
US-3167 1313
|
||||
US-3180 2022
|
||||
US-3190 135
|
||||
US-3237 1816
|
||||
US-3261 5
|
||||
US-3273 91
|
||||
US-3293 20
|
||||
US-3300 39
|
||||
US-3311 1
|
||||
US-983 27
|
||||
US-990 860
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
US-3300
|
||||
easMap/data.xml
|
||||
[REDACTED]
|
||||
DYNAMIC PAGE - HIGHEST CLASSIFICATION IS
|
||||
TOP SECRET//COMINT//NOFORN
|
||||
SMA Tools Blog | WOC | SSO Dashboard| BROKENRECORD | DNR Deduplicator | Mission Support Hub | REDHAWK
|
||||
```
|
||||
|
||||
TOP SECRET//COMINT//NOFORN
|
||||
92
data/EFF_-_20131210-wapo-ghost_machine/metadata.json
Normal file
92
data/EFF_-_20131210-wapo-ghost_machine/metadata.json
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
{
|
||||
"title": "SSO GHOSTMACHINE Analytics",
|
||||
"type": "REPORTED",
|
||||
"description": "This document describes the NSA’s cloud analytics platform GHOSTMACHINE, used by Special Source Operations (SSO) to process and analyze massive volumes of data (e.g., from FASCIA) using Hadoop-based infrastructure. It’s essentially the backbone system for querying, correlating, and enriching surveillance data at scale — not just a single analytic, but the analytic/cloud environment powering many programs.",
|
||||
"long_description": "This top secret NSA Special Source Operations presentation introduces GHOSTMACHINE, the agency's cloud analytics platform built on Hadoop distributed computing infrastructure, designed to overcome the limitations of traditional corporate databases by enabling analysts to query and correlate terabytes of metadata at scale. The platform aggregates data from multiple SSO collection feeds including FASCIA (metadata repository), TUSKATTIRE (call data record normalization), and other sources identified by codenames including CERF, POPTOP, DRT, SIGDASYS, AST128, JUGGERNAUT, and SEADIVER. A chart titled \"Hitting The Wall\" displays TUSKATTIRE volume totals from 9 December 2010 to 5 January 2011, with total daily record counts approaching and potentially exceeding 5 billion — suggesting the platform was encountering infrastructure scaling limits at the time of the presentation.\n\nThe document's \"Data at a Larger Scale\" slide illustrates GHOSTMACHINE's Hadoop Distributed File System (HDFS) architecture, showing ASDF-format metadata and FASCIA data stored in distributed nodes, and emphasizes that the platform's real value lies in the ability to correlate and enrich data across multiple collection types simultaneously — a capability not available in traditional corporate database environments. A SIGAD summary table lists active collection sources feeding the platform; per other documents in the Snowden archive, US-3180 (SPINNERET, a RAMPART-A site) contributes the largest number of OPC/DPC pairs at 2,022, followed by US-3237 (SMOKYSINK, another RAMPART-A site) at 1,816 and US-990 (FAIRVIEW) at 860. The interface screenshot reveals associated SSO tools including BROKENRECORD, REDHAWK, a DNR Deduplicator, and Mission Support Hub — suggesting GHOSTMACHINE operates within a broader SSO analytics ecosystem rather than as a standalone platform.",
|
||||
"tags": [
|
||||
"cloud analytics",
|
||||
"cloud computing",
|
||||
"big data",
|
||||
"data storage",
|
||||
"data deduplication",
|
||||
"data correlation",
|
||||
"telecommunications",
|
||||
"metadata processing",
|
||||
"Hadoop",
|
||||
"data infrastructure",
|
||||
"AST 128",
|
||||
"SMA Tools",
|
||||
"web operations center",
|
||||
"SSO Dashboard",
|
||||
"DNR Deduplicator",
|
||||
"mission support",
|
||||
"SSO",
|
||||
"HDFS",
|
||||
"distributed computing",
|
||||
"SIGDASYS",
|
||||
"metadata",
|
||||
"bulk collection",
|
||||
"OPC",
|
||||
"DPC",
|
||||
"call detail records",
|
||||
"infrastructure",
|
||||
"analytic tools"
|
||||
],
|
||||
"document": {
|
||||
"creator": "NSA",
|
||||
"date_created": "",
|
||||
"classification": [
|
||||
"TOP SECRET",
|
||||
"COMINT"
|
||||
],
|
||||
"dissemination": [
|
||||
"NOFORN"
|
||||
],
|
||||
"banner": "TOP SECRET//COMINT//NOFORN",
|
||||
"genre": "slides",
|
||||
"programs": [
|
||||
"GHOSTMACHINE",
|
||||
"FASCIA",
|
||||
"CERF",
|
||||
"POPTOP",
|
||||
"DRT",
|
||||
"SIGDASYS",
|
||||
"JUGGERNAUT",
|
||||
"SEADIVER",
|
||||
"AST128",
|
||||
"TUSKATTIRE",
|
||||
"ASDF",
|
||||
"BROKENRECORD",
|
||||
"REDHAWK",
|
||||
"SSO"
|
||||
],
|
||||
"countries": [],
|
||||
"people": []
|
||||
},
|
||||
"publication": [
|
||||
{
|
||||
"publisher": "The Washington Post",
|
||||
"date_published": "2013-12-10",
|
||||
"title": "New documents show how the NSA infers relationships based on mobile location data",
|
||||
"link": "https://www.washingtonpost.com/news/the-switch/wp/2013/12/10/new-documents-show-how-the-nsa-infers-relationships-based-on-mobile-location-data/",
|
||||
"reporters": [
|
||||
"Soltani, Ashkan",
|
||||
"Gellman, Barton"
|
||||
],
|
||||
"contributors": []
|
||||
}
|
||||
],
|
||||
"aka_id": [
|
||||
{
|
||||
"id": "nsa-ghost-machine"
|
||||
},
|
||||
{
|
||||
"id": "ACLU_-_2013-12-10_Ghost_Machine"
|
||||
},
|
||||
{
|
||||
"id": "2013-12-10_Ghost_Machine"
|
||||
}
|
||||
],
|
||||
"pdf_sha256": "d178b4df88d6aa2d9ed57283a89fb89e80858b3fd96af4733be1b2930ec1f2f2",
|
||||
"pages": 4
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue