first commit
This commit is contained in:
commit
8c4dcae475
89 changed files with 24623 additions and 0 deletions
335
api.go
Normal file
335
api.go
Normal file
|
|
@ -0,0 +1,335 @@
|
|||
// api.go - read-only JSON API mirroring the UI's features.
|
||||
//
|
||||
// Endpoints (all GET):
|
||||
//
|
||||
// /api/docs list documents; params: coll (repeatable:
|
||||
// news, sidtoday-archive, foia), type
|
||||
// (repeatable), sort (any list-page sort: title,
|
||||
// agency, date, pubdate, classification, ...; see
|
||||
// docSorts), order (asc|desc), p, per_page (max
|
||||
// 500, or "all" for exports)
|
||||
// /api/doc/{id} full document: metadata, full text, images,
|
||||
// related IDs, published-together IDs,
|
||||
// version_copies (other published copies: PDF,
|
||||
// page images, transcript, publication indices)
|
||||
// /api/search params: q (simple) OR fqv/fqf/fqc clause
|
||||
// triples (same field codes and value syntax as
|
||||
// the UI: word, "text", =exact, *wildcard*),
|
||||
// plus coll and type (both repeatable, as on
|
||||
// /api/docs; no coll means every collection),
|
||||
// date_doc_start/_end, date_pub_start/_end
|
||||
// (partial END dates are inclusive: 2013 means
|
||||
// through 2013-12-31), sort, order, and the same
|
||||
// p / per_page pagination as /api/docs (default
|
||||
// 50, max 500, "all" for every match without
|
||||
// snippets see apiPageSize). results come
|
||||
// as one flat list, mirroring the search page:
|
||||
// all collections rank equally. a result whose match
|
||||
// is in another published copy of the document
|
||||
// lists that copy in matched_versions (slug,
|
||||
// label, url, snippet)
|
||||
// /api/facets facet maps (whole corpus, like the UI; the
|
||||
// one news-scoped map is news_types) plus
|
||||
// collection counts
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
apiPerPageDefault = 50
|
||||
apiMaxPerPage = 500
|
||||
)
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, v interface{}) {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
log.Printf("API: marshal error: %v", err)
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.Write([]byte(`{"error":"internal error"}`))
|
||||
return
|
||||
}
|
||||
writeJSONBytes(w, status, b)
|
||||
}
|
||||
|
||||
func writeJSONBytes(w http.ResponseWriter, status int, b []byte) {
|
||||
setJSONHeaders(w)
|
||||
w.WriteHeader(status)
|
||||
w.Write(b)
|
||||
}
|
||||
|
||||
func setJSONHeaders(w http.ResponseWriter) {
|
||||
cc := w.Header().Get("Cache-Control")
|
||||
if cc == "" {
|
||||
cc = "public, max-age=60"
|
||||
w.Header().Set("Cache-Control", cc)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
|
||||
if !strings.Contains(cc, "private") {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
}
|
||||
}
|
||||
|
||||
func apiError(w http.ResponseWriter, status int, msg string) {
|
||||
writeJSON(w, status, map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
type apiDocSummary struct {
|
||||
Document
|
||||
URL string `json:"url"`
|
||||
PDF string `json:"pdf"`
|
||||
}
|
||||
|
||||
func docSummary(d Document) apiDocSummary {
|
||||
return apiDocSummary{
|
||||
Document: d,
|
||||
URL: docPath(d.ID),
|
||||
PDF: filePath(d.ID, d.PDFFile),
|
||||
}
|
||||
}
|
||||
|
||||
type apiDocDetail struct {
|
||||
apiDocSummary
|
||||
FullText string `json:"full_text,omitempty"`
|
||||
Images []string `json:"images,omitempty"`
|
||||
Related []string `json:"related,omitempty"`
|
||||
PublishedTogether []string `json:"published_together,omitempty"`
|
||||
VersionCopies []apiVersion `json:"version_copies,omitempty"`
|
||||
}
|
||||
|
||||
type apiSearchResult struct {
|
||||
apiDocSummary
|
||||
Score float64 `json:"score"`
|
||||
Snippet string `json:"snippet,omitempty"`
|
||||
MatchedVersions []apiMatchedVersion `json:"matched_versions,omitempty"`
|
||||
}
|
||||
|
||||
func (co *Corpus) handleAPIDocs(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
|
||||
typeFilter, _ := selectedTypes(q)
|
||||
|
||||
// coll=, repeatable, as /api/search and the pages take it.
|
||||
pool := filterByCollection(co.everyDoc, collectionFilter(q))
|
||||
docs := filterByType(append([]*Document(nil), pool...), typeFilter)
|
||||
|
||||
sortDocs(docs, q.Get("sort"), q.Get("order"))
|
||||
|
||||
perPage := apiPageSize(q)
|
||||
page, _ := strconv.Atoi(q.Get("p"))
|
||||
start, end, page, totalPages := paginate(len(docs), page, perPage)
|
||||
|
||||
out := make([]apiDocSummary, 0, end-start)
|
||||
for _, d := range docs[start:end] {
|
||||
out = append(out, docSummary(*d))
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"total": len(docs),
|
||||
"page": page,
|
||||
"per_page": apiRespPerPage(perPage, len(docs)),
|
||||
"total_pages": totalPages,
|
||||
"docs": out,
|
||||
})
|
||||
}
|
||||
|
||||
func apiPageSize(q url.Values) int {
|
||||
if q.Get("per_page") == "all" {
|
||||
return 0
|
||||
}
|
||||
if v, err := strconv.Atoi(q.Get("per_page")); err == nil && v > 0 {
|
||||
if v > apiMaxPerPage {
|
||||
return apiMaxPerPage
|
||||
}
|
||||
return v
|
||||
}
|
||||
return apiPerPageDefault
|
||||
}
|
||||
|
||||
func apiRespPerPage(perPage, total int) int {
|
||||
if perPage > 0 {
|
||||
return perPage
|
||||
}
|
||||
if total == 0 {
|
||||
return 1
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (co *Corpus) handleAPIDoc(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
doc := co.docIndex[id]
|
||||
if doc == nil {
|
||||
apiError(w, http.StatusNotFound, "document not found")
|
||||
return
|
||||
}
|
||||
|
||||
var pubIDs []string
|
||||
for _, d := range co.publishedTogether(doc) {
|
||||
pubIDs = append(pubIDs, d.ID)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, apiDocDetail{
|
||||
apiDocSummary: docSummary(*doc),
|
||||
FullText: doc.FullText,
|
||||
Images: doc.Images,
|
||||
Related: co.archive.RelatedDocs[id],
|
||||
PublishedTogether: pubIDs,
|
||||
VersionCopies: apiVersions(doc),
|
||||
})
|
||||
}
|
||||
|
||||
func (co *Corpus) handleAPISearch(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
|
||||
sp := readSearchParams(q)
|
||||
if len(sp.Queries) == 0 {
|
||||
apiError(w, http.StatusBadRequest,
|
||||
"missing query: pass ?q=term or fqv/fqf/fqc clause parameters")
|
||||
return
|
||||
}
|
||||
if sp.OverClauses > 0 {
|
||||
apiError(w, http.StatusBadRequest, fmt.Sprintf(
|
||||
"too many clauses: %d (the limit is %d)", sp.OverClauses, maxSearchClauses))
|
||||
return
|
||||
}
|
||||
if sp.OverWords > 0 {
|
||||
apiError(w, http.StatusBadRequest, fmt.Sprintf(
|
||||
"too many words: %d (the limit is %d)", sp.OverWords, maxSearchWords))
|
||||
return
|
||||
}
|
||||
scored, _ := co.executeSearch(sp)
|
||||
snippetTerm, _ := snippetAnchor(sp.Queries, sp.Fields, sp.Conjs)
|
||||
|
||||
perPage := apiPageSize(q)
|
||||
page, _ := strconv.Atoi(q.Get("p"))
|
||||
start, end, page, totalPages := paginate(len(scored), page, perPage)
|
||||
withSnippets := snippetTerm != "" && perPage > 0
|
||||
|
||||
results := make([]apiSearchResult, 0, end-start)
|
||||
for _, s := range scored[start:end] {
|
||||
res := apiSearchResult{apiDocSummary: docSummary(*s.Doc), Score: s.Score}
|
||||
if withSnippets {
|
||||
if body := s.Doc.bodySnippets(snippetTerm, 1); len(body) > 0 {
|
||||
res.Snippet = body[0]
|
||||
}
|
||||
for _, h := range versionHits(s.Doc, snippetTerm, 1) {
|
||||
res.MatchedVersions = append(res.MatchedVersions, apiMatchedVersion{
|
||||
Slug: h.Version.Slug,
|
||||
Label: h.Version.Label,
|
||||
URL: versionPath(s.Doc.ID, h.Version.Slug),
|
||||
Snippet: h.Snippets[0],
|
||||
})
|
||||
}
|
||||
}
|
||||
results = append(results, res)
|
||||
}
|
||||
|
||||
body := map[string]interface{}{
|
||||
"total": len(scored),
|
||||
"page": page,
|
||||
"per_page": apiRespPerPage(perPage, len(scored)),
|
||||
"total_pages": totalPages,
|
||||
"results": results,
|
||||
}
|
||||
if snippetTerm != "" && !withSnippets {
|
||||
body["snippets_omitted"] = true
|
||||
}
|
||||
writeJSON(w, http.StatusOK, body)
|
||||
}
|
||||
|
||||
func (co *Corpus) buildFacetsJSON() {
|
||||
b, err := json.Marshal(map[string]interface{}{
|
||||
"note": "facet counts cover the whole corpus; news_types covers the news collection only",
|
||||
"collections": co.archive.CollectionCounts,
|
||||
"sidtoday_total": co.archive.SIDTodayCount,
|
||||
"types": co.archive.Types,
|
||||
"news_types": co.archive.NewsTypes,
|
||||
"agencies": co.archive.Facets["agency"],
|
||||
"programs": co.archive.Facets["program"],
|
||||
"journalists": co.archive.Facets["journalist"],
|
||||
"publishers": co.archive.Facets["publisher"],
|
||||
"countries": co.archive.Facets["country"],
|
||||
"people": co.archive.Facets["people"],
|
||||
"classification": co.archive.Facets["classification"],
|
||||
"dissemination": co.archive.Facets["dissemination"],
|
||||
"banners": co.archive.Facets["banner"],
|
||||
"genres": co.archive.Facets["genre"],
|
||||
"series": co.archive.Facets["series"],
|
||||
"tags": co.archive.Facets["tag"],
|
||||
"pub_years": co.archive.Facets["published"],
|
||||
"doc_years": co.archive.Facets["docdate"],
|
||||
"by_collection": co.archive.FacetsByColl,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("facets: marshal failed: %v", err)
|
||||
}
|
||||
co.facetsJSON = b
|
||||
sum := sha256.Sum256(b)
|
||||
co.facetsETag = `"` + hex.EncodeToString(sum[:8]) + `"`
|
||||
log.Printf("Facets JSON pre-built: %d bytes", len(co.facetsJSON))
|
||||
}
|
||||
|
||||
func (co *Corpus) handleAPIFacets(w http.ResponseWriter, r *http.Request) {
|
||||
setJSONHeaders(w)
|
||||
w.Header().Set("ETag", co.facetsETag)
|
||||
http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(co.facetsJSON))
|
||||
}
|
||||
|
||||
// "version" as in document versions (metadata.json can have "versions" object)
|
||||
type apiVersion struct {
|
||||
Slug string `json:"slug"`
|
||||
URL string `json:"url"`
|
||||
PDF string `json:"pdf"`
|
||||
PDFSha256 string `json:"pdf_sha256,omitempty"`
|
||||
Images []string `json:"images,omitempty"`
|
||||
FullText string `json:"full_text,omitempty"`
|
||||
Publications []int `json:"publications,omitempty"`
|
||||
ChangedPages []int `json:"changed_pages,omitempty"`
|
||||
}
|
||||
|
||||
func apiVersions(doc *Document) []apiVersion {
|
||||
var out []apiVersion
|
||||
for _, v := range doc.Versions {
|
||||
out = append(out, apiVersion{
|
||||
Slug: v.Slug,
|
||||
URL: versionPath(doc.ID, v.Slug),
|
||||
PDF: versionFilePath(doc.ID, v.Slug, v.PDFFile),
|
||||
PDFSha256: v.SHA,
|
||||
Images: v.Images,
|
||||
FullText: v.FullText,
|
||||
Publications: v.PubIndex,
|
||||
ChangedPages: changedPageNumbers(v.ChangedPages),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type apiMatchedVersion struct {
|
||||
Slug string `json:"slug"`
|
||||
Label string `json:"label"`
|
||||
URL string `json:"url"`
|
||||
Snippet string `json:"snippet"`
|
||||
}
|
||||
|
||||
func changedPageNumbers(pages []ChangedPage) []int {
|
||||
var out []int
|
||||
for _, p := range pages {
|
||||
out = append(out, p.Page)
|
||||
}
|
||||
return out
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue