// seo.go - robots.txt, sitemap.xml, and the per-document metadata embeds. package main import ( "bytes" "encoding/json" "encoding/xml" "fmt" "html/template" "net/http" "net/url" "strings" ) var canonicalBase string func requestBase(r *http.Request) string { if canonicalBase != "" { return canonicalBase } scheme := "http" if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { scheme = "https" } return scheme + "://" + r.Host } func handleRobots(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "public, max-age=86400") w.Header().Set("Content-Type", "text/plain; charset=utf-8") fmt.Fprintf(w, `User-agent: * Allow: / Disallow: /search Disallow: /api/ Disallow: /random Disallow: /pinned Disallow: /preferences Sitemap: %s/sitemap.xml `, requestBase(r)) } type sitemapURL struct { Loc string `xml:"loc"` } type sitemapSet struct { XMLName xml.Name `xml:"urlset"` Xmlns string `xml:"xmlns,attr"` URLs []sitemapURL `xml:"url"` } func (co *Corpus) buildSitemapPaths() { paths := []string{ "/", "/browse", "/articles", "/glossary", "/changelog", "/about", } for _, c := range categories { paths = append(paths, "/browse/"+c.Slug) } paths = append(paths, "/explore") for _, m := range exploreMetrics { paths = append(paths, "/explore/"+m.Slug) } paths = append(paths, "/explore/terms") paths = append(paths, "/explore/classification") paths = append(paths, "/explore/portions") paths = append(paths, "/explore/outlets") if len(co.assemblies) > 0 { paths = append(paths, assembliesPath) for _, a := range co.assemblies { paths = append(paths, assemblyPath(a.ID)) } } for _, d := range co.archive.Docs { paths = append(paths, docPath(d.ID)) } for _, a := range co.articles { if a.Title != "" { paths = append(paths, "/article/"+a.Key) } } co.sitemapPaths = paths } func (co *Corpus) marshalSitemap(base string) ([]byte, error) { set := sitemapSet{Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9"} set.URLs = make([]sitemapURL, len(co.sitemapPaths)) for i, p := range co.sitemapPaths { set.URLs[i] = sitemapURL{Loc: base + p} } out, err := xml.MarshalIndent(set, "", " ") if err != nil { return nil, err } return append([]byte(xml.Header), out...), nil } func parseCanonicalBase(base string) (string, error) { u, err := url.Parse(base) if err != nil { return "", fmt.Errorf("invalid base URL %q: %w", base, err) } if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.Fragment != "" { return "", fmt.Errorf("base must be scheme://host with no path/query/fragment/userinfo, e.g. https://archive.example.org (got %q)", base) } return u.Scheme + "://" + u.Host, nil } const sitemapBaseMark = "{base url}" func (co *Corpus) buildCanonicalSitemap() { co.sitemapParts = nil if canonicalBase == "" { if b, err := co.marshalSitemap(sitemapBaseMark); err == nil { co.sitemapParts = bytes.Split(b, []byte(sitemapBaseMark)) } return } b, err := co.marshalSitemap(canonicalBase) if err != nil { errorf("sitemap: %v - building it per request instead", err) return } co.canonicalSitemap = b } func (co *Corpus) handleSitemap(w http.ResponseWriter, r *http.Request) { body := co.canonicalSitemap var base bytes.Buffer if body == nil && co.sitemapParts != nil { xml.EscapeText(&base, []byte(requestBase(r))) } else if body == nil { var err error body, err = co.marshalSitemap(requestBase(r)) if err != nil { serverError(w, err) return } } w.Header().Set("Cache-Control", "public, max-age=3600") w.Header().Set("Content-Type", "application/xml; charset=utf-8") if body != nil { w.Write(body) return } for i, part := range co.sitemapParts { if i > 0 { w.Write(base.Bytes()) } w.Write(part) } } // docEmbeds is handed to doc.html as data["Embeds"]. type docEmbeds struct { JSONLDTag template.HTML COinS string } func buildDocEmbeds(doc *Document, absURL, pdfURL string) docEmbeds { type entity struct { Type string `json:"@type"` Name string `json:"name"` } type media struct { Type string `json:"@type"` ContentURL string `json:"contentUrl"` EncodingFormat string `json:"encodingFormat"` } keywords := strings.Join(append(append([]string{}, doc.DocMeta.Programs...), doc.Tags...), ", ") ld := struct { Context string `json:"@context"` Type string `json:"@type"` Name string `json:"name"` URL string `json:"url"` Identifier string `json:"identifier"` Description string `json:"description,omitempty"` DateCreated string `json:"dateCreated,omitempty"` DatePublished string `json:"datePublished,omitempty"` InLanguage string `json:"inLanguage"` Keywords string `json:"keywords,omitempty"` Author *entity `json:"author,omitempty"` Publisher entity `json:"publisher"` AssociatedMedia media `json:"associatedMedia"` }{ Context: "https://schema.org", Type: "DigitalDocument", Name: doc.Title, URL: absURL, Identifier: absURL, Description: doc.Description, DateCreated: doc.DocMeta.DateCreated, DatePublished: oldestPubDate(doc), InLanguage: "en", Keywords: keywords, Publisher: entity{Type: "Organization", Name: "Snowden Archive"}, AssociatedMedia: media{ Type: "MediaObject", ContentURL: pdfURL, EncodingFormat: "application/pdf", }, } if doc.DocMeta.Creator != "" { ld.Author = &entity{Type: "Organization", Name: doc.DocMeta.Creator} } var tag template.HTML if b, err := json.Marshal(ld); err == nil { tag = template.HTML(``) } v := url.Values{} v.Set("ctx_ver", "Z39.88-2004") v.Set("rft_val_fmt", "info:ofi/fmt:kev:mtx:dc") v.Set("rft.title", doc.Title) if doc.DocMeta.Creator != "" { v.Set("rft.creator", doc.DocMeta.Creator) } if doc.DocMeta.DateCreated != "" { v.Set("rft.date", doc.DocMeta.DateCreated) } v.Set("rft.type", "Text") v.Set("rft.format", "application/pdf") v.Set("rft.identifier", absURL) v.Set("rft.publisher", "Snowden Archive") for _, p := range doc.DocMeta.Programs { v.Add("rft.subject", p) } return docEmbeds{JSONLDTag: tag, COinS: v.Encode()} }