first commit
This commit is contained in:
commit
8c4dcae475
89 changed files with 24623 additions and 0 deletions
114
explore_ids.go
Normal file
114
explore_ids.go
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
// explore_ids.go - /explore/ids: every document's identifiers in one table.
|
||||
//
|
||||
// /explore/ids.txt is the same checksums in sha256sum(1) format, so a mirror
|
||||
// or a download verifies with "sha256sum -c" in the corpus root.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type idVersionRow struct {
|
||||
Slug string
|
||||
PDFFile string
|
||||
PDFURL string
|
||||
Size string
|
||||
Pages int
|
||||
SHA string
|
||||
}
|
||||
|
||||
type idRow struct {
|
||||
ID string
|
||||
Collection string
|
||||
Pages int
|
||||
PDFFile string
|
||||
PDFURL string
|
||||
Size string
|
||||
SHA string
|
||||
Aka []AkaEntry
|
||||
Versions []idVersionRow
|
||||
}
|
||||
|
||||
func humanSize(n int64) string {
|
||||
switch {
|
||||
case n <= 0:
|
||||
return ""
|
||||
case n < 1<<10:
|
||||
return fmt.Sprintf("%d B", n)
|
||||
case n < 1<<20:
|
||||
return fmt.Sprintf("%.0f KB", float64(n)/(1<<10))
|
||||
case n < 1<<30:
|
||||
return fmt.Sprintf("%.1f MB", float64(n)/(1<<20))
|
||||
}
|
||||
return fmt.Sprintf("%.2f GB", float64(n)/(1<<30))
|
||||
}
|
||||
|
||||
func (co *Corpus) buildIDRows() {
|
||||
co.idRows = make([]idRow, 0, len(co.archive.Docs))
|
||||
var sums strings.Builder
|
||||
|
||||
for i := range co.archive.Docs {
|
||||
d := &co.archive.Docs[i]
|
||||
row := idRow{
|
||||
ID: d.ID,
|
||||
Collection: d.Collection,
|
||||
Pages: d.Pages,
|
||||
PDFFile: d.PDFFile,
|
||||
PDFURL: filePath(d.ID, d.PDFFile),
|
||||
Size: humanSize(d.PDFSize),
|
||||
SHA: strings.TrimSpace(d.PDFSha256),
|
||||
Aka: d.AkaID,
|
||||
}
|
||||
for _, v := range d.Versions {
|
||||
row.Versions = append(row.Versions, idVersionRow{
|
||||
Slug: v.Slug,
|
||||
PDFFile: v.PDFFile,
|
||||
PDFURL: versionFilePath(d.ID, v.Slug, v.PDFFile),
|
||||
Size: humanSize(v.PDFSize),
|
||||
Pages: len(v.Images),
|
||||
SHA: v.SHA,
|
||||
})
|
||||
}
|
||||
co.idRows = append(co.idRows, row)
|
||||
|
||||
rel, err := filepath.Rel(dataDir, d.Dir)
|
||||
if err != nil {
|
||||
rel = d.ID
|
||||
}
|
||||
if row.SHA != "" {
|
||||
sums.WriteString(row.SHA + " " + filepath.ToSlash(filepath.Join(rel, d.PDFFile)) + "\n")
|
||||
}
|
||||
for _, v := range d.Versions {
|
||||
if v.SHA != "" {
|
||||
sums.WriteString(v.SHA + " " + filepath.ToSlash(filepath.Join(rel, "versions", v.Slug, v.PDFFile)) + "\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(co.idRows, func(i, j int) bool {
|
||||
a, b := strings.ToLower(co.idRows[i].ID), strings.ToLower(co.idRows[j].ID)
|
||||
if a != b {
|
||||
return a < b
|
||||
}
|
||||
return co.idRows[i].ID < co.idRows[j].ID
|
||||
})
|
||||
co.idChecksums = sums.String()
|
||||
}
|
||||
|
||||
func (co *Corpus) handleExploreIDs(w http.ResponseWriter, r *http.Request) {
|
||||
render(w, r, &idsPage{
|
||||
Rows: co.idRows,
|
||||
RowsHTML: co.preHTML("ids"),
|
||||
Shown: len(co.idRows),
|
||||
})
|
||||
}
|
||||
|
||||
func (co *Corpus) handleExploreIDsText(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "public, max-age=3600")
|
||||
w.Write([]byte(co.idChecksums))
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue