591 lines
16 KiB
Go
591 lines
16 KiB
Go
// mdparser.go - minimal dependency-free Markdown parser.
|
|
//
|
|
// Input trust: everything rendered through this parser is the archive's own
|
|
// curator-produced OCR never user-supplied input.
|
|
|
|
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"html"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// -- Inline patterns ----------------------------------------------------
|
|
|
|
var (
|
|
reInlineCode = regexp.MustCompile("`(.+?)`")
|
|
reStrikethrough = regexp.MustCompile(`~~(.+?)~~`)
|
|
reHardBreak = regexp.MustCompile(`(?i)<br\s*/?>`)
|
|
reEscape = regexp.MustCompile("\\\\([!-/:-@\\[-`|{}~])")
|
|
reOrderedItem = regexp.MustCompile(`^(\s*)(\d{1,9})\.(?:\s+(.*))?\s*$`)
|
|
reUnorderedItem = regexp.MustCompile(`^(\s*)[-*+](?:\s+(.*))?\s*$`)
|
|
reTableSep = regexp.MustCompile(`^\|[-| :]+\|$`)
|
|
)
|
|
|
|
// parseInline converts inline Markdown to HTML.
|
|
func parseInline(s string) string {
|
|
s = strings.ReplaceAll(s, "\x00", "")
|
|
s = strings.ReplaceAll(s, "\x01", "")
|
|
s = html.EscapeString(s)
|
|
|
|
// Backslash escapes (CommonMark): "\*" is a literal "*", not emphasis
|
|
// syntax. Pull escaped ASCII punctuation out to placeholders BEFORE the
|
|
// emphasis passes so the character can't be interpreted as a marker,
|
|
// then restore it at the end.
|
|
var escaped []string
|
|
if strings.IndexByte(s, '\\') >= 0 {
|
|
s = reEscape.ReplaceAllStringFunc(s, func(m string) string {
|
|
// m is "\X": EscapeString leaves the backslash alone. Keep
|
|
// the char after it.
|
|
ch := m[len(m)-1:]
|
|
escaped = append(escaped, ch)
|
|
return "\x01" + strconv.Itoa(len(escaped)-1) + "\x01"
|
|
})
|
|
}
|
|
|
|
// Extract inline code spans next and replace them with placeholders so
|
|
// the strikethrough/bold/italic passes can't reformat code content
|
|
// (e.g. `*x*` must render as literal *x*, not <em>x</em>). Escape
|
|
// placeholders that landed INSIDE a code span are put back as their
|
|
// original "\X" source text first: backslash escapes are not escapes
|
|
// within code spans (CommonMark), so `\*` renders as \*, not *. The
|
|
// escape pass still runs before this one on purpose - that ordering is
|
|
// what lets \` be a literal backtick that cannot OPEN a code span.
|
|
var codes []string
|
|
if strings.IndexByte(s, '`') >= 0 {
|
|
s = reInlineCode.ReplaceAllStringFunc(s, func(m string) string {
|
|
inner := reInlineCode.FindStringSubmatch(m)[1]
|
|
for i, ch := range escaped {
|
|
inner = strings.ReplaceAll(inner, "\x01"+strconv.Itoa(i)+"\x01", "\\"+ch)
|
|
}
|
|
codes = append(codes, "<code>"+inner+"</code>")
|
|
return "\x00" + strconv.Itoa(len(codes)-1) + "\x00"
|
|
})
|
|
}
|
|
|
|
// After the code-span pass, so `<br>` inside backticks stays literal.
|
|
if strings.Contains(s, "<") {
|
|
s = reHardBreak.ReplaceAllString(s, "<br>")
|
|
}
|
|
if strings.Contains(s, "~~") {
|
|
s = reStrikethrough.ReplaceAllString(s, `<del>$1</del>`)
|
|
}
|
|
// ***x*** first, so it nests as <em><strong> rather than crossing.
|
|
s = emphasize(s, '*', 3, "<em><strong>", "</strong></em>")
|
|
s = emphasize(s, '*', 2, "<strong>", "</strong>")
|
|
s = emphasize(s, '_', 2, "<strong>", "</strong>")
|
|
s = emphasize(s, '*', 1, "<em>", "</em>")
|
|
s = emphasize(s, '_', 1, "<em>", "</em>")
|
|
|
|
// Restore code spans.
|
|
for i, code := range codes {
|
|
s = strings.Replace(s, "\x00"+strconv.Itoa(i)+"\x00", code, 1)
|
|
}
|
|
// Restore backslash-escaped characters as their literal selves.
|
|
// (Placeholders that sat inside a code span were already restored to
|
|
// their "\X" source form up in the code-extraction pass and no longer
|
|
// occur here; Replace on an absent placeholder is a no-op.)
|
|
for i, ch := range escaped {
|
|
s = strings.Replace(s, "\x01"+strconv.Itoa(i)+"\x01", ch, 1)
|
|
}
|
|
return s
|
|
}
|
|
|
|
// emphasize wraps the text between two runs of k delimiters c ('*' or '_')
|
|
// in open/close. It pairs them as CommonMark does: left to right, a run
|
|
// that can close gives its first k delimiters to the nearest unpaired
|
|
// opener before it, and a run that can open offers its last k to a later
|
|
// closer. Which runs can open or close follows CommonMark's flanking rules
|
|
// (delimiterRun).
|
|
func emphasize(s string, c byte, k int, open, close string) string {
|
|
var openers []int // offsets of unpaired openers, nearest last
|
|
tags := map[int]string{} // offset of k paired delimiters -> their tag
|
|
for i := 0; i < len(s); {
|
|
if s[i] != c {
|
|
i++
|
|
continue
|
|
}
|
|
end := i
|
|
for end < len(s) && s[end] == c {
|
|
end++
|
|
}
|
|
canOpen, canClose := delimiterRun(s, i, end, c)
|
|
n := end - i
|
|
if n >= k && canClose && len(openers) > 0 {
|
|
tags[openers[len(openers)-1]], tags[i] = open, close
|
|
openers = openers[:len(openers)-1]
|
|
n -= k
|
|
}
|
|
if n >= k && canOpen {
|
|
openers = append(openers, end-k)
|
|
}
|
|
i = end
|
|
}
|
|
if len(tags) == 0 {
|
|
return s
|
|
}
|
|
var b strings.Builder
|
|
for i := 0; i < len(s); i++ {
|
|
if t, ok := tags[i]; ok {
|
|
b.WriteString(t)
|
|
i += k - 1
|
|
} else {
|
|
b.WriteByte(s[i])
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// delimiterRun reports whether the run of delimiters c at s[start:end]
|
|
// can open and can close emphasis, by CommonMark's flanking rules, which
|
|
// look at the whole run.
|
|
func delimiterRun(s string, start, end int, c byte) (canOpen, canClose bool) {
|
|
prev, next := ' ', ' '
|
|
if start > 0 {
|
|
prev, _ = utf8.DecodeLastRuneInString(s[:start])
|
|
}
|
|
if end < len(s) {
|
|
next, _ = utf8.DecodeRuneInString(s[end:])
|
|
}
|
|
left := !unicode.IsSpace(next) &&
|
|
(!isPunct(next) || unicode.IsSpace(prev) || isPunct(prev))
|
|
right := !unicode.IsSpace(prev) &&
|
|
(!isPunct(prev) || unicode.IsSpace(next) || isPunct(next))
|
|
if c == '_' {
|
|
return left && (!right || isPunct(prev)), right && (!left || isPunct(next))
|
|
}
|
|
return left, right
|
|
}
|
|
|
|
// isPunct is CommonMark's punctuation (Unicode P and S). The \x00/\x01
|
|
// placeholders count too: they stand for a code span (opened by a
|
|
// backtick) or a backslash-escaped punctuation character.
|
|
func isPunct(r rune) bool {
|
|
return r == 0 || r == 1 || unicode.IsPunct(r) || unicode.IsSymbol(r)
|
|
}
|
|
|
|
// -- Block-level parser --------------------------------------------
|
|
|
|
type parser struct {
|
|
lines []string
|
|
pos int
|
|
out strings.Builder
|
|
open []listLevel // the lists parseList is inside, outermost first
|
|
// numbered holds the last number of the ordered list at each indent,
|
|
// for as long as list lines keep coming (see endsItem).
|
|
numbered map[int]int
|
|
// top is the source's highest heading level and last the level of the
|
|
// heading emitted last (see parseHeading).
|
|
top, last int
|
|
}
|
|
|
|
// listLevel is one open list: the indent of its items and their kind.
|
|
type listLevel struct {
|
|
indent int
|
|
ordered bool
|
|
}
|
|
|
|
func newParser(src string) *parser {
|
|
src = strings.ReplaceAll(src, "\r\n", "\n")
|
|
src = strings.ReplaceAll(src, "\r", "\n")
|
|
p := &parser{lines: strings.Split(src, "\n"), top: 7, last: 1}
|
|
fence := ""
|
|
for _, line := range p.lines {
|
|
switch {
|
|
case fence != "":
|
|
if strings.HasPrefix(line, fence) {
|
|
fence = ""
|
|
}
|
|
case isFencedCodeStart(line):
|
|
fence = line[:3]
|
|
case isHeading(line):
|
|
p.top = min(p.top, headingLevel(line))
|
|
}
|
|
}
|
|
return p
|
|
}
|
|
|
|
func (p *parser) peek() (string, bool) {
|
|
if p.pos >= len(p.lines) {
|
|
return "", false
|
|
}
|
|
return p.lines[p.pos], true
|
|
}
|
|
|
|
func (p *parser) next() (string, bool) {
|
|
l, ok := p.peek()
|
|
if ok {
|
|
p.pos++
|
|
}
|
|
return l, ok
|
|
}
|
|
|
|
func (p *parser) emit(s string) { p.out.WriteString(s) }
|
|
|
|
// parse processes all lines and returns HTML.
|
|
func (p *parser) parse() string {
|
|
for p.pos < len(p.lines) {
|
|
line, _ := p.peek()
|
|
if !isListItem(line) {
|
|
p.numbered = nil // the run of list lines is over
|
|
}
|
|
|
|
switch {
|
|
case isFencedCodeStart(line):
|
|
p.parseFencedCode()
|
|
case isHR(line):
|
|
p.next()
|
|
p.emit("<hr>\n")
|
|
case isHeading(line):
|
|
p.parseHeading()
|
|
case isTableRow(line):
|
|
p.parseTable()
|
|
case isListItem(line):
|
|
p.parseList(reOrderedItem.MatchString(line), indentOf(line))
|
|
case strings.TrimSpace(line) == "":
|
|
p.next()
|
|
default:
|
|
p.parseParagraph()
|
|
}
|
|
}
|
|
return p.out.String()
|
|
}
|
|
|
|
// -- Heading --------------------------------------------------
|
|
|
|
// headingLevel returns the heading level of an ATX heading line - one to
|
|
// six '#' followed by whitespace or the end of the line - or 0 when the
|
|
// line is not a heading.
|
|
func headingLevel(s string) int {
|
|
n := 0
|
|
for n < len(s) && s[n] == '#' {
|
|
n++
|
|
}
|
|
if n == 0 || n > 6 || (n < len(s) && s[n] != ' ' && s[n] != '\t') {
|
|
return 0
|
|
}
|
|
return n
|
|
}
|
|
|
|
func isHeading(s string) bool { return headingLevel(s) > 0 }
|
|
|
|
// parseHeading emits a heading under the page's own <h1>: the source's top
|
|
// level becomes <h2> and no heading skips a level. The md-hN class keeps the
|
|
// size of the level the source wrote.
|
|
func (p *parser) parseHeading() {
|
|
line, _ := p.next()
|
|
level := headingLevel(line)
|
|
content := strings.TrimSpace(line[level:])
|
|
tag := min(level-p.top+2, p.last+1, 6)
|
|
p.last = tag
|
|
p.emit(fmt.Sprintf("<h%d class=\"md-h%d\">%s</h%d>\n", tag, level, parseInline(content), tag))
|
|
}
|
|
|
|
// -- Horizontal rule -------------------------------------------------
|
|
|
|
// isHR reports whether s is a thematic break: three or more of the same
|
|
// '-', '*' or '_', optionally with spaces or tabs between them.
|
|
func isHR(s string) bool {
|
|
t := strings.TrimSpace(s)
|
|
if t == "" || (t[0] != '-' && t[0] != '*' && t[0] != '_') {
|
|
return false
|
|
}
|
|
n := 0
|
|
for i := 0; i < len(t); i++ {
|
|
switch t[i] {
|
|
case t[0]:
|
|
n++
|
|
case ' ', '\t':
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
return n >= 3
|
|
}
|
|
|
|
// Fenced code block ---------------------------------------------------------
|
|
|
|
func isFencedCodeStart(s string) bool {
|
|
return strings.HasPrefix(s, "```") || strings.HasPrefix(s, "~~~")
|
|
}
|
|
|
|
func (p *parser) parseFencedCode() {
|
|
opener, _ := p.next()
|
|
fence := opener[:3]
|
|
|
|
var body strings.Builder
|
|
for {
|
|
line, ok := p.next()
|
|
if !ok || strings.HasPrefix(line, fence) {
|
|
break
|
|
}
|
|
body.WriteString(html.EscapeString(line))
|
|
body.WriteByte('\n')
|
|
}
|
|
p.emit("<pre><code>" + body.String() + "</code></pre>\n")
|
|
}
|
|
|
|
// -- Lists (unified, handles nesting) -----------------------------------------
|
|
|
|
// indentOf returns the number of leading whitespace characters in s.
|
|
func indentOf(s string) int {
|
|
return len(s) - len(strings.TrimLeft(s, " \t"))
|
|
}
|
|
|
|
// isListItem returns true if the line is a ul or ol item at any indent.
|
|
// A thematic break wins over a bullet ("* * *" is a rule, not an item).
|
|
func isListItem(s string) bool {
|
|
// A list marker is the first non-space byte: rule out the rest (most
|
|
// lines) before any regex runs.
|
|
t := strings.TrimLeft(s, "\t\n\f\r ")
|
|
if t == "" || (t[0] != '-' && t[0] != '*' && t[0] != '+' && (t[0] < '0' || t[0] > '9')) {
|
|
return false
|
|
}
|
|
return !isHR(s) && (reUnorderedItem.MatchString(s) || reOrderedItem.MatchString(s))
|
|
}
|
|
|
|
// itemContent strips the list marker and returns the item text (possibly
|
|
// empty for a bare marker like "6.").
|
|
func itemContent(s string) string {
|
|
if sub := reUnorderedItem.FindStringSubmatch(s); sub != nil {
|
|
return sub[2]
|
|
}
|
|
if sub := reOrderedItem.FindStringSubmatch(s); sub != nil {
|
|
return sub[3]
|
|
}
|
|
return strings.TrimSpace(s)
|
|
}
|
|
|
|
// orderedNum returns the source number of an ordered-list line ("7." -> 7)
|
|
// and true, or false if the line isn't an ordered item. "0." is a real
|
|
// number, so it cannot double as the "not ordered" answer.
|
|
func orderedNum(s string) (int, bool) {
|
|
if sub := reOrderedItem.FindStringSubmatch(s); sub != nil {
|
|
n, _ := strconv.Atoi(sub[2]) // at most nine digits: cannot fail
|
|
return n, true
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
// startsList reports whether list-item line s may open a new list on the
|
|
// line after text, where it would otherwise continue that text.
|
|
func startsList(s string) bool {
|
|
if itemContent(s) == "" {
|
|
return false
|
|
}
|
|
n, ok := orderedNum(s)
|
|
return !ok || n == 1
|
|
}
|
|
|
|
// endsItem reports whether line s, met while collecting the lazily
|
|
// continued text of a list item, ends that item rather than continuing it.
|
|
func (p *parser) endsItem(s string) bool {
|
|
if isHR(s) || isHeading(s) || isFencedCodeStart(s) || isTableRow(s) {
|
|
return true
|
|
}
|
|
if !isListItem(s) {
|
|
return false
|
|
}
|
|
if startsList(s) {
|
|
return true
|
|
}
|
|
indent, ordered := indentOf(s), reOrderedItem.MatchString(s)
|
|
for _, l := range p.open {
|
|
if l.indent == indent && l.ordered == ordered {
|
|
return true
|
|
}
|
|
}
|
|
if n, ok := orderedNum(s); ok {
|
|
for _, last := range p.numbered {
|
|
if n > last && n <= last+2 {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// parseList recursively parses a ul or ol at the given indent level.
|
|
func (p *parser) parseList(ordered bool, minIndent int) {
|
|
tag := "ul"
|
|
if ordered {
|
|
tag = "ol"
|
|
}
|
|
p.emit("<" + tag + ">\n")
|
|
p.open = append(p.open, listLevel{minIndent, ordered})
|
|
defer func() { p.open = p.open[:len(p.open)-1] }()
|
|
|
|
expected := 0 // running ordered-list counter, for value="N" decisions
|
|
|
|
for {
|
|
line, ok := p.peek()
|
|
if !ok || strings.TrimSpace(line) == "" || !isListItem(line) {
|
|
break
|
|
}
|
|
indent := indentOf(line)
|
|
if indent < minIndent {
|
|
// Belongs to a parent list - stop.
|
|
break
|
|
}
|
|
if indent == minIndent && reOrderedItem.MatchString(line) != ordered {
|
|
// Bullets after numbers, or numbers after bullets: a new
|
|
// list (as in CommonMark). Kept in this one, a number would
|
|
// lose itself to a bullet and a bullet would gain a number.
|
|
break
|
|
}
|
|
if indent > minIndent {
|
|
// Deeper indent mid-list without a parent <li> - treat as nested.
|
|
p.parseList(reOrderedItem.MatchString(line), indent)
|
|
continue
|
|
}
|
|
|
|
p.next()
|
|
|
|
// Item body: the marker line plus any lazily-continued lines after it.
|
|
var body []string
|
|
if c := itemContent(line); c != "" {
|
|
body = append(body, c)
|
|
}
|
|
for {
|
|
cont, ok := p.peek()
|
|
if !ok || strings.TrimSpace(cont) == "" || p.endsItem(cont) {
|
|
break
|
|
}
|
|
p.next()
|
|
body = append(body, strings.TrimSpace(cont))
|
|
}
|
|
|
|
liOpen := "<li>"
|
|
if ordered {
|
|
n, _ := orderedNum(line)
|
|
expected++
|
|
if n != expected {
|
|
liOpen = "<li value=\"" + strconv.Itoa(n) + "\">"
|
|
expected = n
|
|
}
|
|
if p.numbered == nil {
|
|
p.numbered = map[int]int{}
|
|
}
|
|
p.numbered[minIndent] = n
|
|
}
|
|
|
|
var parts []string
|
|
for _, l := range body {
|
|
parts = append(parts, parseInline(l))
|
|
}
|
|
text := strings.Join(parts, "<br>\n")
|
|
|
|
// Peek: if next line is deeper, nest a list inside this <li> -
|
|
// as many as follow, since bullets turning into numbers down
|
|
// there start a second nested list, which belongs here too.
|
|
next, hasNext := p.peek()
|
|
if hasNext && isListItem(next) && indentOf(next) > indent {
|
|
p.emit(liOpen + text + "\n")
|
|
for hasNext && isListItem(next) && indentOf(next) > indent {
|
|
p.parseList(reOrderedItem.MatchString(next), indentOf(next))
|
|
next, hasNext = p.peek()
|
|
}
|
|
p.emit("</li>\n")
|
|
} else {
|
|
p.emit(liOpen + text + "</li>\n")
|
|
}
|
|
}
|
|
|
|
p.emit("</" + tag + ">\n")
|
|
}
|
|
|
|
// -- Table ---------------------------------------------------------------------
|
|
|
|
func isTableRow(s string) bool {
|
|
t := strings.TrimSpace(s)
|
|
return len(t) >= 3 && t[0] == '|' && t[len(t)-1] == '|' && !strings.Contains(t, "\n")
|
|
}
|
|
|
|
// splitCells splits a pipe-delimited row into trimmed cell strings.
|
|
func splitCells(row string) []string {
|
|
row = strings.TrimPrefix(strings.TrimSpace(row), "|")
|
|
if strings.HasSuffix(row, "|") && !strings.HasSuffix(row, `\|`) {
|
|
row = row[:len(row)-1]
|
|
}
|
|
var cells []string
|
|
var cell strings.Builder
|
|
for i := 0; i < len(row); i++ {
|
|
switch {
|
|
case row[i] == '\\' && i+1 < len(row) && row[i+1] == '|':
|
|
cell.WriteByte('|')
|
|
i++
|
|
case row[i] == '|':
|
|
cells = append(cells, strings.TrimSpace(cell.String()))
|
|
cell.Reset()
|
|
default:
|
|
cell.WriteByte(row[i])
|
|
}
|
|
}
|
|
return append(cells, strings.TrimSpace(cell.String()))
|
|
}
|
|
|
|
func (p *parser) parseTable() {
|
|
headerLine, _ := p.next()
|
|
headers := splitCells(headerLine)
|
|
|
|
// Consume separator row (|---|---|) if present.
|
|
if sep, ok := p.peek(); ok && reTableSep.MatchString(strings.TrimSpace(sep)) {
|
|
p.next()
|
|
}
|
|
|
|
p.emit("<div class=\"table-scroll\">\n<table>\n<thead>\n<tr>\n")
|
|
for _, h := range headers {
|
|
p.emit("<th>" + parseInline(h) + "</th>\n")
|
|
}
|
|
p.emit("</tr>\n</thead>\n<tbody>\n")
|
|
|
|
for {
|
|
line, ok := p.peek()
|
|
if !ok || !isTableRow(line) {
|
|
break
|
|
}
|
|
p.next()
|
|
p.emit("<tr>\n")
|
|
for _, cell := range splitCells(line) {
|
|
p.emit("<td>" + parseInline(cell) + "</td>\n")
|
|
}
|
|
p.emit("</tr>\n")
|
|
}
|
|
|
|
p.emit("</tbody>\n</table>\n</div>\n")
|
|
}
|
|
|
|
// -- Paragraph -----------------------------------------------------------------
|
|
|
|
func (p *parser) parseParagraph() {
|
|
var lines []string
|
|
for {
|
|
line, ok := p.peek()
|
|
if !ok || strings.TrimSpace(line) == "" {
|
|
break
|
|
}
|
|
if isHeading(line) || isFencedCodeStart(line) || isHR(line) ||
|
|
(isListItem(line) && startsList(line)) || isTableRow(line) {
|
|
break
|
|
}
|
|
p.next()
|
|
lines = append(lines, strings.TrimSpace(line))
|
|
}
|
|
if len(lines) > 0 {
|
|
var parts []string
|
|
for _, l := range lines {
|
|
parts = append(parts, parseInline(l))
|
|
}
|
|
p.emit("<p>" + strings.Join(parts, "<br>\n") + "</p>\n")
|
|
}
|
|
}
|
|
|
|
// Render converts Markdown src to an HTML string.
|
|
func Render(src string) string {
|
|
return newParser(src).parse()
|
|
}
|