4 # EXPERIMENTAL Markdown processor with minimal dependencies.
5 # Meant to support all features of John Grubers basic Markdown
6 # + a number of common extensions, mostly inspired by Pandoc Markdown
9 # - HTML processing / escaping (according to environment flag)
10 # - em-dashes and arrows
12 # Supported Features / TODO:
13 # ==========================
14 # [x] done [ ] todo [-] not planned ? unsure
16 # Basic Markdown - Block elements:
17 # -------------------------------
19 # - [x] Double space line breaks
20 # - [x] Proper block element nesting
22 # - [x] ATX-Style Headings
24 # - [x] Lists (ordered, unordered)
25 # - [x] Code blocks (using indention)
26 # - [x] Horizontal rules
27 # - [x] Verbatim HTML block (disabled by default)
29 # Basic Markdown - Inline elements:
30 # ---------------------------------
32 # - [x] Reference style links
33 # - [x] Emphasis *em*/**strong** (*Asterisk*, _Underscore_)
34 # - [x] `code`, also ``code containing `backticks` ``
35 # - [x] Images / reference style images
36 # - [x] <automatic links>
37 # - [x] backslash escapes
38 # - [x] Verbatim HTML inline (disabled by default)
41 # NOTE: Set the environment variable MD_HTML=true to enable verbatim HTML
43 # Extensions - Block elements:
44 # ----------------------------
45 # - ? Heading identifiers (php md, pandoc)
46 # - [x] Fenced code blocks (php md, pandoc)
47 # - [-] Fenced code attributes
49 # - ? Simple table (pandoc)
50 # - ? Multiline table (pandoc)
51 # - ? Grid table (pandoc)
52 # - ? Pipe table (php md pandoc)
53 # - [x] Line blocks (pandoc)
54 # - [x] Task lists (pandoc)
55 # - [ ] Definition lists (php md, pandoc)
56 # - [-] Numbered example lists (pandoc)
57 # - [-] Metadata blocks (pandoc)
58 # - [-] Fenced Divs (pandoc)
60 # Extensions - Inline elements:
61 # ----------------------------
62 # - [x] Ignore embedded_underscores (php md, pandoc)
63 # - [x] ~~strikeout~~ (pandoc)
64 # - [x] ^Superscript^ ~Subscript~ (pandoc)
65 # - [-] Bracketed spans (pandoc)
66 # - [-] Inline attributes (pandoc)
67 # - [-] TEX-Math (pandoc)
68 # - ? Footnotes (php md)
69 # - ? Abbreviations (php md)
70 # - ? "Curly quotes" (smartypants)
71 # - [ ] em-dashes (--) (smartypants old)
72 # - ? ... three-dot ellipsis (smartypants)
73 # - [-] en-dash (smartypants)
74 # - [ ] Automatic em-dash / en-dash
75 # - [ ] Automatic -> Arrows <-
77 function debug(text) { printf "\n---\n%s\n---\n", text > "/dev/stderr"; }
79 function HTML ( text ) {
80 gsub( /&/, "\\&", text );
81 gsub( /</, "\\<", text );
82 gsub( />/, "\\>", text );
83 gsub( /"/, "\\"", text );
84 gsub( /'/, "\\'", text );
85 gsub( /\\/, "\\\", text );
89 function inline( line, LOCAL, len, code, href, guard ) {
90 nu = "(\\\\\\\\|\\\\[^\\\\]|[^\\\\_]|_[[:alnum:]])*" # not underline (except when escaped)
91 na = "(\\\\\\\\|\\\\[^\\\\]|[^\\\\\\*])*" # not asterisk (except when escaped)
92 ieu = "_([^_[:space:]]|[^_[:space:]]" nu "[^_[:space:]])_" # inner <em> (underline)
93 isu = "__([^_[:space:]]|[^_[:space:]]" nu "[^_[:space:]])__" # inner <strong> (underline)
94 iea = "\\*([^\\*[:space:]]|[^\\*[:space:]]" na "[^\\*[:space:]])\\*" # inner <em> (asterisk)
95 isa = "\\*\\*([^\\*[:space:]]|[^\\*[:space:]]" na "[^\\*[:space:]])\\*\\*" # inner <strong> (asterisk)
97 if ( line ~ /^$/ ) { # Recursion End
100 # omit processing of escaped characters
101 } else if ( line ~ /^\\[]\\`\*_\{\}\(\)#\+-\.![]/) {
102 return substr(line, 2, 1) inline( substr(line, 3) );
105 } else if ( match(line, /^ \n/) ) {
106 return "<br />\n" inline( substr(line, RLENGTH + 1) );
109 } else if ( match( line, /^`+/) ) {
111 guard = substr( line, 1, len )
112 if ( match(line, guard ".*" guard) ) {
113 code = substr( line, len + 1, match( substr(line, len + 1), guard ) - 1)
114 len = 2 * length(guard) + length(code)
115 # strip single surrounding white spaces
116 code = gensub( / (.*) /, "\\1", "1" , code)
117 # escape HTML within code span
118 gsub( /&/, "\\&", code ); gsub( /</, "\\<", code ); gsub( />/, "\\>", code );
119 return "<code>" code "</code>" inline( substr( line, len + 1 ) )
122 # quick links ("automatic links" in md doc)
123 } else if ( match( line, /^<[a-zA-Z]+:\/\/([-\.[:alnum:]]+)(:[0-9]*)?(\/[^>]*)?>/ ) ) {
125 href = HTML( substr( line, 2, len - 2) );
126 return "<a href=\"" href "\">" href "</a>" inline( substr( line, len + 1) );
129 } else if ( match(line, /^\[([^]]+)\]\(([^"\)]+)([ \t]+"([^"]+)")?\)/) ) {
131 text = gensub(/^\[([^]]+)\]\(([^"\)]+)([ \t]+"([^"]+)")?\)/, "\\1", "g", line);
132 href = gensub(/^\[([^]]+)\]\(([^"\)]+)([ \t]+"([^"]+)")?\)/, "\\2", "g", line);
133 title = gensub(/^\[([^]]+)\]\(([^"\)]+)([ \t]+"([^"]+)")?\)/, "\\4", "g", line);
135 return "<a href=\"" HTML(href) "\" title=\"" HTML(title) "\">" inline( text ) "</a>" inline( substr( line, len + 1) );
137 return "<a href=\"" HTML(href) "\">" inline( text ) "</a>" inline( substr( line, len + 1) );
140 # reference style links
141 } else if ( match(line, /^\[([^]]+)\] ?\[([^]]*)\]/ ) ) {
143 text = gensub(/^\[([^\n]+)\] ?\[([^\n]*)\].*/, "\\1", 1, line);
144 id = gensub(/^\[([^\n]+)\] ?\[([^\n]*)\].*/, "\\2", 1, line);
145 if ( ! id ) id = text;
146 if ( rl_href[id] && rl_title[id] ) {
147 return "<a href=\"" rl_href[id] "\" title=\"" rl_title[id] "\">" inline(text) "</a>" inline( substr( line, len + 1) );
148 } else if ( rl_href[id] ) {
149 return "<a href=\"" rl_href[id] "\">" inline(text) "</a>" inline( substr( line, len + 1) );
151 return "" substr(line, 1, len) inline( substr(line, len + 1) );
155 } else if ( match(line, /^!\[([^]]+)\]\(([^"\)]+)([ \t]+"([^"]+)")?\)/) ) {
157 text = gensub(/^!\[([^]]+)\]\(([^"\)]+)([ \t]+"([^"]+)")?\)/, "\\1", "g", line);
158 href = gensub(/^!\[([^]]+)\]\(([^"\)]+)([ \t]+"([^"]+)")?\)/, "\\2", "g", line);
159 title = gensub(/^!\[([^]]+)\]\(([^"\)]+)([ \t]+"([^"]+)")?\)/, "\\4", "g", line);
161 return "<img src=\"" HTML(href) "\" alt=\"" HTML(text) "\" title=\"" HTML(title) "\" />" inline( substr( line, len + 1) );
163 return "<img src=\"" HTML(href) "\" alt=\"" HTML(text) "\" />" inline( substr( line, len + 1) );
166 # reference style images
167 } else if ( match(line, /^!\[([^]]+)\] ?\[([^]]*)\]/ ) ) {
169 text = gensub(/^!\[([^\n]+)\] ?\[([^\n]*)\].*/, "\\1", 1, line);
170 id = gensub(/^!\[([^\n]+)\] ?\[([^\n]*)\].*/, "\\2", 1, line);
171 if ( ! id ) id = text;
172 if ( rl_href[id] && rl_title[id] ) {
173 return "<img src=\"" rl_href[id] "\" alt=\"" HTML(text) "\" title=\"" rl_title[id] "\" />" inline( substr( line, len + 1) );
174 } else if ( rl_href[id] ) {
175 return "<img src=\"" rl_href[id] "\" alt=\"" HTML(text) "\" />" inline( substr( line, len + 1) );
177 return "" substr(line, 1, len) inline( substr(line, len + 1) );
180 # ~~strikeout~~ (pandoc)
181 } else if ( match(line, /^~~([[:graph:]]|[[:graph:]]([^~]|~[^~])*[[:graph:]])~~/) ) {
183 return "<del>" inline( substr( line, 3, len - 4 ) ) "</del>" inline( substr( line, len + 1 ) );
185 # ^superscript^ (pandoc)
186 } else if ( match(line, /^\^([^[:space:]^]|\\[ ^])+\^/) ) {
188 return "<sup>" inline( substr( line, 2, len - 2 ) ) "</sup>" inline( substr( line, len + 1 ) );
190 # ~subscript~ (pandoc)
191 } else if ( match(line, /^~([^[:space:]~]|\\[ ~])+~/) ) {
193 return "<sub>" inline( substr( line, 2, len - 2 ) ) "</sub>" inline( substr( line, len + 1 ) );
195 # ignore embedded underscores (pandoc, php md)
196 } else if ( match(line, "^[[:alnum:]](__|_)") ) {
197 return substr( line, 1, RLENGTH) inline( substr(line, RLENGTH + 1) );
200 } else if ( match(line, "^__(([^_[:space:]]|" ieu ")|([^_[:space:]]|" ieu ")(" nu "|" ieu ")*([^_[:space:]]|" ieu "))__$") ) {
202 return "<strong>" inline( substr( line, 3, len - 4 ) ) "</strong>" inline( substr( line, len + 1 ) );
205 } else if ( match(line, "^__(([^_[:space:]]|" ieu ")|([^_[:space:]]|" ieu ")(" nu "|" ieu ")*([^_[:space:]]|" ieu "))__[[:space:][:punct:]]") ) {
207 return "<strong>" inline( substr( line, 3, len - 5 ) ) "</strong>" inline( substr( line, len) );
210 } else if ( match(line, "^\\*\\*(([^\\*[:space:]]|" iea ")|([^\\*[:space:]]|" iea ")(" na "|" iea ")*([^\\*[:space:]]|" iea "))\\*\\*") ) {
212 return "<strong>" inline( substr( line, 3, len - 4 ) ) "</strong>" inline( substr( line, len + 1 ) );
215 } else if ( match(line, "^_(([^_[:space:]]|" isu ")|([^_[:space:]]|" isu ")(" nu "|" isu ")*([^_[:space:]]|" isu "))_$") ) {
217 return "<em>" inline( substr( line, 2, len - 2 ) ) "</em>" inline( substr( line, len + 1 ) );
220 } else if ( match(line, "^_(([^_[:space:]]|" isu ")|([^_[:space:]]|" isu ")(" nu "|" isu ")*([^_[:space:]]|" isu "))_[[:space:][:punct:]]") ) {
222 return "<em>" inline( substr( line, 2, len - 3 ) ) "</em>" inline( substr( line, len ) );
225 } else if ( match(line, "^\\*(([^\\*[:space:]]|" isa ")|([^\\*[:space:]]|" isa ")(" na "|" isa ")*([^\\*[:space:]]|" isa "))\\*") ) {
227 return "<em>" inline( substr( line, 2, len - 2 ) ) "</em>" inline( substr( line, len + 1 ) );
229 # Verbatim inline HTML
230 } else if ( AllowHTML && match( line, /^(<!--([^-]|-[^-]|--[^>])*-->|<\?([^\?]|\?[^>])*\?>|<![A-Z][^>]*>|<!\[CDATA\[([^\]]|\][^\]]|\]\][^>])*\]\]>|<\/[A-Za-z][A-Za-z0-9-]*[[:space:]]*>|<[A-Za-z][A-Za-z0-9-]*([[:space:]]+[A-Za-z_:][A-Za-z0-9_\.:-]*([[:space:]]*=[[:space:]]*([[:space:]"'=<>`]+|"[^"]*"|'[^']*'))?)*[[:space:]]*\/?>)/) ) {
232 return substr( line, 1, len) inline(substr(line, len + 1));
234 # Literal HTML entities
235 } else if ( match( line, /^&([a-zA-Z]{2,32}|#[0-9]{1,7}|#[xX][0-9a-fA-F]{1,6});/) ) {
237 return substr( line, 1, len ) inline(substr(line, len + 1));
239 # Escape lone HTML character
240 } else if ( match( line, /^[&<>"']/) ) {
241 return HTML(substr(line, 1, 1)) inline(substr(line, 2));
243 # continue walk over string
245 return substr(line, 1, 1) inline( substr(line, 2) );
249 function _block( block, LOCAL, st, len, hlvl, htxt, guard, code, indent ) {
250 gsub( /^\n+|\n+$/, "", block );
256 } else if ( AllowHTML && match( block, /(^|\n) ? ? ?(<!--([^-]|-[^-]|--[^>])*(-->|$)|<\?([^\?]|\?[^>])*(\?>|$)|<![A-Z][^>]*(>|$)|<!\[CDATA\[([^\]]|\][^\]]|\]\][^>])*(\]\]>|$))/) ) {
257 len = RLENGTH; st = RSTART;
258 return _block(substr(block, 1, st - 1)) substr(block, st, len) _block(substr(block, st + len));
261 } else if ( AllowHTML && match( tolower(block), /(^|\n) ? ? ?<\/?(address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|h[123456]|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|optgroup|option|p|param|section|source|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul)([[:space:]\n>]|\/>)([^\n]|\n[ \t]*[^\n])*(\n[[:space:]]*\n|$)/) ) {
262 len = RLENGTH; st = RSTART;
263 return _block(substr(block, 1, st - 1)) substr(block, st, len) _block(substr(block, st + len));
266 } else if ( AllowHTML && match( tolower(block), /(^|\n) ? ? ?<(script|pre|style)([[:space:]\n>]).*(<\/script>|<\/pre>|<\/style>|$)/) ) {
267 len = RLENGTH; st = RSTART;
268 match( tolower(substr(block, st, len)), /(<\/script>|<\/pre>|<\/style>)/);
269 len = RSTART + RLENGTH;
270 return _block(substr(block, 1, st - 1)) substr(block, st, len) _block(substr(block, st + len));
273 } else if ( AllowHTML && match( block, /^ ? ? ?(<\/[A-Za-z][A-Za-z0-9-]*[[:space:]]*>|<[A-Za-z][A-Za-z0-9-]*([[:space:]]+[A-Za-z_:][A-Za-z0-9_\.:-]*([[:space:]]*=[[:space:]]*([[:space:]"'=<>`]+|"[^"]*"|'[^']*'))?)*[[:space:]]*\/?>)([[:space:]]*\n)([^\n]|\n[ \t]*[^\n])*(\n[[:space:]]*\n|$)/) ) {
274 len = RLENGTH; st = RSTART;
275 return substr(block, st, len) _block(substr(block, st + len));
278 } else if ( match( block, /(^|\n) ? ? ?((\* *){3,}|(- *){3,}|(_ *){3,})($|\n)/) ) {
279 len = RLENGTH; st = RSTART;
280 return _block(substr(block, 1, st - 1)) "<hr />\n" _block(substr(block, st + len));
282 # Blockquote (leading >)
283 } else if ( match( block, /^> /) ) {
284 match( block, /(^|\n)[[:space:]]*(\n|$)/ ) || match(block, /$/);
285 len = RLENGTH; st = RSTART;
286 return "<blockquote>\n" _block( gensub( /(^|\n)> /, "\n", "g", substr(block, 1, st - 1) ) ) "</blockquote>\n\n" \
287 _block( substr(block, st + len) );
289 # Line Blocks (pandoc)
290 } else if ( match(block, /^\| [^\n]*(\n|$)(\| [^\n]*(\n|$)|[ \t]+[^\n[:space:]][^\n]*(\n|$))*/) ) {
291 len = RLENGTH; st = RSTART;
292 code = substr(block, 1, len);
293 gsub(/\n[[:space:]]+/, " ", code);
294 gsub(/\n\| /, "\n", code);
295 gsub(/^\| |\n$/, "", code);
296 return "<div class=\"line-block\">" gensub(/\n/, "<br />\n", "g", inline( code )) "</div>\n" \
297 _block( substr( block, len + 1) );
299 # Indented Code Block
300 } else if ( match(block, /^( |\t)[^\n]+(\n|$)(( |\t)[^\n]+(\n|$)|[ \t]*(\n|$))*/) ) {
301 len = RLENGTH; st = RSTART;
302 code = substr(block, 1, len);
303 gsub(/(^|\n)( |\t)/, "\n", code);
304 gsub(/^\n|\n+$/, "", code);
305 return "<pre><code>" HTML( code ) "</code></pre>\n" \
306 _block( substr( block, len + 1 ) );
308 # Fenced Code Block (pandoc)
309 } else if ( match( block, /^(~~~+|```+)/ ) ) {
310 guard = substr( block, 1, RLENGTH );
311 code = gensub(/^[^\n]+\n/, "", 1, block);
312 if ( match(code, "(^|\n)" guard "+(\n|$)" ) ) {
313 len = RLENGTH; st = RSTART;
314 return "<pre><code>" HTML( substr(code, 1, st - 1) ) "</code></pre>\n" \
315 _block( substr( code, st + len ) );
317 match( block, /(^|\n)[[:space:]]*(\n|$)/ ) || match( block, /$/ );
318 len = RLENGTH; st = RSTART;
319 return "<p>" inline( substr(block, 1, st - 1) ) "</p>\n" \
320 _block( substr(block, st + len) );
324 } else if ( match( block, "^ ? ? ?[-+*][ \t]+[^\n]+(\n|$)" \
325 "(([ \t]*\n)* ? ? ?[-+*][ \t]+[^\n]+(\n|$)" \
326 "|([ \t]*\n)*( ? ? ?\t| +)[^\n]+(\n|$)" \
327 "|[^\n]+(\n|$))*" ) ) {
328 list = substr( block, 1, RLENGTH);
329 block = substr( block, RLENGTH + 1);
330 indent = length( gensub(/[-+*][ \t]+[^\n]+.*$/, "", 1, list) );
332 gsub("(^|\n) {0," indent "}", "\n", list);
333 return "\n<ul>\n" _list( substr(list, 2) ) "</ul>\n" _block( block );
336 } else if ( match( block, "^ ? ? ?([0-9]+|#)\\.[ \t]+[^\n]+(\n|$)" \
337 "(([ \t]*\n)* ? ? ?([0-9]+|#)\\.[ \t]+[^\n]+(\n|$)" \
338 "|([ \t]*\n)*( ? ? ?\t| +)[^\n]+(\n|$)" \
339 "|[^\n]+(\n|$))*" ) ) {
340 list = substr( block, 1, RLENGTH);
341 block = substr( block, RLENGTH + 1);
342 indent = length( gensub(/([0-9]+|#)\.[ \t]+[^\n]+.*$/, "", 1, list) );
344 gsub("(^|\n) {0," indent "}", "\n", list);
345 return "\n<ol>\n" _list( substr(list, 2) ) "</ol>\n" _block( block );
347 # First Order Heading
348 } else if ( match( block, /^[^\n]+\n===+(\n|$)/ ) ) {
350 return "<h1>" inline( gensub( /\n.*$/, "", "g", block ) ) "</h1>\n\n" \
351 _block( substr( block, len + 1 ) );
353 # Second Order Heading
354 } else if ( match( block, /^[^\n]+\n---+(\n|$)/ ) ) {
356 return "<h2>" inline( gensub( /\n.*$/, "", "g", block ) ) "</h2>\n\n" \
357 _block( substr( block, len + 1) );
360 } else if ( match( block, /^#{1,6}[[:space:]]*[^\n]+([[:space:]]*#*)(\n|$)/ ) ) {
362 hlvl = length( gensub( /^(#{1,6}).*$/, "\\1", "g", block ) );
363 htxt = gensub( /[[:space:]]*#*$/, "", "1", gensub( /^#{1,6}[[:space:]]*([^\n]+)([[:space:]]*#*)\n.*$/, "\\1", "g", block ) )
364 return "<h" hlvl ">" inline( htxt ) "</h" hlvl ">\n\n" \
365 _block( substr( block, len + 1) );
369 match( block, /(^|\n)[[:space:]]*(\n|$)/ ) || match( block, /$/ );
370 len = RLENGTH; st = RSTART;
371 return "<p>" inline( substr(block, 1, st - 1) ) "</p>\n" \
372 _block( substr(block, st + len) );
376 function _list( block, last, LOCAL, p) {
377 if ( ! length(block) ) return "";
378 gsub(/^([-+*]|[0-9]+\.|#\.)( ? ? ?|\t)/, "", block)
380 # slice next list item from input
381 if ( match( block, /\n([-+*]|[0-9]+\.|#\.)[ \t]+[^\n]+/) ) {
382 p = substr( block, 1, RSTART);
383 block = substr( block, RSTART + 1);
385 p = block; block = "";
387 sub( /\n +([-+*]|[0-9]+\.|#\.)/, "\n&", p );
389 # if this should be a paragraph item
390 # either previous item (last) or current item (p) contains blank lines
391 if (match(last, /\n[[:space:]]*\n/) || match(p, /\n[[:space:]]*\n/) ) {
392 last = p; p = _block(p);
394 last = p; p = _block(p);
395 sub( /^<p>/, "", p );
396 sub( /<\/p>\n/, "", p );
401 if ( p ~ /^\[ \].*/ ) { p = "<input type=checkbox disabled />" substr(p, 4); }
402 else if ( p ~ /^\[[xX]\].*/ ) { p = "<input type=checkbox disabled checked />" substr(p, 4); }
403 else if ( p ~ /^<p>\[ \].*/ ) { p = "<p><input type=checkbox disabled />" substr(p, 7); }
404 else if ( p ~ /^<p>\[[xX]\].*/ ) { p = "<p><input type=checkbox disabled checked />" substr(p, 7); }
405 return "<li>" p "</li>\n" _list( block, last );
410 file = ""; rl_href[""] = ""; rl_title[""] = "";
411 if (ENVIRON["MD_HTML"] == "true") { AllowHTML = "true"; }
413 # Buffering of full file ist necessary, e.g. to find reference links
414 while (getline) { file = file $0 "\n"; }
415 # Clean up MS-DOS line breaks
416 gsub(/\r\n/, "\n", file);
418 # Fill array of reference links
420 re_reflink = "(^|\n) ? ? ?\\[([^]\n]+)\\]: ([^ \t\n]+)(\n?[ \t]+(\"([^\"]+)\"|'([^']+)'|\\(([^)]+)\\)))?(\n|$)";
421 # /(^|\n) ? ? ?\[([^]\n]+)\]: ([^ \t\n]+)(\n?[ \t]+("([^"]+)"|'([^']+)'|\(([^)]+)\)))?(\n|$)/
422 while ( match(f, re_reflink ) ) {
423 rl_id = gensub( re_reflink, "\\2", 1, substr(f, RSTART, RLENGTH) );
424 rl_href[rl_id] = gensub( re_reflink, "\\3", 1, substr(f, RSTART, RLENGTH) );
425 rl_title[rl_id] = gensub( re_reflink, "\\5", 1, substr(f, RSTART, RLENGTH) );
426 f = substr(f, RSTART + RLENGTH);
427 rl_title[rl_id] = substr( rl_title[rl_id], 2, length(rl_title[rl_id]) - 2 );
428 if ( rl_href[rl_id] ~ /<.*>/ ) rl_href[rl_id] = substr( rl_href[rl_id], 2, length(rl_href[rl_id]) - 2 );
430 # Clear reflinks from File
431 while( gsub(re_reflink, "\n", file ) );
432 # for (n in rl_href) { debug(n " | " rl_href[n] " | " rl_title[n] ); }
434 # Run Block Processing -> The Actual Markdown!
435 printf "%s", _block( file );