Major YAR!!

Moved the post download code from the SuckIt's to the main script, and
combined what's left.

Minor icon tweak.

Think I have almost figured out the HTML and CSS to get the layout to
behave.

Major rearrange of the main script.

Use more of lunamark and RE.

Moar colours!

Solved the generic colour shenanigans.

Solved a few other bugs.

Hidden directories.

Lots of things I forgot about.

1 files changed, 422 insertions, 240 deletions

diff --git a/notYetAnotherWiki.lua b/notYetAnotherWiki.lua
index bddf63a..88a69fe 100755
--- a/notYetAnotherWiki.lua
+++ b/notYetAnotherWiki.lua
@@ -1,77 +1,49 @@
 #!/usr/bin/env luajit
 
--- Read the README file for what this is all about.  If there is no README or similar, then you can find the link to the source below.
+--[[ Read the README file for what this is all about.
+     If there is no README or similar, then you can find the link to the source below.
 
-local lunamark = require("lunamark")    -- https://github.com/jgm/lunamark
+    Normally I define functions and globals at the top, but here I'm interleaving them.
+]]
 
-local opts = {
-    layout='compact',
--- This list is copied from the lunamark source code, until I discover a way to discover it.  The descriptions are useful to.
-    containers=false,            -- Put sections in containers (e.g. div or section tags)
-    slides=false,                -- Like containers, but do not nest them
-    startnum=true,               -- Start number of an ordered list is significant
-    smart=false,                 -- Smart typography (quotes, dashes, ellipses)
-    preserve_tabs=true,          -- Don't expand tabs to spaces
-    notes=true,                  -- Footnotes
-    inline_notes=true,           -- Inline footnotes
-    definition_lists=true,       -- Definition lists
-    citations=true,              -- Citations
-    citation_nbsps=true,         -- Turn spacing into non-breaking spaces in citations
-    fenced_code_blocks=true,     -- Fenced code blocks
-    lua_metadata=true,           -- Lua metadata
-    pandoc_title_blocks=true,    -- Pandoc style title blocks
-    hash_enumerators=true,       -- may be used as ordered list enumerator
-    require_blank_before_blockquote=false,
-    require_blank_before_header=false,
-    require_blank_before_fenced_code_block=false,
-    fancy_lists=true,            -- Pandoc style fancy lists
-    task_list=true,              -- GitHub-Flavored Markdown task list
-    strikeout=true,              -- Strike-through with double tildes
-    mark=true,                   -- Highlight with double equals
-    subscript=true,              -- Subscripted text between tildes
-    superscript=true,            -- Superscripted text between circumflexes
-    bracketed_spans=true,        -- Spans with attributes
-    fenced_divs=true,            -- Divs with attributes
-    raw_attribute=true,          -- Raw pass-through on code elements
-    fenced_code_attributes=true, -- Fenced code block attributes
-    link_attributes=true,        -- Link attributes
-    pipe_tables=true,            -- PHP Markdown Extra pipe table support
-    table_captions=true,         -- Table caption syntax extension
-    header_attributes=true,      -- Header attributes
-    line_blocks=true,            -- Line blocks
-    escaped_line_breaks=true,    -- Pandoc-style escaped hard line breaks
+local Lunamark = require("lunamark")    -- https://github.com/jgm/lunamark
+local Lpeg = require("lpeg")            -- https://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html  Lunamark uses this, so we can to.
+local RE = require("re")                -- Part of lpeg.  https://www.inf.puc-rio.br/~roberto/lpeg/re.html
+
+
+---------------------------------------------------------------------------------
+-- Some global data.
+
+local GlobalMetaData = {
+    dlr = '$', perc = '%',
+    devuanCinnabarDark   = '#310202',    devuanCinnabarLight  = '#510505',
+    devuanDarkPurpyDark  = '#33313b',    devuanDarkPurpyLight = '#3c3a45',
+    devuanDeepSeaDark    = '#132f40',    devuanDeepSeaLight   = '#1a4562',
+    devuanSaphireDark    = '#004489',    devuanSaphireLight   = '#00509f',
+    devuanDevuanDark     = '#000000',    devuanDevuanLight    = '#ffffff',
+    -- HTML link colours.  Naturally HTML5 deprecated the simple version, replacing it with less simple CSS.
+    -- <body> has alink, link, vlink; CSS has active, link, visited, and hover.
+    devuanDevuanalink    = '#03a4ff',    devuanDevuanlink     = '#0076b6',    devuanDevuanvlink    = '#6aa4db',   devuanDevuanhlink   = '#03a4ff',
+    devuanSDevuanalink   = '#98c3db',    devuanSDevuanlink    = '#ffffff',    devuanSDevuanvlink   = '#ffffff',   devuanSDevuanhlink  = '#98c3db',
+    karenPurple          = '#8800ff',    onefangPurple        = '#cc00ff',
+    PinkFloyd            = '#AA00AA',    DeepPurple           = '#220022',      -- From an ancient site of mine, which went from PinkFloyd to DeepPurple as a background gradient.
+    favicon = 'nYAW_icon.png', logo = 'nYAW.png',
+    footer = 'Powered by <a href="https://sledjhamr.org/cgit/notYetAnotherWiki/about/">notYetAnotherWiki</a> version 0.0. &nbsp;',
 }
-local writer = lunamark.writer.html5.new(opts)
-local template = ''
+
+local Files, Subs, xLinks = {}, {}, {}
+
+local Template = ''
 local h = io.open("default.template", 'r')
 if nil ~= h then
-    template = h:read('*a')
+    Template = h:read('*a')
     h:close()
 else
     print('oops! No such file ' .. 'default.template')
 end
--- Can override the various writer functions, there's something for each of the basic HTML elements.
---    Override anything that's generic.
-local parse = lunamark.reader.markdown.new(writer, opts)
-
-
-
--- Some global data.
-local globalData = {
-    ['s'] = '&nbsp;', ['_'] = '&nbsp;', ['dlr'] = '&dollar;',
-    ['devuanCinnabarDark']   = '#310202',    ['devuanCinnabarLight']  = '#510505',
-    ['devuanDarkPurpyDark']  = '#33313b',    ['devuanDarkPurpyLight'] = '#3c3a45',
-    ['devuanDeepSeaDark']    = '#132f40',    ['devuanDeepSeaLight']   = '#1a4562',
-    ['devuanSaphireDark']    = '#004489',    ['devuanSaphireLight']   = '#00509f',
-    ['karenPurple']          = '#8800ff',    ['onefangPurple']        = '#cc00ff',
-    favicon = 'nYAW_icon.png', logo = 'nYAW.png',
-    header = '', home = '', history = '',  footer = 'Powered by <a href="https://sledjhamr.org/cgit/notYetAnotherWiki/about/">notYetAnotherWiki</a> version 0.0. &nbsp;',
-     --menu = '',
-}
-local Files, Subs = {}, {}
-
 
 
+---------------------------------------------------------------------------------
 -- Useful functions, part 0.
 
 -- A simple table.subtable = subtable wont work, you end up with a reference so that changes to the later get applied to the former.
@@ -87,6 +59,15 @@ local derefTable = function(t, strip)
 end
 
 
+local writeString = function(base, body)
+    local a, e = io.open(base, 'w')
+    if nil == a then print('Could not open ' .. base .. ' - ' .. e) else
+        a:write(body)
+        a:close()
+    end
+end
+
+
 -- String together the bits array into a path string.  Or the other way around.  lol
 local stringBits = function(l)
     local bits = {}
@@ -115,14 +96,55 @@ local toFile = function(name, key, value)
         end
         Files[name].path = path
 --      Files[name].body = ''
+        if ("Foswiki" == bits[1]) or ("PmWiki" == bits[1]) then Files[name].ogWiki = bits[1] end
     end
     if nil ~= key         then Files[name][key] = value end
     for i, v in ipairs{'metadata', 'bits', } do
         if nil == Files[name][v] then Files[name][v] = {} end
     end
+    -- Open the files and do the initial cleanups.
+    local body = ''
+    if '.md' ~= string.sub(name, -3, -1) then
+        h = io.open(name .. '.md', 'r')
+        if nil ~= h then
+--          print('Parsing ' .. name .. '.md')
+            body = h:read('*a') ; h:close()
+            -- Deal with my typical double spaced sentence endings, and other things.
+            local result = RE.compile( [=[{~
+ (
+  {[.?!]{"  "}}              -> '%1&nbsp;' /            -- '&nbsp;' gets turned into hex 0xA0 by parse().  So feed it another metadata token that gets translated to &nbsp;.  Seems we now skip this issue.
+  {[\\]{['"|$]}}             -> '%2 ' /                 -- Do the same for fixing the \' \" \| etc mess pandoc left.
+  {"[" {([^]])+} "]{.foswiki" {([^FG}])+} "FG}" }  -> "<span style='color: %3;'>%2</span>" /
+  {"::: {."[A-Za-z_. ]+"}"}  -> '' /
+  {":::"}                    -> '' /
+  {"-noComment-"}            -> ' -- ' /
+  {"{#"[A-Za-z_]+"}"}        -> '' /
+  .
+ )* ~}]=],   { } ):match(body)
+            body = result
+--  {"<!--".*"-->"}          -> '' /
+--  {"[$]"}                  -> '$dlr$' /               -- Replace $, coz otherwise it confuses things later.
+            body = RE.gsub(body, '{[$]}', '$dlr$')
+--  {"[%]"}                  -> '$perc$' /              -- Gotta be done after the %1's above. otherwise screws things up when included above.
+--          body = RE.gsub(body, '{[%]}', '$perc$')     -- Coz otherwise stray % trip up the capture part.
+        end
+        Files[name].body = body
+    end
 end
 local toSub = function(name, key, value)
-    if nil == Subs[name] then Subs[name] = {} end
+    if nil == Subs[name] then
+        local bits, bit = stringBits(name)
+        local path = ''
+        Subs[name] = {}
+        table.insert(bits, bit)
+        Subs[name].bits = bits
+        Subs[name].bit = bit
+        for i, d in ipairs(bits) do
+            if '' ~= path then path = path .. '/' end
+            path = path .. d
+        end
+        Subs[name].path = path
+    end
     if nil ~= key        then Subs[name][key] = value end
     for i, v in ipairs{'metadata', 'bits', 'files', 'subs'} do
         if nil == Subs[name][v] then Subs[name][v] = {} end
@@ -131,32 +153,108 @@ end
 
 
 
+---------------------------------------------------------------------------------
 -- Actually start doing things.
 
 -- Create the base of everything.md here, so it gets picked up as usual in the file scan.
-local body, h = '', io.open('everything.md', 'w')
-h:write('# All the pages\n')
-h:close()
+h = io.open('everything.md', 'w')
+if nil ~= h then
+    h:close()
+else
+    print("Can't open everything.md for writing.")
+end
 
 
--- Scan the subdirectories looking for .md files.
-local directory = arg[1]
+-- Scan the subdirectories looking for our files.
+local Directory = arg[1]
 toSub('')
-if nil == directory then directory = '.' end
-if '.' ~= directory then
-    for l in io.popen('find . -name "*.md" -type f,l -printf "%P\n"'):lines() do
+if nil == Directory then Directory = '.' end
+
+for l in io.popen('find -L ' .. Directory .. ' -name "*.HTM" -type f,l -printf "%P\n"'):lines() do
+--    print('pandoc converting '  .. l .. ' -> ' .. string.sub(l, 1, -4) .. 'md')
+    -- Open the HTM files and do the initial cleanups, then pandoc them.
+    h = io.open(l, 'r')
+    if nil ~= h then
+        local body = h:read('*a') ; h:close()
+        if 'Foswiki' == string.sub(l, 1, 7) then
+            -- Strip out the actual content.
+            local beg, en = RE.find(body, [['<div id="patternMainContents">']]) if nil ~= beg then body = string.sub(body, en + 1)  end
+                  beg, en = RE.find(body, [['<div class="patternContent">']])   if nil ~= beg then body = string.sub(body, en + 1)  end
+                  beg, en = RE.find(body, [['<div class="foswikiTopic">']])     if nil ~= beg then
+                if ' -- ' == string.sub(body, en + 1, en + 4) then
+                    beg, en = RE.find(body, '[%nl]', en + 4)
+                    body = string.sub(body, en + 1)
+                end
+            end
+                  beg, en = RE.find(body, [['<div class="patternInfo">']])                                                      if nil ~= beg then body = string.sub(body, 1, beg - 1) end
+--                beg, en = RE.find(body, [['<div class="foswikiForm foswikiFormStep">']])                                      if nil ~= beg then body = string.sub(body, 1, en + 1) end
+                  beg, en = RE.find(body, [['<div class="foswikiAttachments foswikiFormStep" style="overflow:auto">']])         if nil ~= beg then body = string.sub(body, 1, beg - 1) end
+                  beg, en = RE.find(body, [['<div class="foswikiSearchResultsPager">']])                                        if nil ~= beg then body = string.sub(body, 1, beg - 1) end
+            -- Some clean ups.
+            local result = RE.compile( [[{~
+ (
+  {'class="foswikiCurrentTopicLink"'}     -> blank /
+  {'class="foswikiNewLink"'}     -> blank /
+  {" -- "}                       -> '-noComment-' /
+--  {"-- " ([^%nl])* }             -> blank /
+  {'<div class="foswikiTopic">'} -> blank /
+  {'rel="nofollow"'}             -> blank / {"rel='nofollow'"}             -> blank /
+  {"target='_blank'"}            -> blank /
+  {"</div>" ([%nl])* }           -> blank /
+  {'style="' ([^"])+ '"'}        -> ''    / {"style='" ([^'])+ "'"}        -> '' /
+  .
+ )* ~}]],   { blank = function(a) return '' end } ):match(body)
+            body = result
+--          body = RE.gsub(body, [=[{"<!-- ".*"-->"}]=], '')    -- FIXME
+            writeString(l .. '_NEW', body)
+        elseif 'PmWiki'  == string.sub(l, 1, 6) then
+            local beg, en = RE.find(body, [['<!--PageText-->']])        if nil ~= beg then body = string.sub(body, en + 2)  end
+                  beg, en = RE.find(body, [["div id='wikitext'>"]])     if nil ~= beg then body = string.sub(body, en + 2)  end
+                  beg, en = RE.find(body, [["<div id='printfoot'>"]])           if nil ~= beg then body = string.sub(body, 1, beg - (2 + 9)) end        -- There's a </div> to get rid of to.
+                  beg, en = RE.find(body, [['<!--HTMLFooter-->']])              if nil ~= beg then body = string.sub(body, 1, beg - 2) end
+            local result = RE.compile( [[{~
+ (
+  {"class='categorylink'"}       -> blank /
+  {"class='createlink'"}         -> blank /
+  {"class='createlinktext'"}     -> blank /
+  {"class='escaped'"}            -> blank /
+  {"class='diffmarkup'"}         -> blank /
+  {"class='selflink'"}           -> blank /
+  {"class='urllink'"}            -> blank /
+  {"<div class='vspace'></div><hr /> <a class='wikilink' href='https://wiki.devuan.org?n=Profiles." .* ([%nl])* } -> blank /
+  {"<a class='selflink' href='https://wiki.devuan.org?n=Profiles." .* ([%nl])* } -> blank /
+  {"<div class='vspace'></div><hr />"} -> blank /
+  {"<div class='vspace'></div>"} -> blank /
+  {"class='wikilink'"}           -> blank /
+  {'rel="nofollow"'}             -> blank / {"rel='nofollow'"}             -> blank /
+  {"target='_blank'"}            -> blank /
+  {'style="' ([^"])+ '"'}        -> ''    / {"style='" ([^'])+ "'"}        -> '' /
+  {"<span class='hlt " {([a-z])+} "'></span><pre" }  -> "<pre class='%2'" /
+  .
+ )* ~}]],   { blank = function(a) return '' end } ):match(body)
+            body = result
+            writeString(l .. '_NEW', body)
+        end
+    end
+
+    ok, rslt, status = os.execute('pandoc -f html -t commonmark_x --self-contained ' .. l .. '_NEW' .. ' >' .. string.sub(l, 1, -4) .. 'md')
+end
+
+if '.' ~= Directory then
+    for l in io.popen('find -L . -name "*.md" -type f,l -printf "%P\n"'):lines() do
         toFile(string.gsub(l, '%.md$', ''))
     end
 end
+
 -- Can add in a distant directory to, for putting it's results in the current directory.
-for l in io.popen('find ' .. directory .. ' -name "*.md" -type f,l -printf "%P\n"'):lines() do
+for l in io.popen('find -L ' .. Directory .. ' -name "*.md" -type f,l -printf "%P\n"'):lines() do
     local n = string.gsub(l, '%.md$', '')
     if nil == Files[n] then toFile(n) end
 end
 
 
 -- Gotta figure out all the files and subs first.  File and sub metadata comes along for the ride, coz we need them later.
-local newMeta = {}
+local NewMeta = {}
 for name, file in pairs(Files) do
     local bitter, path = '', ''
     local bits, bit = file.bits, file.bit
@@ -169,7 +267,7 @@ for name, file in pairs(Files) do
     for i, d in ipairs(bits) do
         if '' ~= path then path = path .. '/' end
         path = path .. d
-        toSub(path, 'bits', derefiTable(bits, true))
+        toSub(path)
         if i < ln               then
             Subs[path].subs[bits[i + 1]] = bits[i + 1]
             table.remove(Subs[path].bits, #bits)
@@ -178,121 +276,45 @@ for name, file in pairs(Files) do
 
     if '.md' == string.sub(name, -3, -1) then
         -- This is a metadata only file, no content, stash the matadata.
-        for l in io.open(name .. '.md'):lines() do
-            for k, v in string.gmatch(l, "(%w+)%s*=%s*(.+)") do 
-                if nil == v then
-                    print(name .. ' ' .. k)
-                else
-                    metadata[k] = v
+        local h1 = io.open(name .. '.md')
+        if nil == h1 then print('Could not open ' .. name .. '.md') else
+            for l in h1:lines() do
+                for k, v in string.gmatch(l, "(%w+)%s*=%s*(.+)") do 
+                    if nil == v then
+                        print(name .. ' ' .. k)
+                    else
+                        metadata[k] = v
+                    end
                 end
             end
         end
         if '.md' == name                          then toSub(path, 'metadata', metadata)
         elseif '/.md' == string.sub(name, -4, -1) then toSub(path, 'metadata', metadata)
 --      else                                           toFile(string.sub(name, 1, -4), 'metadata', metadata)
-        else                                            newMeta[string.sub(name, 1, -4)] = metadata
+        else                                            NewMeta[string.sub(name, 1, -4)] = metadata     -- Coz we can't add to Files here.
         end
         Files[name] = nil
     end
 end
 
--- FIXTHEM - Lua doesn't like modifying the thing you are pair()ing, like we want to do in the last loop.
-for name, file in pairs(newMeta) do
+-- FIXED - Lua doesn't like modifying the thing you are pair()ing, like we want to do in the last loop.
+for name, file in pairs(NewMeta) do
     if nil == Files[name] then toFile(name) end
-    if nil == Files[name].metadata then Files[name].metadata = {} end
     for k, v in pairs(file) do
         if nil == Files[name].metadata[k] then Files[name].metadata[k] = v end
     end
 end
 
--- Open the files and do the iniital cleanups.
+-- Fix up subs now we have all the file bits.
 for name, file in pairs(Files) do
     if '.md' ~= string.sub(name, -3, -1) then
---      print('Parsing ' .. name .. '.md')
-        h = io.open(name .. '.md', 'r')
-        if nil ~= h then
-            body = h:read('*a') ; h:close()
-            local f0, f1, token
-            -- Deal with my typical double spaced sentence endings.
-            local result = ''
-            local start = 1
-            repeat
-                f0, f1, token = string.find(body, '([%.%?%!] )', start)
-                if fail ~= f0 then
-                    result = result .. string.sub(body, start, f0)
-                    if ' ' == string.sub(body, f1 + 1, f1 + 1) then
-                        result = result .. '$s$ '       -- '&nbsp;' gets turned into hex 0xA0 by parse().  So feed it another metadata token that gets translated to &nbsp;.
-                    else
-                        result = result .. ' '
-                    end
-                    start = f1 + 1
-                end
-            until fail == f0
-            result = result .. string.sub(body, start)
-            -- Do the same for fixing the \' \" \| etc mess pandoc left that sed can't fix.
-            body = result
-            result = ''
-            start = 1
-            repeat
-                f0, f1, token = string.find(body, '(%\\[%"%\'%|%$])', start)
-                if fail ~= f0 then
-                    result = result .. string.sub(body, start, f0 - 1)
-                    start = f1
-                end
-            until fail == f0
-            result = result .. string.sub(body, start)
-            body = result
-        end
-        Files[name].body = body
-        table.insert(Subs[Files[name].path].files, Files[name].bit)
-    end
-end
-
-
-
--- Figure out the original title and link for the original wiki.
--- This functions assume the above file and sub scan has completed.
-local whichWiki = function(metadata)
-    local title, link = '', ''
-    if 'PmWiki'  == metadata.ogWiki then
-        title = '[' .. metadata.ogBase .. '.'    .. metadata.ogFile .. ']'
-        link  = '(' .. metadata.ogURL  .. '/?n=' .. metadata.ogBase .. '.' .. metadata.ogFile .. ')'
-    end
-    if 'Foswiki' == metadata.ogWiki then
-        title = '[' .. metadata.ogBase .. '/'    .. metadata.ogFile .. ']'
-        link  = '(' .. metadata.ogURL  .. '/'    .. metadata.ogBase .. '/' .. metadata.ogFile .. ')'
-    end
-    return title, link
-end
-
-
--- Create an "everything" page, for URL links to every file.HTML.
-local bdy, h = Files['everything'].body, io.open('everything.md', 'a+')
-bdy = bdy .. '\n\n| page | converted | original page | last edited UTC | \n| ---- | --------- | ------- | --------------- | '
-pages = {}
-for name, file in pairs(Files) do
-    local metadata = derefTable(Files[name].metadata, true)
-    if 'everything' ~= name then
-        local ln, fw, pw, ts = 'DUNNO', '', '', ''
-        local title, link = whichWiki(metadata)
-        if 'PmWiki'  == metadata.ogWiki then pw = 'PmWiki  ' .. title .. link end
-        if 'Foswiki' == metadata.ogWiki then fw = 'Foswiki ' .. title .. link end
-        if nil       ~= metadata.timestamp then ts = metadata.timestamp end
-        if nil ~= file.bit   then ln = file.bit
-        end
-        table.insert(pages, '\n| ' .. name .. ' | [' .. ln .. '](<' .. name .. '.HTML>) | ' .. fw .. '  ' .. pw .. ' | ' .. ts .. ' |')
+        table.insert(Subs[file.path].files, file.bit)
     end
 end
-table.sort(pages, function(a, b) return (string.lower(a) < string.lower(b))  end)
-for i, f in ipairs(pages) do
-    bdy = bdy .. f
-end
-h:write(bdy)
-h:close()
-toFile('everything', 'body', parse(bdy))
 
 
 
+---------------------------------------------------------------------------------
 -- These functions assume the above file and sub scan has completed.
 
 -- Which page in this directory should we show?
@@ -323,93 +345,246 @@ local whichPage = function(f)
 end
 
 
+-- Figure out the original title and link for the original wiki.
+local whichWiki = function(metadata)
+    local title, link = '', ''
+    if 'PmWiki'  == metadata.ogWiki then
+        title = metadata.ogBase .. '.'    .. metadata.ogFile
+        link  = metadata.ogURL  .. '/?n=' .. metadata.ogBase .. '.' .. metadata.ogFile
+    end
+    if 'Foswiki' == metadata.ogWiki then
+        title = metadata.ogBase .. '/'    .. metadata.ogFile
+        link  = metadata.ogURL  .. '/'    .. metadata.ogBase .. '/' .. metadata.ogFile
+    end
+    return title, link
+end
+
+
 -- Calculate a link from the source directory to the destination directory.
 local linkFrom = function(source, dest)
+    -- Evil hacks!
+    if 'Profiles' == dest then dest = 'PmWiki/Profiles' end
+    if 'Onefang'  == dest then dest = 'PmWiki/Onefang' end
+
+    if source == dest then return '' end
     local depth = 0
-    local link = ''
+    local lnk = ''
     if source ~= dest then
-        for i, v in ipairs(Subs[source].bits) do
-            if v ~= Subs[dest].bits[i] then
-                depth = i
-                break
+        if nil == Subs[source] then
+--          print('!!!! No idea where to find source ' .. source)
+            return 'DUNNO'
+        end
+        if nil == Subs[dest] then
+            if dest == Subs[source].bit then
+                return ''
+            else
+--              print('!!!! No idea where to find dest   ' .. dest .. ' from ' .. Subs[source].path .. ' / ' .. Subs[source].bit)
+                return 'DUNNO'
             end
         end
+        local s = Subs[source].bits
+        local d = Subs[dest].bits
+        local sl = #s
+        local dl = #d
 
-        if #(Subs[dest].bits) >= #(Subs[source].bits) then
-            depth = #(Subs[source].bits)
-            for i, v in ipairs(Subs[dest].bits) do
-                if i > depth then
-                    if '' ~= link then link = link .. '/' end
-                    link = link .. Subs[dest].bits[i]
+        if 0 == dl then
+            depth = sl
+        else
+            for i, v in ipairs(s) do
+                if (nil == d[i]) or (v ~= d[i]) then
+                    depth = i
+                    break
                 end
             end
-            if '' ~= link then link = link .. '/' end
-        else
-            depth = #(Subs[source].bits) - depth
-            depth = depth + 1
-            link = string.rep('../', depth)
         end
+        -- depth is where they DON'T match.
+        local m = depth - 1
+        if 0 > m then m = 0 end
+        lnk = string.rep('../', sl - m)
+        if 0 ~= (m + 1) then lnk = lnk .. table.concat(d, '/', m + 1, dl) end
     end
-    return link
+    return lnk
 end
 
 
+
+---------------------------------------------------------------------------------
 -- More of this actually doing things nonsense.
 
+-- Create an "everything" page, for URL links to every file.HTML.
+local Bdy = '# All the pages\n\n| page | converted | original page | last edited UTC | \n| ---- | --------- | ------- | --------------- | '
+Pages = {}
+for name, file in pairs(Files) do
+    local metadata = derefTable(Files[name].metadata, true)
+    if ('everything' ~= name) then
+        local ln, fw, pw, ts = 'DUNNO', '', '', ''
+        local title, link = whichWiki(metadata)
+        link = string.gsub(link, 'https://', 'HTTPS://')        -- Prevent this one from being converted.
+        if 'PmWiki'  == metadata.ogWiki then pw = 'PmWiki  [' .. title .. '](' .. link .. ')' end
+        if 'Foswiki' == metadata.ogWiki then fw = 'Foswiki [' .. title .. '](' .. link .. ')' end
+        if nil       ~= metadata.timestamp then ts = metadata.timestamp end
+        if nil ~= file.bit   then ln = file.bit
+        end
+        table.insert(Pages, '\n| ' .. name .. ' | [' .. ln .. '](<' .. name .. '.HTML>) | ' .. fw .. '  ' .. pw .. ' | ' .. ts .. ' |')
+
+        -- Track our external links.
+        if (nil ~= metadata.ogBase) and (nil ~= metadata.ogFile) then
+            local n = metadata.ogBase
+            if 'PmWiki'  == metadata.ogWiki then n = n .. '.' else n = n .. '/' end
+            xLinks[n .. metadata.ogFile] = file.path
+        end
+    end
+end
+table.sort(Pages, function(a, b) return (string.lower(a) < string.lower(b))  end)
+for i, f in ipairs(Pages) do
+    Bdy = Bdy .. f
+end
+h = io.open('everything.md', 'a+')
+if nil ~= h then
+    h:write(Bdy)
+    h:close()
+else
+    print("Can't open everything.md for writing.")
+end
+toFile('everything', 'body', Bdy)
+
+
 -- Loop through Subs, doing whichPage and inheritance.
+-- It gets to testing/even/deeper BEFORE it gets to testing/even sometimes.  So sort them.
+SUBS = {}
 for name, sub in pairs(Subs) do
+    table.insert(SUBS, sub)
+end
+table.sort(SUBS, function(a, b) return (string.lower(a.path) < string.lower(b.path))  end)
+for n, sub in pairs(SUBS) do
+    local name = sub.path
     sub.whichPage = whichPage(name)
     local metadata = sub.metadata
     for i, s in pairs(sub.subs) do
         local nm = i
         if '' ~= name then nm = name .. '/' .. i end
-        ss = Subs[nm]
         for k, v in pairs(metadata) do
-            if nil == ss.metadata[k] then
+            if nil == Subs[nm].metadata[k] then
                 if ('favicon' == k) or ('logo' == k) then
-                    ss.metadata[k] = linkFrom(nm, name) .. v
+                    Subs[nm].metadata[k] = linkFrom(nm, name) .. v
                 else
-                    ss.metadata[k] = v
+                    if 'hidden' ~= k then       -- Don't inherit hidden.
+                        Subs[nm].metadata[k] = v
+                    end
                 end
             end
         end
     end
 end
 
--- Loop through the files we found and actually create their HTML files.
+-- Files inheritance.
 for name, file in pairs(Files) do
-    local path, result = '', ''
-    local body, metadata = Files[name].body, derefTable(Files[name].metadata, true)
-    local bits, bit = Files[name].bits, Files[name].bit
-    local ln = #bits
-
-    path = table.concat(bits, '/', 1, ln)
-
-    if '' ~= body then
-        -- Inherit stuff from sub and global.
-        local mdata = Subs[path].metadata
+    if '' ~= file.body then
+        local mdata = Subs[file.path].metadata
         for k, v in pairs(mdata) do
             if nil == file.metadata[k] then
-                file.metadata[k] = v
+                Files[name].metadata[k] = v
             end
         end
-        Files[name].metadata = file.metadata
-        metadata = derefTable(Files[name].metadata, true)
-        for             m, x in pairs(globalData)               do if nil == metadata[m] then metadata[m] = x end end
+    end
+end
 
-        if nil ~= metadata.ogURL then
-            local title, link = whichWiki(metadata)
-            body = body .. '\n\n---\n\n[Original page]' .. link .. ', maybe you can edit it.\n'
+
+---------------------------------------------------------------------------------
+-- Setup the lunarmark stuff.
+local LunamarkOpts = {
+    layout='compact',
+-- This list is copied from the lunamark source code, until I discover a way to discover it.  The descriptions are useful to.
+    containers=false,            -- Put sections in containers (e.g. div or section tags)
+    slides=false,                -- Like containers, but do not nest them
+    startnum=true,               -- Start number of an ordered list is significant
+    smart=false,                 -- Smart typography (quotes, dashes, ellipses)
+    preserve_tabs=true,          -- Don't expand tabs to spaces
+    notes=true,                  -- Footnotes
+    inline_notes=true,           -- Inline footnotes
+    definition_lists=true,       -- Definition lists
+    citations=true,              -- Citations
+    citation_nbsps=true,         -- Turn spacing into non-breaking spaces in citations
+    fenced_code_blocks=true,     -- Fenced code blocks
+    lua_metadata=true,           -- Lua metadata
+    pandoc_title_blocks=true,    -- Pandoc style title blocks
+    hash_enumerators=true,       -- may be used as ordered list enumerator
+    require_blank_before_blockquote=false,
+    require_blank_before_header=false,
+    require_blank_before_fenced_code_block=false,
+    fancy_lists=true,            -- Pandoc style fancy lists
+    task_list=true,              -- GitHub-Flavored Markdown task list
+    strikeout=true,              -- Strike-through with double tildes
+    mark=true,                   -- Highlight with double equals
+    subscript=true,              -- Subscripted text between tildes
+    superscript=true,            -- Superscripted text between circumflexes
+    bracketed_spans=true,        -- Spans with attributes
+    fenced_divs=true,            -- Divs with attributes
+    raw_attribute=true,          -- Raw pass-through on code elements
+    fenced_code_attributes=true, -- Fenced code block attributes
+    link_attributes=true,        -- Link attributes
+    pipe_tables=true,            -- PHP Markdown Extra pipe table support
+    table_captions=true,         -- Table caption syntax extension
+    header_attributes=true,      -- Header attributes
+    line_blocks=true,            -- Line blocks
+    escaped_line_breaks=true,    -- Pandoc-style escaped hard line breaks
+}
+local Writer = Lunamark.writer.html5.new(LunamarkOpts)
+
+-- Can override the various writer functions, there's something for each of the basic HTML elements.
+local Context = {}                      -- Coz can't otherwise pass context through to the deeper Lunamark functions I'm overriding.
+-- Fix up the links.
+local OgWriterLink = Writer.link        -- So we can call the original from within mine, we are just changing the URL.
+function Writer.link(lab, url, tit)
+    if ('https://wiki.devuan.org/' ~= url) and ('https://fos.wiki.devuan.org/' ~= url) then
+        local label = lab
+        local uri = url
+        if 'string' ~= type(lab) then label = type(lab) end
+        for i, p in ipairs{'https://wiki.devuan.org/?n=', 'https://wiki.devuan.org?n=', 'https://fos.wiki.devuan.org/'} do
+            if p == string.sub(url, 1, #p) then
+                local ur = string.sub(url, #p + 1)
+-- TODO - could probably replace some of this mess with RE.gsub() and friends.  Meh, it works.
+                local f4, f5, tk1 = string.find(ur, '?', 1, true)
+                if fail ~= f4 then
+                    local u = string.sub(ur, 1, f4 - 1)
+                    ur = u
+                end
+                if nil == Context.path then
+                    url = string.gsub(ur, '%.', '/', 1)
+                else
+                    local xlnk = xLinks[string.gsub(ur, '%..*', '', 1)]
+                    if nil == xlnk then xlnk = string.gsub(ur, '%..*', '', 1) end
+                    if '' ~= Context.path then xlnk = linkFrom(Context.path, xlnk) end
+                    if '/' == string.sub(xlnk, 1, 1) then xlnk = string.sub(xlnk, 2) end
+                    if ('' ~= xlnk) and ('/' ~= string.sub(xlnk, -1)) then xlnk = xlnk .. '/' end
+                    if 'DUNNO/' == xlnk then print('OOPS! Page not found - ' .. Context.path .. ' / ' .. Context.bit .. ' ' .. url .. ' -> ' .. xlnk .. ' ' .. string.gsub(ur, '.*%.', '', 1) .. '.HTML') end
+                    url = xlnk .. string.gsub(ur, '.*%.', '', 1) .. '.HTML'
+                end
+            end
         end
+    end
+    return OgWriterLink(lab, url, tit)
+end
+local Parse = Lunamark.reader.markdown.new(Writer, LunamarkOpts)
+
 
+---------------------------------------------------------------------------------
+-- Loop through the files we found and actually create their HTML files.
+for name, file in pairs(Files) do
+    local body, metadata = Files[name].body, derefTable(Files[name].metadata, true)
+    local bits, bit = Files[name].bits, Files[name].bit
+    local ln = #bits
+    local result = ''
+
+    if '' ~= body then
         -- Figure out this pages trail links.
-        metadata.home = linkFrom(path, '') .. Subs[''].whichPage
+        metadata.home = linkFrom(file.path, '') .. Subs[''].whichPage
         metadata.trail = ''
         for i, b in ipairs(bits) do
             local p = table.concat(bits, '/', 1, i)
             if i < #bits then
-                metadata.trail = metadata.trail .. '<a href="' .. linkFrom(path, p) .. Subs[p].whichPage .. '">' .. b .. '</a> &#x1f463; &nbsp; '
-                linkFrom(path, table.concat(bits, '/', 1, i))
+                metadata.trail = metadata.trail .. '<a href="' .. linkFrom(file.path, p) .. Subs[p].whichPage .. '">' .. b .. '</a> &#x1f463; &nbsp; '
+                linkFrom(file.path, table.concat(bits, '/', 1, i))
             else
                 metadata.trail = metadata.trail ..  b .. ' &nbsp; '
             end
@@ -418,28 +593,30 @@ for name, file in pairs(Files) do
         -- Figure out this pages header links.
         metadata.header = ''
         subs = {}
-        for i, f in pairs(Subs[path].subs) do
+        for i, f in pairs(Subs[file.path].subs) do
             table.insert(subs, f)
         end
         table.sort(subs, function(a, b) return (string.lower(a) < string.lower(b))  end)
         for i, f in ipairs(subs) do
-            local pth = path
-            if '' ~= path then pth = path .. '/' end
-            metadata.header = metadata.header .. '<a href="' .. f .. '/' .. whichPage(pth .. f) .. '">' .. f .. '</a> &nbsp; '
+            local pth = file.path
+            if '' ~= file.path then pth = file.path .. '/' end
+            if 'true' ~= Subs[pth .. f].metadata.hidden then
+                metadata.header = metadata.header .. '<a href="' .. f .. '/' .. whichPage(pth .. f) .. '">' .. f .. '</a> &nbsp; '
+            end
         end
 
         -- Figure out this pages menu links.
         metadata.menu = ''
         if nil == metadata.title then metadata.title = bit end
-        if nil ~= Subs[path].files then table.sort(Subs[path].files, function(a, b) return (string.lower(a) < string.lower(b))  end)  end
-        for i, f in ipairs(Subs[path].files) do
+        if nil ~= Subs[file.path].files then table.sort(Subs[file.path].files, function(a, b) return (string.lower(a) < string.lower(b))  end)  end
+        for i, f in ipairs(Subs[file.path].files) do
             local title, url = nil, nil
-            if '' == path then
+            if '' == file.path then
                 title = Files[f].metadata.title
                 url   = Files[f].metadata.URL
             else
-                title = Files[path .. '/' .. f].metadata.title
-                url   = Files[path .. '/' .. f].metadata.URL
+                title = Files[file.path .. '/' .. f].metadata.title
+                url   = Files[file.path .. '/' .. f].metadata.URL
             end
             if nil == title then title = f end
             if bit == f then            metadata.menu = metadata.menu .. '<p>' .. title .. '</p>'
@@ -451,42 +628,47 @@ for name, file in pairs(Files) do
         end
 
         -- Figure out this pages footer links.
-        if nil ~= metadata.pagehistory then metadata.history = '<p>Page&nbsp;<a href="' .. metadata.pagehistory .. '/' .. name .. '.md">history</a></p>' end
+        metadata.footer = GlobalMetaData.footer
+        if nil ~= metadata.pagehistory then metadata.history = '<p>Page&nbsp;<a href="' .. metadata.pagehistory .. '/' .. name .. '.md">history</a></p>' else metadata.history = '' end
         if nil ~= metadata.sourcecode   then metadata.footer = '<a href="' .. metadata.sourcecode       .. '">source code</a> &nbsp; &nbsp; ' .. metadata.footer    end
         if nil ~= metadata.feedatom             then metadata.footer = '<a href="' .. metadata.feedatom         ..   '">atom feed</a> &nbsp; &nbsp; ' .. metadata.footer    end
-        if metadata.footer ~= globalData.footer then metadata.footer = 'Web site ' .. metadata.footer end
+        if metadata.footer ~= GlobalMetaData.footer then metadata.footer = 'Web site ' .. metadata.footer end
+        -- Add a link to the original page.
+        if nil ~= metadata.ogURL then
+            local title, link = whichWiki(metadata)
+            link = string.gsub(link, 'https://', 'HTTPS://')    -- Prevent this one from being converted.
+            metadata.footer = '<a href="' .. link .. '">Original page</a>, maybe you can edit it. &nbsp; ' .. metadata.footer
+        end
         metadata.footer = '<p>' .. metadata.footer .. '</p>'
 
         -- Do our own metadata replacement, it's simple and works better.
-        local temp = template
-        local start = 1
-        local f0, f1, token
+        local temp = Template
         -- Toss the body in first, so the scan can deal with it to.
-        f0, f1, token = string.find(temp, '%$(body)%$')
-        if fail ~= f0 then
-            -- NOTE - this is where we actually parse the markup into HTML.
-            temp = string.sub(temp, 1, f0 - 1) .. parse(body) .. string.sub(temp, f1 + 1)
-        end
-        -- The actual metadata replacement scan.
-        result = ''
-        repeat
-            f0, f1, token = string.find(temp, '%$([%w_]+)%$', start)
-            if fail ~= f0 then
-                if nil ~= metadata[token] then result = result .. string.sub(temp, start, f0 - 1) .. metadata[token] end
-                start = f1 + 1
+        -- NOTE - this is where we actually parse the markup into HTML.
+        Context = file
+        local bd, md = Parse(body)              -- The md is a table of extracted metadata, not likely to be any, and we wont do anything with it.
+        bd = RE.gsub(bd, '{[%]}', '$perc$')     -- Coz otherwise stray % trip up the capture part.
+        temp = RE.gsub(temp, '"$body$"', bd)
+        -- The actual metadata replacement.
+        result = RE.compile ('{~ ({[$][A-Za-z_]+[$]} -> meta / .)* ~}',
+        {
+            meta = function(a)
+                a = string.sub(a, 2, -2)
+                local md = metadata[a]
+                if nil == md then
+                    md = GlobalMetaData[a]
+                    if nil == md then
+                        md = a
+                    end
+                end
+                return md
             end
-        until fail == f0
-        result = result .. string.sub(temp, start)
+        } ):match(temp)
 
         -- Write the file.
         if '' ~= result then
-            local base = name .. '.HTML'
 --          print('From ' .. name .. '.md -> ' .. base)
-            local a, e = io.open(base, 'w')
-            if nil == a then print('Could not open ' .. base .. ' - ' .. e) else
-                a:write(result)
-                a:close()
-            end
+            writeString(name .. '.HTML', result)
         end
     end
 end