#!/usr/bin/env -S deno run –allow-read –allow-write –allow-env –allow-run /*
let’s make a site with markdown files
/**/
import { exec } from "node:child_process";
import { parse } from "https://deno.land/std@0.106.0/path/mod.ts";
import { t as sectionsToHentry } from "./translators/section-to-hentry.js";
import { t as hiddenToGone } from "./translators/hidden-to-gone.js";
// import { readFileStr, writeFileStr } from "https://deno.land/std@0.54.0/fs/mod.ts";; // Use appropriate Deno file system module
const readFileStr = Deno.readTextFile
const writeFileStr = Deno.writeTextFile
const stat = Deno.stat
const zip = (...arr) => Array(Math.max(...arr.map(a => a.length))).fill().map((_,i) => arr.map(a => a[i]));
// Extract links from a markdown file
async function extractLinks(filePath) {
const content = await readFileStr(filePath, "utf-8");
const linkRegex = /(?<!\!)\[([^\]]+)\]\(([^\)]+)\)/g;
const links = [];
let match;
while ((match = linkRegex.exec(content)) !== null) {
.push(match[2]);
links
}return links;
}
async function fileListToRemoteMtimeList (fileList, options) {
const sshPipe = makePipe({
cmd: ["ssh", options.serverName, "/bin/bash"],
stdin: "piped",
stdout: "piped",
stderr: "piped",
infile: "fileList",
outfile: "remoteMtimeList"
;
})
return (await sshPipe(fileList.map(x=>`stat -c %Y "${x}"`).join('\n'))).split('\n').slice(0,fileList.length);
}
// Function to get modification times of files in milliseconds since the epoch
async function fileListToLocalMtimeList(fileList) {
const mtimeList = await Promise.all(
.map(async (filePath) => {
fileListtry {
const fileStats = await Deno.stat(filePath);
return +fileStats.mtime/1000; // Modification time in milliseconds
catch (error) {
} console.error(`Error getting stats for ${filePath}: ${error.message}`);
return null; // Return null if there was an error
}
});
)
// Filter out any null values in case of errors
return mtimeList.filter(mtime => mtime !== null);
}
async function runPipe(inputString, options) {
try {
// Prepare the Pandoc command
const process = Deno.run(options);
// Write Markdown content to Pandoc's stdin
await process.stdin.write(new TextEncoder().encode(inputString));
process.stdin.close();
// Capture the output from Pandoc's stdout
const output = await process.output();
const outputString = await (new TextDecoder().decode(output));
// Capture and log any errors from Pandoc's stderr
const errorOutput = await process.stderrOutput();
if (errorOutput.length > 0) {
console.error(`Error from ${options.cmd[0]}: ${new TextDecoder().decode(errorOutput)}`);
else {
} console.log(`🔛 Converted ${options.infile} to ${options.outfile}`);
}
return outputString
process.close();
catch (error) {
} console.error(`Error converting ${options.infile}: ${error.message}`);
}
}
function makePipe(options) {
return (inputString) => runPipe(inputString, options);
}function composeAsync(left,right) {
return async (x) => right(await left(x));
}
async function buildTask(infile, outfile, filter=(x=>x)) {
// Read the contents of the Markdown file
const inputString = await readFileStr(infile);
const outputString = await filter(inputString);
const filteredString = outputString;
/// outstring
await writeFileStr(outfile, filteredString);
}
// Function to convert Markdown to HTML using Pandoc and a template with stdin and stdout
async function convertMarkdownWithPandoc(markdownFile, htmlFile, filters=(x=>x)) {
const pandocFilter = makePipe({
cmd: [
"pandoc",
"--template", "./template.html",
"-f", "markdown",
"-t", "html"
,
]stdin: "piped",
stdout: "piped",
stderr: "piped",
infile: markdownFile,
outfile: htmlFile
;
})return buildTask(markdownFile, htmlFile, composeAsync(pandocFilter, filters) );
}
// Convert markdown files to HTML
// Takes filename
// Returns list of files
async function convertMarkdownToHtml(fileListPath) {
try {
const links = await extractLinks(fileListPath);
const list = [];
for (const link of links) {
const htmlFile = link.trim();
if (htmlFile) {
const escapedPath = htmlFile.replace(/"/g, '\\"');
const markdownFile = escapedPath.replace(/\.html$/, ".md");
const templateFile = "template.html";
.push(markdownFile);
list.push(escapedPath);
list
const [markdownStats, htmlStats, templateStats] = await Promise.all([
.stat(markdownFile),
Deno.stat(htmlFile).catch(() => null),
Deno.stat(templateFile).catch(() => null),
Deno;
])
// factor out to runtaskIfNewer
if (!htmlStats || markdownStats.mtime > htmlStats.mtime || templateStats.mtime > htmlStats.mtime) {
await convertMarkdownWithPandoc(markdownFile, escapedPath, composeAsync(sectionsToHentry, hiddenToGone));
console.log(`🔛 Converted infile to outfile`);
else {
} console.log(`⏭️ ${htmlFile} is up to date. Skipping conversion.`);
}
}
}return list;
catch (error) {
} console.error(`Error reading file: ${error.message}`);
}
}
// Extract image references from HTML and Markdown files
async function extractImageReferences(fileName, imageReferences) {
try {
const content = await Deno.readTextFile(fileName);
const regex = /!\[.*?\]\((.*?) .*?p\)|<img.*?src=["']([^"']+)/g;
let match;
while ((match = regex.exec(content)) !== null) {
const imagePath = match[1] || match[2];
if (imagePath && !imagePath.startsWith("http") && !imagePath.startsWith("data")) {
.add(decodeURIComponent(imagePath));
imageReferences
}
}catch (error) {
} console.warn(`Skipping file ${fileName}: ${error.message}`);
}
}
async function runTasks (indexPath, serverName, remoteSubdirectory) {
const fileNames = await convertMarkdownToHtml(fileListPath);
const imageReferences = new Set();
const createdDirs = new Set();
await Promise.all(fileNames.map((fileName) => extractImageReferences(fileName, imageReferences)));
for (const imagePath of imageReferences) {
const parsedPath = parse(imagePath);
const remoteDir = `${remoteSubdirectory}${parsedPath.dir}`;
const remoteFile = `${remoteSubdirectory}/${parsedPath.base}`;
if (!createdDirs.has(remoteDir)) {
.add(parsedPath.dir);
createdDirs
}
}
const list = [...createdDirs,...fileNames,...imageReferences].filter(x=>x.trim()!=="");
const types = [
...Array([...createdDirs].filter(x=>x.trim()!=="").length).fill('dir'),
...Array([...fileNames].filter(x=>x.trim()!=="").length).fill('text'),
...Array([...imageReferences].filter(x=>x.trim()!=="").length).fill('image'),
;
]
const remoteMtimeList = await fileListToRemoteMtimeList(list.map(x=>`${remoteSubdirectory}${x}`),{serverName});
const localMtimeList = await fileListToLocalMtimeList(list.map(x=>`${x}`));
const isLocalNewer = remoteMtimeList.map((x,i)=>x<localMtimeList[i]);
const zipped = zip(
,
types.map(x=>`${x}`),
list.map(x=>`${remoteSubdirectory}${x}`),
list,
localMtimeList,
remoteMtimeList
isLocalNewer;
)const headers = ["type","localFile","remoteFile","localMtime","remoteMtime","isLocalNewer"];
const manifest = [headers,...zipped];
const commentOut = x => ([ '#' , '' ][ +x.isLocalNewer ]);
const mkdirOrScp = x => ([
`scp "${x.localFile}" "linode:'${x.remoteFile}'"`,
`ssh linode "mkdir -p '${x.remoteFile}'"`
+(x.type === 'dir') ])
][
const shellScriptTemplate = x => (`${ commentOut(x) } ${ mkdirOrScp(x) }`);
const shellScriptContent = zipped
.map( x => Object.fromEntries(zip(headers,x) ) )
.map( shellScriptTemplate )
.join('\n');
await writeFileStr("manifest.tsv", manifest.map(x=>x.join('\t')).join('\n'));
console.log("'manifest.tsv' has been created.");
await writeFileStr("copy_files2.sh", shellScriptContent);
console.log("'copy_files.sh' has been created.");
// console.log([...createdDirs,...fileNames,...imageReferences].join('\n'));
}
// Main function to generate the shell script
async function generateShellScript(fileListPath, serverName, remoteSubdirectory) {
const fileNames = await convertMarkdownToHtml(fileListPath);
const imageReferences = new Set();
await Promise.all(fileNames.map((fileName) => extractImageReferences(fileName, imageReferences)));
// let shellScriptContent = "";
// const createdDirs = new Set();
// for (const fileName of fileNames) {
// const remoteFile = `${remoteSubdirectory}${fileName}`;
// const isNewer = await isRemoteFileNewer(fileName, remoteFile, serverName);
// shellScriptContent += isNewer
// ? `# scp "${fileName}" "${serverName}:'${remoteFile}'"\n`
// : `scp "${fileName}" "${serverName}:'${remoteFile}'"\n`;
// }
// for (const imagePath of imageReferences) {
// const parsedPath = parse(imagePath);
// const remoteDir = `${remoteSubdirectory}${parsedPath.dir}`;
// const remoteFile = `${remoteDir}/${parsedPath.base}`;
// if (!createdDirs.has(remoteDir)) {
// shellScriptContent += `ssh ${serverName} "mkdir -p '${remoteDir}'"\n`;
// createdDirs.add(remoteDir);
// }
// const isNewer = await isRemoteFileNewer(imagePath, remoteFile, serverName);
// shellScriptContent += isNewer
// ? `# scp "${imagePath}" "${serverName}:'${remoteFile}'"\n`
// : `scp "${imagePath}" "${serverName}:'${remoteFile}'"\n`;
// }
// await Deno.writeTextFile("copy_files.sh", shellScriptContent);
// console.log("Shell script 'copy_files.sh' has been created.");
}
const fileListPath = "index.md";
const serverName = "linode";
const remoteSubdirectory = "www/notes/";
runTasks(fileListPath, serverName, remoteSubdirectory);
/*
And that’s how it’s done
todo:
undefined scp "Analog/2e787efac197f283.png" "linode:'www/notes/Analog/2e787efac197f283.png'"
undefined scp "Analog/d46c4f16962752b6.png" "linode:'www/notes/Analog/d46c4f16962752b6.png'"
<audio controls>
<source src="Slovenska%20Rapsodie.ogg" type="audio/ogg"></source>
<source src="Slovenska%20Rapsodie.mp3" type="audio/mpeg"></source>
Your browser does not support the audio tag.
(id3 info?)</audio>
error
Skipping file List of tools on tools.yip.pe.html: No such file or directory (os error 2): readfile 'List of tools on tools.yip.pe.html'
isRemoteFileNewer failed List of tools on tools.yip.pe.html www/notes/List of tools on tools.yip.pe.html NotFound: No such file or directory (os error 2): stat 'List of tools on tools.yip.pe.html'
at async Object.stat (ext:deno_fs/30_fs.js:420:15)
at async isRemoteFileNewer (file:///Volumes/data/Obsidian/build.md:29:23)
at async generateShellScript (file:///Volumes/data/Obsidian/build.md:158:21) {
name: "NotFound",
code: "ENOENT"
}
the script cannot handle links in index that contain html entitites
Error reading file: ENOENT: no such file or directory, stat
error: Uncaught (in promise) TypeError: Cannot read properties of undefined (reading 'map')
await Promise.all(fileNames.map((fileName) => extractImageReferences(fileName, imageReferences)));
^
at generateShellScript (file:///Volumes/data/Obsidian/build.md:150:31)
at eventLoopTick (ext:core/01_core.js:168:7)
isRemoteFileNewer failed image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABYAAAASCAQAAAD1LOamAAAAU0lEQVR42qWSwQ4AIAhC+f+fpkutphS67GZPhQ0gFkEUi/M10I2LwdWKsLhz+8535DZUYWM3W2obPLHLiMIfYiJsbH7AQgYtDq+ZlRgV0tdAbaIHWyDHOfjqTLEAAAAASUVORK5CYII= www/notes/image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABYAAAASCAQAAAD1LOamAAAAU0lEQVR42qWSwQ4AIAhC+f+fpkutphS67GZPhQ0gFkEUi/M10I2LwdWKsLhz+8535DZUYWM3W2obPLHLiMIfYiJsbH7AQgYtDq+ZlRgV0tdAbaIHWyDHOfjqTLEAAAAASUVORK5CYII= NotFound: No such file or directory (os error 2): stat 'image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABYAAAASCAQAAAD1LOamAAAAU0lEQVR42qWSwQ4AIAhC+f+fpkutphS67GZPhQ0gFkEUi/M10I2LwdWKsLhz+8535DZUYWM3W2obPLHLiMIfYiJsbH7AQgYtDq+ZlRgV0tdAbaIHWyDHOfjqTLEAAAAASUVORK5CYII='
at async Object.stat (ext:deno_fs/30_fs.js:420:15)
at async isRemoteFileNewer (file:///Volumes/data/Obsidian/Build.md:32:23)
at async generateShellScript (file:///Volumes/data/Obsidian/Build.md:178:21) {
name: "NotFound",
code: "ENOENT"
}
*/