refactor: replace markdown parser
- replace markdown-it with remark and rehype - add bunch of markdown features
This commit is contained in:
parent
c0195e02fd
commit
ae5ecaaccc
31 changed files with 912 additions and 247 deletions
|
|
@ -34,53 +34,57 @@ export const portfolioData: PortfolioData = {
|
|||
projects: {},
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete previously generated files
|
||||
*/
|
||||
async function main() {
|
||||
/**
|
||||
* Delete previously generated files
|
||||
*/
|
||||
|
||||
try {
|
||||
fs.rmSync("dist", { recursive: true })
|
||||
// eslint-disable-next-line no-empty
|
||||
} catch (err) {}
|
||||
try {
|
||||
fs.rmSync("dist", { recursive: true })
|
||||
// eslint-disable-next-line no-empty
|
||||
} catch (err) {}
|
||||
|
||||
/**
|
||||
* Checking
|
||||
*/
|
||||
/**
|
||||
* Checking
|
||||
*/
|
||||
|
||||
if (!fs.lstatSync(markdownPath).isDirectory())
|
||||
throw Error("Invalid markdown path")
|
||||
if (!fs.lstatSync(markdownPath).isDirectory())
|
||||
throw Error("Invalid markdown path")
|
||||
|
||||
if (!fs.lstatSync(markdownPath + "/posts").isDirectory())
|
||||
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
|
||||
if (!fs.lstatSync(markdownPath + "/posts").isDirectory())
|
||||
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
|
||||
|
||||
if (!fs.lstatSync(markdownPath + "/series").isDirectory())
|
||||
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
|
||||
if (!fs.lstatSync(markdownPath + "/series").isDirectory())
|
||||
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
|
||||
|
||||
/**
|
||||
* Parse
|
||||
*/
|
||||
/**
|
||||
* Parse
|
||||
*/
|
||||
|
||||
// parse markdown
|
||||
recursiveParse(ParseMode.POSTS, markdownPath + "/posts")
|
||||
recursiveParse(ParseMode.SERIES, markdownPath + "/series")
|
||||
recursiveParse(ParseMode.PORTFOLIO, markdownPath + "/projects")
|
||||
// parse markdown
|
||||
await recursiveParse(ParseMode.POSTS, markdownPath + "/posts")
|
||||
await recursiveParse(ParseMode.SERIES, markdownPath + "/series")
|
||||
await recursiveParse(ParseMode.PORTFOLIO, markdownPath + "/projects")
|
||||
|
||||
sortDates()
|
||||
fillTags()
|
||||
parseSeries()
|
||||
generatePortfolioSVGs()
|
||||
sortDates()
|
||||
fillTags()
|
||||
parseSeries()
|
||||
generatePortfolioSVGs()
|
||||
|
||||
/**
|
||||
* Save results
|
||||
*/
|
||||
/**
|
||||
* Save results
|
||||
*/
|
||||
|
||||
fs.writeFileSync(mapFilePath, JSON.stringify(contentMap))
|
||||
fs.writeFileSync(
|
||||
portfolioFilePath,
|
||||
JSON.stringify({
|
||||
...portfolioData,
|
||||
skills: Array.from(portfolioData.skills),
|
||||
})
|
||||
)
|
||||
fs.writeFileSync(mapFilePath, JSON.stringify(contentMap))
|
||||
fs.writeFileSync(
|
||||
portfolioFilePath,
|
||||
JSON.stringify({
|
||||
...portfolioData,
|
||||
skills: Array.from(portfolioData.skills),
|
||||
})
|
||||
)
|
||||
|
||||
saveIndex()
|
||||
saveIndex()
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,58 +1,48 @@
|
|||
import "katex/contrib/mhchem" // chemical formula
|
||||
import "katex/contrib/mhchem" // chemical formula, https://katex.org/docs/node.html#using-mhchem-extension
|
||||
|
||||
import remarkCalloutDirectives from "@microflash/remark-callout-directives"
|
||||
import matter from "gray-matter"
|
||||
import hljs from "highlight.js" // code block syntax highlighting
|
||||
import { JSDOM } from "jsdom" // HTML DOM parsing
|
||||
import katex from "katex" // rendering mathematical expression
|
||||
import markdownIt from "markdown-it" // rendering markdown
|
||||
import markdownItAnchor from "markdown-it-anchor" // markdown anchor
|
||||
import markdownItFootnote from "markdown-it-footnote" // markdown footnote
|
||||
import highlightLines from "markdown-it-highlight-lines" // highlighting specific lines in code blocks
|
||||
import markDownItMark from "markdown-it-mark" // text highlighting
|
||||
import markdownItSub from "markdown-it-sub" // markdown subscript
|
||||
import markdownItSup from "markdown-it-sup" // markdown superscript
|
||||
import markdownItTaskCheckbox from "markdown-it-task-checkbox" // a TODO list checkboxes
|
||||
import markdownItTexMath from "markdown-it-texmath" // rendering mathematical expression
|
||||
import toc from "markdown-toc" // table of contents generation
|
||||
import slugify from "slugify"
|
||||
import { JSDOM } from "jsdom"
|
||||
import toc from "markdown-toc"
|
||||
import rehypeAutolinkHeadings from "rehype-autolink-headings"
|
||||
import rehypeColorChips from "rehype-color-chips"
|
||||
import rehypeHighlight from "rehype-highlight"
|
||||
import rehypeKatex from "rehype-katex"
|
||||
import rehypeRaw from "rehype-raw"
|
||||
import rehypeSlug from "rehype-slug"
|
||||
import rehypeStringify from "rehype-stringify"
|
||||
import rehypeTitleFigure from "rehype-title-figure"
|
||||
import remarkDirective from "remark-directive"
|
||||
import remarkFlexibleMarkers from "remark-flexible-markers"
|
||||
import remarkFrontmatter from "remark-frontmatter"
|
||||
import remarkGfm from "remark-gfm"
|
||||
import remarkMath from "remark-math"
|
||||
import remarkParse from "remark-parse"
|
||||
import remarkRehype from "remark-rehype"
|
||||
import supersub from "remark-supersub"
|
||||
import { unified } from "unified"
|
||||
|
||||
import { MarkdownData, ParseMode } from "./types/types"
|
||||
import { nthIndex } from "./util"
|
||||
|
||||
const slugifyIt = (s: string) => slugify(s, { lower: true, strict: true })
|
||||
|
||||
const md = markdownIt({
|
||||
// https://github.com/highlightjs/highlight.js/blob/main/SUPPORTED_LANGUAGES.md
|
||||
highlight: (str, lang) => {
|
||||
if (lang && hljs.getLanguage(lang)) {
|
||||
try {
|
||||
return hljs.highlight(str, { language: lang }).value
|
||||
// eslint-disable-next-line no-empty
|
||||
} catch (error) {}
|
||||
}
|
||||
|
||||
return "" // use external default escaping
|
||||
},
|
||||
html: true,
|
||||
})
|
||||
.use(markdownItTexMath, {
|
||||
engine: katex,
|
||||
delimiters: "dollars",
|
||||
})
|
||||
.use(markdownItAnchor, {
|
||||
permalink: markdownItAnchor.permalink.ariaHidden({
|
||||
placement: "before",
|
||||
symbol: "#",
|
||||
renderHref: (s) => `#${slugifyIt(s)}`,
|
||||
}),
|
||||
slugify: slugifyIt,
|
||||
})
|
||||
.use(markdownItTaskCheckbox)
|
||||
.use(markDownItMark)
|
||||
.use(markdownItSub)
|
||||
.use(markdownItSup)
|
||||
.use(highlightLines)
|
||||
.use(markdownItFootnote)
|
||||
const processor = unified() // interface for remark and rehype
|
||||
.use(remarkParse) // markdown to AST
|
||||
.use(remarkGfm, { singleTilde: false }) // https://github.com/remarkjs/remark-gfm
|
||||
.use(supersub) // https://github.com/Symbitic/remark-plugins/tree/master/packages/remark-supersub
|
||||
.use(remarkDirective) // https://github.com/remarkjs/remark-directive
|
||||
.use(remarkCalloutDirectives) // https://github.com/Microflash/remark-callout-directives
|
||||
.use(remarkMath) // https://github.com/remarkjs/remark-math
|
||||
.use(remarkFlexibleMarkers) // https://github.com/ipikuka/remark-flexible-markers
|
||||
.use(remarkFrontmatter, ["yaml", "toml"]) // https://github.com/remarkjs/remark-frontmatter
|
||||
.use(remarkRehype, { allowDangerousHtml: true }) // markdown to HTML
|
||||
.use(rehypeRaw) // https://github.com/rehypejs/rehype-raw
|
||||
.use(rehypeSlug) // https://github.com/rehypejs/rehype-slug
|
||||
.use(rehypeTitleFigure) // https://github.com/y-temp4/rehype-title-figure
|
||||
.use(rehypeAutolinkHeadings, { content: { type: "text", value: "#" } }) // https://github.com/rehypejs/rehype-autolink-headings
|
||||
.use(rehypeHighlight) // https://github.com/rehypejs/rehype-highlight
|
||||
.use(rehypeKatex) // math and formula and stuff
|
||||
.use(rehypeColorChips) // https://github.com/shreshthmohan/rehype-color-chips
|
||||
.use(rehypeStringify) // syntax tree (hast) to HTML
|
||||
|
||||
/**
|
||||
* parse the front matter if it exists
|
||||
|
|
@ -61,11 +51,11 @@ const md = markdownIt({
|
|||
* @param {string} path - filename of the markdown file
|
||||
* @param {ParseMode} mode
|
||||
*/
|
||||
export default function parseMarkdown(
|
||||
export default async function parseMarkdown(
|
||||
markdownRaw: string,
|
||||
path: string,
|
||||
mode: ParseMode
|
||||
): MarkdownData {
|
||||
): Promise<MarkdownData> {
|
||||
const fileHasFrontMatter = markdownRaw.startsWith("---")
|
||||
|
||||
const frontMatter = fileHasFrontMatter
|
||||
|
|
@ -83,31 +73,24 @@ export default function parseMarkdown(
|
|||
|
||||
if (mode === ParseMode.PORTFOLIO) {
|
||||
if (frontMatter.overview) {
|
||||
frontMatter.overview = md.render(frontMatter.overview)
|
||||
frontMatter.overview = String(
|
||||
processor.processSync(frontMatter.overview)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// work with rendered DOM
|
||||
//
|
||||
|
||||
const dom = new JSDOM(
|
||||
md.render(
|
||||
fileHasFrontMatter
|
||||
? markdownRaw.slice(nthIndex(markdownRaw, "---", 2) + 3)
|
||||
: markdownRaw
|
||||
) || ""
|
||||
frontMatter.content = touchupHTML(
|
||||
String(processor.processSync(markdownRaw))
|
||||
)
|
||||
|
||||
// add .hljs class to all block codes
|
||||
return frontMatter as MarkdownData
|
||||
}
|
||||
|
||||
dom.window.document.querySelectorAll("pre > code").forEach((item) => {
|
||||
item.classList.add("hljs")
|
||||
})
|
||||
function touchupHTML(html: string): string {
|
||||
const dom = new JSDOM(html)
|
||||
|
||||
// add parent div to tables (horizontally scroll table on small displays)
|
||||
|
||||
dom.window.document.querySelectorAll("table").forEach((item) => {
|
||||
// `element` is the element you want to wrap
|
||||
const parent = item.parentNode
|
||||
|
|
@ -119,13 +102,27 @@ export default function parseMarkdown(
|
|||
wrapper.appendChild(item)
|
||||
})
|
||||
|
||||
frontMatter.content = dom.window.document.documentElement.innerHTML
|
||||
|
||||
return frontMatter as MarkdownData
|
||||
}
|
||||
|
||||
export function generateToc(markdownRaw: string): string {
|
||||
return md.render(toc(markdownRaw).content, {
|
||||
slugify: slugifyIt,
|
||||
// add hr before footnotes
|
||||
dom.window.document.querySelectorAll(".footnotes").forEach((item) => {
|
||||
item.parentNode?.insertBefore(
|
||||
dom.window.document.createElement("hr"),
|
||||
item
|
||||
)
|
||||
})
|
||||
|
||||
// https://developer.chrome.com/docs/lighthouse/best-practices/external-anchors-use-rel-noopener/
|
||||
// https://github.com/cure53/DOMPurify/issues/317#issuecomment-698800327
|
||||
dom.window.document.querySelectorAll("a").forEach((item) => {
|
||||
if ("target" in item && item["target"] === "_blank")
|
||||
item.setAttribute("rel", "noopener")
|
||||
})
|
||||
|
||||
return dom.window.document.documentElement.innerHTML
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate Table of Contents as a HTML string
|
||||
*/
|
||||
export async function generateToc(markdownRaw: string): Promise<string> {
|
||||
return String(processor.processSync(toc(markdownRaw).content))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import { optimize } from "svgo"
|
|||
import tinycolor from "tinycolor2"
|
||||
|
||||
import { contentMap, seriesMap } from "."
|
||||
import skills from "./portfolio/skills.json"
|
||||
import skills from "./portfolio/skills.json" assert { type: "json" }
|
||||
import { Badge } from "./types/types"
|
||||
import { writeToFile } from "./util"
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,10 @@ export interface DataToPass {
|
|||
* @param {ParseMode} mode - parse mode
|
||||
* @param {string} path - path of file or folder
|
||||
*/
|
||||
export function recursiveParse(mode: ParseMode, path: string): void {
|
||||
export async function recursiveParse(
|
||||
mode: ParseMode,
|
||||
path: string
|
||||
): Promise<void> {
|
||||
// get name of the file or folder that's currently being parsed
|
||||
const fileOrFolderName = path2FileOrFolderName(path)
|
||||
|
||||
|
|
@ -41,11 +44,11 @@ export function recursiveParse(mode: ParseMode, path: string): void {
|
|||
// if it's a directory, call this function to every files/directories in it
|
||||
// if it's a file, parse it and then save it to file
|
||||
if (stats.isDirectory()) {
|
||||
fs.readdirSync(path).map((childPath) => {
|
||||
recursiveParse(mode, `${path}/${childPath}`)
|
||||
})
|
||||
for (const childPath of fs.readdirSync(path)) {
|
||||
await recursiveParse(mode, `${path}/${childPath}`)
|
||||
}
|
||||
} else if (stats.isFile()) {
|
||||
parseFile(mode, path)
|
||||
await parseFile(mode, path)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -55,7 +58,7 @@ export function recursiveParse(mode: ParseMode, path: string): void {
|
|||
* @param {ParseMode} mode - decides which function to use to parse the file
|
||||
* @param {string} path - path of the markdown file
|
||||
*/
|
||||
function parseFile(mode: ParseMode, path: string): void {
|
||||
async function parseFile(mode: ParseMode, path: string): Promise<void> {
|
||||
// stop if it is not a markdown file
|
||||
if (!path.endsWith(".md")) {
|
||||
console.log(`Ignoring non markdown file at: ${path}`)
|
||||
|
|
@ -67,8 +70,10 @@ function parseFile(mode: ParseMode, path: string): void {
|
|||
*/
|
||||
|
||||
const markdownRaw = fs.readFileSync(path, "utf8")
|
||||
const markdownData = parseMarkdown(markdownRaw, path, mode)
|
||||
const { humanizedDuration, totalWords } = readTimeEstimate(
|
||||
const markdownData = await parseMarkdown(markdownRaw, path, mode)
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
const { humanizedDuration, totalWords } = readTimeEstimate.default(
|
||||
markdownData.content,
|
||||
275,
|
||||
12,
|
||||
|
|
@ -87,15 +92,15 @@ function parseFile(mode: ParseMode, path: string): void {
|
|||
|
||||
switch (mode) {
|
||||
case ParseMode.POSTS:
|
||||
parsePost(dataToPass)
|
||||
await parsePost(dataToPass)
|
||||
break
|
||||
|
||||
case ParseMode.SERIES:
|
||||
parseSeries(dataToPass)
|
||||
await parseSeries(dataToPass)
|
||||
break
|
||||
|
||||
case ParseMode.PORTFOLIO:
|
||||
parseProjects(dataToPass)
|
||||
await parseProjects(dataToPass)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { PostData } from "../types/types"
|
|||
import { writeToFile } from "../util"
|
||||
import { DataToPass } from "."
|
||||
|
||||
export default function parsePost(data: DataToPass): void {
|
||||
export default async function parsePost(data: DataToPass): Promise<void> {
|
||||
const {
|
||||
urlPath,
|
||||
markdownRaw,
|
||||
|
|
@ -70,7 +70,7 @@ export default function parsePost(data: DataToPass): void {
|
|||
`${contentDirectoryPath}${urlPath}.json`,
|
||||
JSON.stringify({
|
||||
content: markdownData.content,
|
||||
toc: generateToc(markdownRaw),
|
||||
toc: await generateToc(markdownRaw),
|
||||
})
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import icons, { SimpleIcon } from "simple-icons"
|
||||
import type { SimpleIcon } from "simple-icons"
|
||||
import * as icons from "simple-icons"
|
||||
import tinycolor from "tinycolor2"
|
||||
|
||||
import { portfolioData } from ".."
|
||||
|
|
@ -7,9 +8,11 @@ import { generateToc } from "../parseMarkdown"
|
|||
import { writeToFile } from "../util"
|
||||
import { DataToPass } from "."
|
||||
|
||||
export default function parseProjects(data: DataToPass): void {
|
||||
const { urlPath, markdownRaw, markdownData } = data
|
||||
|
||||
export default async function parseProjects({
|
||||
urlPath,
|
||||
markdownRaw,
|
||||
markdownData,
|
||||
}: DataToPass): Promise<void> {
|
||||
if (markdownData.badges) {
|
||||
;(markdownData.badges as string[]).forEach((slug) => {
|
||||
// todo: handle cases when icon is not on simple-icons
|
||||
|
|
@ -48,7 +51,7 @@ export default function parseProjects(data: DataToPass): void {
|
|||
`${contentDirectoryPath}${urlPath}.json`,
|
||||
JSON.stringify({
|
||||
content: markdownData.content,
|
||||
toc: generateToc(markdownRaw),
|
||||
toc: await generateToc(markdownRaw),
|
||||
})
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { PostData } from "../types/types"
|
|||
import { writeToFile } from "../util"
|
||||
import { DataToPass } from "."
|
||||
|
||||
export default function parseSeries(data: DataToPass): void {
|
||||
export default async function parseSeries(data: DataToPass): Promise<void> {
|
||||
const {
|
||||
path,
|
||||
urlPath: _urlPath,
|
||||
|
|
@ -141,7 +141,7 @@ export default function parseSeries(data: DataToPass): void {
|
|||
`${contentDirectoryPath}${urlPath}.json`,
|
||||
JSON.stringify({
|
||||
content: markdownData.content,
|
||||
toc: generateToc(markdownRaw),
|
||||
toc: await generateToc(markdownRaw),
|
||||
})
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
declare module "markdown-it-footnote"
|
||||
|
|
@ -1 +0,0 @@
|
|||
declare module "markdown-it-highlight-lines"
|
||||
|
|
@ -1 +0,0 @@
|
|||
declare module "markdown-it-mark"
|
||||
|
|
@ -1 +0,0 @@
|
|||
declare module "markdown-it-sub"
|
||||
|
|
@ -1 +0,0 @@
|
|||
declare module "markdown-it-sup"
|
||||
|
|
@ -1 +0,0 @@
|
|||
declare module "markdown-it-task-checkbox"
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
declare module "markdown-it-texmath" {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
export default function texmath(md: MarkdownIt, ...params: any[]): void
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue