refactor: replace markdown parser

- replace markdown-it with remark and rehype
- add bunch of markdown features
This commit is contained in:
Kim, Jimin 2023-07-10 10:56:09 +09:00
parent c0195e02fd
commit ae5ecaaccc
31 changed files with 912 additions and 247 deletions

View file

@ -24,11 +24,12 @@ When I was old enough to understand language, my father introduced me to my firs
At the time, it was everything I wished for and more, but little did I know,
IE was already on the decline while another browser was quietly climbing up the market share.
<p align="center">
<img alt="browser market share" src="/img/posts/linux-setup-script/browser-market-share-trend.avif" style="max-width: 100%;" />
<br />
source: <a href="https://gs.statcounter.com/browser-market-share/desktop/worldwide/#monthly-200901-202203" target="_blank">statcounter.com</a>
</p>
<figure>
<img title="browser market share" alt="browser market share" src="/img/posts/linux-setup-script/browser-market-share-trend.avif">
<figcaption>
source: <a href="https://gs.statcounter.com/browser-market-share/desktop/worldwide/#monthly-200901-202203" target="_blank">statcounter.com</a>
</figcaption>
</figure>
One day, probably after my father upgraded the PC to Windows 7,
the default browser was changed to some colorful ball looking thing.

View file

@ -15,7 +15,7 @@ This post exists to test various features such as markdown-to-html conversion, t
## Image
<img src="/icon/icon.svg" alt="developomp icon" width="100">
![alt text](/icon/icon.svg "title")
## Video
@ -51,15 +51,15 @@ This post exists to test various features such as markdown-to-html conversion, t
## Footnote
css only causes pain[^css_bad] and python is overrated[^python_is_overrated].
everyone should use tailwind[^use_tailwind] and pineapple belongs on pizza[^pineapple_pizza_truth].
## Code
Here's a `code`.
Here's a `code`. Here are color chips `#FFFFFF` `#E06C75` `#61AFEF` `#E5C07B`.
```python {7,12,14-15}
```python
print("And here's a language-specific code block")
# with comments and line highlighting!
# with comments!
x = 256
y = 256
@ -81,13 +81,58 @@ print(z is w) # False. id(z) is not equal to id(w)
>
> > nested blockquote
**bold**<br />
_italic_<br />
~~strikethrough~~<br />
<u>underlined</u><br />
==marked==<br />
this is a ^superscript^ (soon^TM^)<br />
and this is a ~subscript~ (H~2~O)
**bold**
_italic_
~~strikethrough~~
<u>underlined</u>
==marked==
=a=marked with color amber==
=b=blue==
=c=cyan==
=d=brown==
=f=fuchsia==
=g=green==
=h=hotpink==
=l=lime==
=m=magenta==
=n=navyblue==
=o=orange==
=p=purple==
=q=pink==
=r=red==
=s=silver==
=t=teal==
=v=violet==
=x=gray==
=y=yellow==.
This is a ^superscript^ (soon^TM^) and this is a ~subscript~ (H~2~O)
## Callouts & Directives
:::note
Some **content** with _Markdown_ `syntax`.
:::
:::commend
Some **content** with _Markdown_ `syntax`.
:::
:::warn
Some **content** with _Markdown_ `syntax`.
:::
:::deter
Some **content** with _Markdown_ `syntax`.
:::
:::assert
Some **content** with _Markdown_ `syntax`.
:::
:::note{title="Custom <u>title</u> using **directives**"}
Some **content** with _Markdown_ `syntax`.
:::
## CSS styling
@ -144,5 +189,5 @@ Lorem ipsum blah blah.
<!-- Footnotes -->
[^css_bad]: Based on my experience building this website, Dec 2021.
[^python_is_overrated]: Based on my infinite wisdom, Dec 2021.
[^use_tailwind]: Based on my experience building this website
[^pineapple_pizza_truth]: Based on my infinite wisdom

View file

@ -26,16 +26,12 @@ The War Brokers Mods (WBM) is a mod for the game
different aspects of the game
- [OBS overlay][overlays] - Customizable overlays for [OBS studio](https://github.com/obsproject/obs-studio)
<p align="center">
<img alt="Overlay image" src="/img/portfolio/wbm-overlays.avif" />
</p>
![Overlay image](/img/portfolio/wbm-overlays.avif "Overlay image")
- [installer][installer] - Utility for installing updating the mod. Built with [tauri][tauri],
[rust][rust], [svelte][svelte], and [tailwind css][tailwindcss].
<p align="center">
<img alt="Installer image" src="/img/portfolio/wbm-installer.avif" />
</p>
![Installer image](/img/portfolio/wbm-installer.avif "Installer image")
[mod]: https://github.com/War-Brokers-Mods/WBM
[overlays]: https://github.com/War-Brokers-Mods/WBM-Overlays

View file

@ -1,52 +1,54 @@
{
"name": "@developomp-site/content",
"version": "0.0.0",
"license": "MIT",
"files": [
"dist/**"
],
"type": "module",
"scripts": {
"build": "ts-node --experimental-specifier-resolution=node ./src",
"build": "rm -rf build && tsc && node --es-module-specifier-resolution=node build/index.js",
"lint": "eslint .",
"clean": "rm -rf .turbo node_modules dist"
"clean": "rm -rf .turbo node_modules dist build"
},
"dependencies": {
"@developomp-site/eslint-config": "workspace:^",
"@developomp-site/prettier-config": "workspace:^",
"@microflash/remark-callout-directives": "^1.6.0",
"@types/dompurify": "^3.0.2",
"@types/ejs": "^3.1.2",
"@types/elasticlunr": "^0.9.5",
"@types/jsdom": "^21.1.1",
"@types/katex": "^0.16.0",
"@types/markdown-it": "^12.2.3",
"@types/node": "^20.4.1",
"@types/read-time-estimate": "^0.0.0",
"@types/svgo": "^3.0.0",
"@types/tinycolor2": "^1.4.3",
"canvas": "^2.11.2",
"ejs": "^3.1.9",
"elasticlunr": "^0.9.5",
"eslint": "^8.44.0",
"gray-matter": "^4.0.3",
"jsdom": "^22.1.0",
"markdown-it": "^13.0.1",
"markdown-it-anchor": "^8.6.7",
"markdown-it-attrs": "^4.1.6",
"markdown-it-footnote": "^3.0.3",
"markdown-it-highlight-lines": "^1.0.2",
"markdown-it-mark": "^3.0.1",
"markdown-it-sub": "^1.0.0",
"markdown-it-sup": "^1.0.0",
"markdown-it-task-checkbox": "^1.0.6",
"markdown-it-texmath": "^1.0.0",
"markdown-toc": "^1.2.0",
"read-time-estimate": "^0.0.3",
"rehype-autolink-headings": "^6.1.1",
"rehype-color-chips": "^0.1.3",
"rehype-highlight": "^6.0.0",
"rehype-katex": "^6.0.3",
"rehype-raw": "^6.1.1",
"rehype-slug": "^5.1.0",
"rehype-stringify": "^9.0.3",
"rehype-title-figure": "^0.1.2",
"remark-directive": "^2.0.1",
"remark-flexible-markers": "^1.0.3",
"remark-frontmatter": "^4.0.1",
"remark-gfm": "^3.0.1",
"remark-math": "^5.1.1",
"remark-parse": "^10.0.2",
"remark-rehype": "^10.1.0",
"remark-supersub": "^1.0.0",
"simple-icons": "^9.4.0",
"slugify": "^1.6.6",
"svgo": "^3.0.2",
"tinycolor2": "^1.6.0"
},
"devDependencies": {
"@developomp-site/eslint-config": "workspace:*",
"@developomp-site/prettier-config": "workspace:*",
"@types/elasticlunr": "^0.9.5",
"@types/jsdom": "^21.1.1",
"@types/node": "^20.4.1",
"eslint": "^8.44.0",
"ts-node": "^10.9.1",
"typescript": "^5.1.6"
"tinycolor2": "^1.6.0",
"typescript": "^5.1.6",
"unified": "^10.1.2"
}
}

View file

@ -34,53 +34,57 @@ export const portfolioData: PortfolioData = {
projects: {},
}
/**
* Delete previously generated files
*/
async function main() {
/**
* Delete previously generated files
*/
try {
fs.rmSync("dist", { recursive: true })
// eslint-disable-next-line no-empty
} catch (err) {}
try {
fs.rmSync("dist", { recursive: true })
// eslint-disable-next-line no-empty
} catch (err) {}
/**
* Checking
*/
/**
* Checking
*/
if (!fs.lstatSync(markdownPath).isDirectory())
throw Error("Invalid markdown path")
if (!fs.lstatSync(markdownPath).isDirectory())
throw Error("Invalid markdown path")
if (!fs.lstatSync(markdownPath + "/posts").isDirectory())
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
if (!fs.lstatSync(markdownPath + "/posts").isDirectory())
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
if (!fs.lstatSync(markdownPath + "/series").isDirectory())
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
if (!fs.lstatSync(markdownPath + "/series").isDirectory())
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
/**
* Parse
*/
/**
* Parse
*/
// parse markdown
recursiveParse(ParseMode.POSTS, markdownPath + "/posts")
recursiveParse(ParseMode.SERIES, markdownPath + "/series")
recursiveParse(ParseMode.PORTFOLIO, markdownPath + "/projects")
// parse markdown
await recursiveParse(ParseMode.POSTS, markdownPath + "/posts")
await recursiveParse(ParseMode.SERIES, markdownPath + "/series")
await recursiveParse(ParseMode.PORTFOLIO, markdownPath + "/projects")
sortDates()
fillTags()
parseSeries()
generatePortfolioSVGs()
sortDates()
fillTags()
parseSeries()
generatePortfolioSVGs()
/**
* Save results
*/
/**
* Save results
*/
fs.writeFileSync(mapFilePath, JSON.stringify(contentMap))
fs.writeFileSync(
portfolioFilePath,
JSON.stringify({
...portfolioData,
skills: Array.from(portfolioData.skills),
})
)
fs.writeFileSync(mapFilePath, JSON.stringify(contentMap))
fs.writeFileSync(
portfolioFilePath,
JSON.stringify({
...portfolioData,
skills: Array.from(portfolioData.skills),
})
)
saveIndex()
saveIndex()
}
main()

View file

@ -1,58 +1,48 @@
import "katex/contrib/mhchem" // chemical formula
import "katex/contrib/mhchem" // chemical formula, https://katex.org/docs/node.html#using-mhchem-extension
import remarkCalloutDirectives from "@microflash/remark-callout-directives"
import matter from "gray-matter"
import hljs from "highlight.js" // code block syntax highlighting
import { JSDOM } from "jsdom" // HTML DOM parsing
import katex from "katex" // rendering mathematical expression
import markdownIt from "markdown-it" // rendering markdown
import markdownItAnchor from "markdown-it-anchor" // markdown anchor
import markdownItFootnote from "markdown-it-footnote" // markdown footnote
import highlightLines from "markdown-it-highlight-lines" // highlighting specific lines in code blocks
import markDownItMark from "markdown-it-mark" // text highlighting
import markdownItSub from "markdown-it-sub" // markdown subscript
import markdownItSup from "markdown-it-sup" // markdown superscript
import markdownItTaskCheckbox from "markdown-it-task-checkbox" // a TODO list checkboxes
import markdownItTexMath from "markdown-it-texmath" // rendering mathematical expression
import toc from "markdown-toc" // table of contents generation
import slugify from "slugify"
import { JSDOM } from "jsdom"
import toc from "markdown-toc"
import rehypeAutolinkHeadings from "rehype-autolink-headings"
import rehypeColorChips from "rehype-color-chips"
import rehypeHighlight from "rehype-highlight"
import rehypeKatex from "rehype-katex"
import rehypeRaw from "rehype-raw"
import rehypeSlug from "rehype-slug"
import rehypeStringify from "rehype-stringify"
import rehypeTitleFigure from "rehype-title-figure"
import remarkDirective from "remark-directive"
import remarkFlexibleMarkers from "remark-flexible-markers"
import remarkFrontmatter from "remark-frontmatter"
import remarkGfm from "remark-gfm"
import remarkMath from "remark-math"
import remarkParse from "remark-parse"
import remarkRehype from "remark-rehype"
import supersub from "remark-supersub"
import { unified } from "unified"
import { MarkdownData, ParseMode } from "./types/types"
import { nthIndex } from "./util"
const slugifyIt = (s: string) => slugify(s, { lower: true, strict: true })
const md = markdownIt({
// https://github.com/highlightjs/highlight.js/blob/main/SUPPORTED_LANGUAGES.md
highlight: (str, lang) => {
if (lang && hljs.getLanguage(lang)) {
try {
return hljs.highlight(str, { language: lang }).value
// eslint-disable-next-line no-empty
} catch (error) {}
}
return "" // use external default escaping
},
html: true,
})
.use(markdownItTexMath, {
engine: katex,
delimiters: "dollars",
})
.use(markdownItAnchor, {
permalink: markdownItAnchor.permalink.ariaHidden({
placement: "before",
symbol: "#",
renderHref: (s) => `#${slugifyIt(s)}`,
}),
slugify: slugifyIt,
})
.use(markdownItTaskCheckbox)
.use(markDownItMark)
.use(markdownItSub)
.use(markdownItSup)
.use(highlightLines)
.use(markdownItFootnote)
const processor = unified() // interface for remark and rehype
.use(remarkParse) // markdown to AST
.use(remarkGfm, { singleTilde: false }) // https://github.com/remarkjs/remark-gfm
.use(supersub) // https://github.com/Symbitic/remark-plugins/tree/master/packages/remark-supersub
.use(remarkDirective) // https://github.com/remarkjs/remark-directive
.use(remarkCalloutDirectives) // https://github.com/Microflash/remark-callout-directives
.use(remarkMath) // https://github.com/remarkjs/remark-math
.use(remarkFlexibleMarkers) // https://github.com/ipikuka/remark-flexible-markers
.use(remarkFrontmatter, ["yaml", "toml"]) // https://github.com/remarkjs/remark-frontmatter
.use(remarkRehype, { allowDangerousHtml: true }) // markdown to HTML
.use(rehypeRaw) // https://github.com/rehypejs/rehype-raw
.use(rehypeSlug) // https://github.com/rehypejs/rehype-slug
.use(rehypeTitleFigure) // https://github.com/y-temp4/rehype-title-figure
.use(rehypeAutolinkHeadings, { content: { type: "text", value: "#" } }) // https://github.com/rehypejs/rehype-autolink-headings
.use(rehypeHighlight) // https://github.com/rehypejs/rehype-highlight
.use(rehypeKatex) // math and formula and stuff
.use(rehypeColorChips) // https://github.com/shreshthmohan/rehype-color-chips
.use(rehypeStringify) // syntax tree (hast) to HTML
/**
* parse the front matter if it exists
@ -61,11 +51,11 @@ const md = markdownIt({
* @param {string} path - filename of the markdown file
* @param {ParseMode} mode
*/
export default function parseMarkdown(
export default async function parseMarkdown(
markdownRaw: string,
path: string,
mode: ParseMode
): MarkdownData {
): Promise<MarkdownData> {
const fileHasFrontMatter = markdownRaw.startsWith("---")
const frontMatter = fileHasFrontMatter
@ -83,31 +73,24 @@ export default function parseMarkdown(
if (mode === ParseMode.PORTFOLIO) {
if (frontMatter.overview) {
frontMatter.overview = md.render(frontMatter.overview)
frontMatter.overview = String(
processor.processSync(frontMatter.overview)
)
}
}
}
//
// work with rendered DOM
//
const dom = new JSDOM(
md.render(
fileHasFrontMatter
? markdownRaw.slice(nthIndex(markdownRaw, "---", 2) + 3)
: markdownRaw
) || ""
frontMatter.content = touchupHTML(
String(processor.processSync(markdownRaw))
)
// add .hljs class to all block codes
return frontMatter as MarkdownData
}
dom.window.document.querySelectorAll("pre > code").forEach((item) => {
item.classList.add("hljs")
})
function touchupHTML(html: string): string {
const dom = new JSDOM(html)
// add parent div to tables (horizontally scroll table on small displays)
dom.window.document.querySelectorAll("table").forEach((item) => {
// `element` is the element you want to wrap
const parent = item.parentNode
@ -119,13 +102,27 @@ export default function parseMarkdown(
wrapper.appendChild(item)
})
frontMatter.content = dom.window.document.documentElement.innerHTML
return frontMatter as MarkdownData
}
export function generateToc(markdownRaw: string): string {
return md.render(toc(markdownRaw).content, {
slugify: slugifyIt,
// add hr before footnotes
dom.window.document.querySelectorAll(".footnotes").forEach((item) => {
item.parentNode?.insertBefore(
dom.window.document.createElement("hr"),
item
)
})
// https://developer.chrome.com/docs/lighthouse/best-practices/external-anchors-use-rel-noopener/
// https://github.com/cure53/DOMPurify/issues/317#issuecomment-698800327
dom.window.document.querySelectorAll("a").forEach((item) => {
if ("target" in item && item["target"] === "_blank")
item.setAttribute("rel", "noopener")
})
return dom.window.document.documentElement.innerHTML
}
/**
* Generate Table of Contents as a HTML string
*/
export async function generateToc(markdownRaw: string): Promise<string> {
return String(processor.processSync(toc(markdownRaw).content))
}

View file

@ -5,7 +5,7 @@ import { optimize } from "svgo"
import tinycolor from "tinycolor2"
import { contentMap, seriesMap } from "."
import skills from "./portfolio/skills.json"
import skills from "./portfolio/skills.json" assert { type: "json" }
import { Badge } from "./types/types"
import { writeToFile } from "./util"

View file

@ -29,7 +29,10 @@ export interface DataToPass {
* @param {ParseMode} mode - parse mode
* @param {string} path - path of file or folder
*/
export function recursiveParse(mode: ParseMode, path: string): void {
export async function recursiveParse(
mode: ParseMode,
path: string
): Promise<void> {
// get name of the file or folder that's currently being parsed
const fileOrFolderName = path2FileOrFolderName(path)
@ -41,11 +44,11 @@ export function recursiveParse(mode: ParseMode, path: string): void {
// if it's a directory, call this function to every files/directories in it
// if it's a file, parse it and then save it to file
if (stats.isDirectory()) {
fs.readdirSync(path).map((childPath) => {
recursiveParse(mode, `${path}/${childPath}`)
})
for (const childPath of fs.readdirSync(path)) {
await recursiveParse(mode, `${path}/${childPath}`)
}
} else if (stats.isFile()) {
parseFile(mode, path)
await parseFile(mode, path)
}
}
@ -55,7 +58,7 @@ export function recursiveParse(mode: ParseMode, path: string): void {
* @param {ParseMode} mode - decides which function to use to parse the file
* @param {string} path - path of the markdown file
*/
function parseFile(mode: ParseMode, path: string): void {
async function parseFile(mode: ParseMode, path: string): Promise<void> {
// stop if it is not a markdown file
if (!path.endsWith(".md")) {
console.log(`Ignoring non markdown file at: ${path}`)
@ -67,8 +70,10 @@ function parseFile(mode: ParseMode, path: string): void {
*/
const markdownRaw = fs.readFileSync(path, "utf8")
const markdownData = parseMarkdown(markdownRaw, path, mode)
const { humanizedDuration, totalWords } = readTimeEstimate(
const markdownData = await parseMarkdown(markdownRaw, path, mode)
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
const { humanizedDuration, totalWords } = readTimeEstimate.default(
markdownData.content,
275,
12,
@ -87,15 +92,15 @@ function parseFile(mode: ParseMode, path: string): void {
switch (mode) {
case ParseMode.POSTS:
parsePost(dataToPass)
await parsePost(dataToPass)
break
case ParseMode.SERIES:
parseSeries(dataToPass)
await parseSeries(dataToPass)
break
case ParseMode.PORTFOLIO:
parseProjects(dataToPass)
await parseProjects(dataToPass)
break
}
}

View file

@ -6,7 +6,7 @@ import { PostData } from "../types/types"
import { writeToFile } from "../util"
import { DataToPass } from "."
export default function parsePost(data: DataToPass): void {
export default async function parsePost(data: DataToPass): Promise<void> {
const {
urlPath,
markdownRaw,
@ -70,7 +70,7 @@ export default function parsePost(data: DataToPass): void {
`${contentDirectoryPath}${urlPath}.json`,
JSON.stringify({
content: markdownData.content,
toc: generateToc(markdownRaw),
toc: await generateToc(markdownRaw),
})
)
}

View file

@ -1,4 +1,5 @@
import icons, { SimpleIcon } from "simple-icons"
import type { SimpleIcon } from "simple-icons"
import * as icons from "simple-icons"
import tinycolor from "tinycolor2"
import { portfolioData } from ".."
@ -7,9 +8,11 @@ import { generateToc } from "../parseMarkdown"
import { writeToFile } from "../util"
import { DataToPass } from "."
export default function parseProjects(data: DataToPass): void {
const { urlPath, markdownRaw, markdownData } = data
export default async function parseProjects({
urlPath,
markdownRaw,
markdownData,
}: DataToPass): Promise<void> {
if (markdownData.badges) {
;(markdownData.badges as string[]).forEach((slug) => {
// todo: handle cases when icon is not on simple-icons
@ -48,7 +51,7 @@ export default function parseProjects(data: DataToPass): void {
`${contentDirectoryPath}${urlPath}.json`,
JSON.stringify({
content: markdownData.content,
toc: generateToc(markdownRaw),
toc: await generateToc(markdownRaw),
})
)
}

View file

@ -6,7 +6,7 @@ import { PostData } from "../types/types"
import { writeToFile } from "../util"
import { DataToPass } from "."
export default function parseSeries(data: DataToPass): void {
export default async function parseSeries(data: DataToPass): Promise<void> {
const {
path,
urlPath: _urlPath,
@ -141,7 +141,7 @@ export default function parseSeries(data: DataToPass): void {
`${contentDirectoryPath}${urlPath}.json`,
JSON.stringify({
content: markdownData.content,
toc: generateToc(markdownRaw),
toc: await generateToc(markdownRaw),
})
)
}

View file

@ -1 +0,0 @@
declare module "markdown-it-footnote"

View file

@ -1 +0,0 @@
declare module "markdown-it-highlight-lines"

View file

@ -1 +0,0 @@
declare module "markdown-it-mark"

View file

@ -1 +0,0 @@
declare module "markdown-it-sub"

View file

@ -1 +0,0 @@
declare module "markdown-it-sup"

View file

@ -1 +0,0 @@
declare module "markdown-it-task-checkbox"

View file

@ -1,4 +0,0 @@
declare module "markdown-it-texmath" {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export default function texmath(md: MarkdownIt, ...params: any[]): void
}

View file

@ -1,13 +1,15 @@
{
"include": ["src"],
"ts-node": {
"esm": true
},
"compilerOptions": {
"outDir": "build",
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "Node",
"noImplicitAny": false,
"resolveJsonModule": true,
"esModuleInterop": true,
"skipLibCheck": true,
"resolveJsonModule": true,
"allowSyntheticDefaultImports": true,
"strict": true
},
"exclude": ["dist", "node_modules"]