pomp/pompy.dev
1
0
Fork 0
Code Issues 8 Pull requests Activity Actions

split markdown parsing code

Browse source
This commit is contained in:
Kim, Jimin 2021-12-17 16:10:59 +09:00
parent 868b386625
commit 6a43e2e44d
10 changed files with 285 additions and 250 deletions
Download patch file Download diff file Expand all files Collapse all files

5
source/generate/config.ts Normal file
View file

@ -0,0 +1,5 @@
export const markdownPath = "./markdown" // where it will look for markdown documents
export const outPath = "./src/data" // path to the json database
export const contentDirectoryPath = `${outPath}/content`
export const mapFilePath = `${outPath}/map.json`

99
source/generate/index.ts Normal file
View file

@ -0,0 +1,99 @@
/**
* @file Read markdown files and write their content and metadata to json files which can then be imported by React.
* - File and directory names starting with an underscore (_) are ignored.
* - Symbolic links are not supported.
* - The filename-to-URL converter isn't perfect. Some non-URL-friendly filenames might cause problems.
* - series must start with a number followed by an underscore
*/
import fs from "fs" // read and write files
import { Map, SeriesMap } from "./types"
import { recursiveParse } from "./recursiveParse"
import { contentDirectoryPath, mapFilePath, markdownPath } from "./config"
import { saveIndex } from "./searchIndex"
// searchable data that will be converted to JSON string
export const map: Map = {
date: {},
tags: {},
meta: {
tags: [],
},
posts: {},
series: {},
unsearchable: {},
}
export const seriesMap: SeriesMap = {}
/**
* Delete existing files
*/
try {
fs.rmSync(contentDirectoryPath, { recursive: true })
// eslint-disable-next-line no-empty
} catch (err) {}
try {
fs.unlinkSync(mapFilePath)
// eslint-disable-next-line no-empty
} catch (err) {}
// check if it's a directory and start recursive parse function
if (!fs.lstatSync(markdownPath).isDirectory())
throw Error("Invalid markdown path")
if (!fs.lstatSync(markdownPath + "/posts").isDirectory())
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
if (!fs.lstatSync(markdownPath + "/unsearchable").isDirectory())
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
if (!fs.lstatSync(markdownPath + "/series").isDirectory())
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
recursiveParse("posts", markdownPath + "/posts")
recursiveParse("unsearchable", markdownPath + "/unsearchable")
recursiveParse("series", markdownPath + "/series")
// sort dates
let dateKeys: string[] = []
for (const dateKey in map.date) {
dateKeys.push(dateKey)
}
dateKeys = dateKeys.sort()
const TmpDate = map.date
map.date = {}
dateKeys.forEach((sortedDateKey) => {
map.date[sortedDateKey] = TmpDate[sortedDateKey]
})
// fill meta data
for (const tag in map.tags) {
map.meta.tags.push(tag)
}
// sort series post
for (const seriesURL in seriesMap) {
seriesMap[seriesURL].sort((a, b) => {
if (a.index < b.index) {
return -1
}
if (a.index > b.index) {
return 1
}
return 0
})
}
for (const seriesURL in seriesMap) {
map.series[seriesURL].length = seriesMap[seriesURL].length
map.series[seriesURL].order = seriesMap[seriesURL].map((item) => item.url)
}
fs.writeFileSync(mapFilePath, JSON.stringify(map))
saveIndex()

33
source/generate/parseMarkdown.ts Normal file
View file

@ -0,0 +1,33 @@
import markdownIt from "markdown-it" // rendering markdown
import markdownItTexMath from "markdown-it-texmath" // rendering mathematical expression
import markdownItAnchor from "markdown-it-anchor" // markdown anchor
import hljs from "highlight.js" // code block highlighting
import katex from "katex" // rendering mathematical expression
import { nthIndex } from "./util"
const md = markdownIt({
// https://github.com/highlightjs/highlight.js/blob/main/SUPPORTED_LANGUAGES.md
highlight: function (str, lang) {
if (lang && hljs.getLanguage(lang)) {
try {
return hljs.highlight(str, { language: lang }).value
// eslint-disable-next-line no-empty
} catch (error) {}
}
return "" // use external default escaping
},
html: true,
})
.use(markdownItTexMath, {
engine: katex,
delimiters: "dollars",
katexOptions: { macros: { "\\RR": "\\mathbb{R}" } },
})
.use(markdownItAnchor, {})
export function parseMarkdown(markdownRaw: string): string {
return (
md.render(markdownRaw.slice(nthIndex(markdownRaw, "---", 2) + 3)) || ""
)
}

250
source/generate.ts → source/generate/recursiveParse.ts
View file

@ -1,169 +1,18 @@
/** import fs from "fs"
* @file Read markdown files and write their content and metadata to json files which can then be imported by React.
* - File and directory names starting with an underscore (_) are ignored.
* - Symbolic links are not supported.
* - The filename-to-URL converter isn't perfect. Some non-URL-friendly filenames might cause problems.
* - series must start with a number followed by an underscore
*/
import fs from "fs" // read and write files
import path from "path" // get relative path
import elasticlunr from "elasticlunr" // search index generation
import readTimeEstimate from "read-time-estimate" // post read time estimation import readTimeEstimate from "read-time-estimate" // post read time estimation
import matter from "gray-matter" // parse markdown metadata import matter from "gray-matter" // parse markdown metadata
import toc from "markdown-toc" // table of contents generation import toc from "markdown-toc" // table of contents generation
import markdownIt from "markdown-it" // rendering markdown
import markdownItTexMath from "markdown-it-texmath" // rendering mathematical expression
import markdownItAnchor from "markdown-it-anchor" // markdown anchor
import hljs from "highlight.js" // code block highlighting
import katex from "katex" // rendering mathematical expression
import { JSDOM } from "jsdom" // HTML DOM parsing import { JSDOM } from "jsdom" // HTML DOM parsing
const markdownPath = "./markdown" // where it will look for markdown documents import { nthIndex, path2FileOrFolderName, path2URL, writeToJSON } from "./util"
const outPath = "./src/data" // path to the json database import { parseMarkdown } from "./parseMarkdown"
const contentDirectoryPath = `${outPath}/content` import { contentDirectoryPath } from "./config"
const mapFilePath = `${outPath}/map.json` import { addDocument } from "./searchIndex"
import { map, seriesMap } from "."
interface Map {
// key: YYYY-MM-DD
// value: url
date: {
[key: string]: string[]
}
// key: tag name
// value: url
tags: {
[key: string]: string[]
}
// list of all meta data
meta: {
tags: string[]
}
// searchable, non-series posts
// must have a post date
// tag is not required
posts: {
[key: string]: {
title: string
date: string
tags: string[]
preview: string
}
}
// series posts have "previous post" and "next post" button so they need to be ordered
series: {
[key: string]: {
title: string
length: number
order: string[] // url order
tags: string[]
}
}
// urls of unsearchable posts
// it is here to quickly check if a post exists or not
unsearchable: {
[key: string]: {
title: string
}
}
}
interface SeriesMap {
// key: url
[key: string]: {
index: number
url: string
}[]
}
// searchable data that will be converted to JSON string
const map: Map = {
date: {},
tags: {},
meta: {
tags: [],
},
posts: {},
series: {},
unsearchable: {},
}
const seriesMap: SeriesMap = {}
const elasticlunrIndex = elasticlunr(function () {
this.addField("title" as never)
this.addField("body" as never)
this.setRef("url" as never)
})
const md = markdownIt({
// https://github.com/highlightjs/highlight.js/blob/main/SUPPORTED_LANGUAGES.md
highlight: function (str, lang) {
if (lang && hljs.getLanguage(lang)) {
try {
return hljs.highlight(str, { language: lang }).value
// eslint-disable-next-line no-empty
} catch (error) {}
}
return "" // use external default escaping
},
html: true,
})
.use(markdownItTexMath, {
engine: katex,
delimiters: "dollars",
katexOptions: { macros: { "\\RR": "\\mathbb{R}" } },
})
.use(markdownItAnchor, {})
// converts file path to url
function path2URL(pathToConvert: string): string {
return `/${path.relative(markdownPath, pathToConvert)}`
.replace(/\.[^/.]+$/, "") // remove the file extension
.replace(/ /g, "-") // replace all space with a dash
}
// gets the text after the last slash
function path2FileOrFolderName(inputPath: string): string {
// remove trailing slash
if (inputPath[-1] == "/")
inputPath = inputPath.slice(0, inputPath.length - 1)
// get the last section
return inputPath.slice(inputPath.lastIndexOf("/") + 1)
}
// gets the nth occurance of a pattern in string
// returns -1 if nothing is found
// https://stackoverflow.com/a/14482123/12979111
function nthIndex(str: string, pat: string, n: number) {
let i = -1
while (n-- && i++ < str.length) {
i = str.indexOf(pat, i)
if (i < 0) break
}
return i
}
function writeToJSON(JSONFilePath: string, dataToWrite: string) {
// create directory to put json content files
fs.mkdirSync(JSONFilePath.slice(0, JSONFilePath.lastIndexOf("/")), {
recursive: true,
})
// write content to json file
fs.writeFileSync(JSONFilePath, dataToWrite)
}
// A recursive function that calls itself for every files and directories that it finds // A recursive function that calls itself for every files and directories that it finds
function recursiveParse( export function recursiveParse(
mode: "posts" | "series" | "unsearchable", mode: "posts" | "series" | "unsearchable",
fileOrFolderPath: string fileOrFolderPath: string
) { ) {
@ -211,10 +60,7 @@ function recursiveParse(
if (!markdownData.title) if (!markdownData.title)
throw Error(`Title is not defined in file: ${fileOrFolderPath}`) throw Error(`Title is not defined in file: ${fileOrFolderPath}`)
const dom = new JSDOM( const dom = new JSDOM(parseMarkdown(markdownRaw))
md.render(markdownRaw.slice(nthIndex(markdownRaw, "---", 2) + 3)) ||
""
)
// add .hljs to all block codes // add .hljs to all block codes
dom.window.document.querySelectorAll("pre > code").forEach((item) => { dom.window.document.querySelectorAll("pre > code").forEach((item) => {
@ -296,7 +142,7 @@ function recursiveParse(
} }
map.posts[urlPath] = postData map.posts[urlPath] = postData
elasticlunrIndex.addDoc({ addDocument({
title: markdownData.title, title: markdownData.title,
body: markdownData.content, body: markdownData.content,
url: urlPath, url: urlPath,
@ -322,7 +168,7 @@ function recursiveParse(
title: markdownData.title, title: markdownData.title,
} }
elasticlunrIndex.addDoc({ addDocument({
title: markdownData.title, title: markdownData.title,
body: markdownData.content, body: markdownData.content,
url: urlPath, url: urlPath,
@ -404,7 +250,7 @@ function recursiveParse(
map.series[urlPath] = { ...postData, order: [], length: 0 } map.series[urlPath] = { ...postData, order: [], length: 0 }
} else { } else {
map.posts[urlPath] = postData map.posts[urlPath] = postData
elasticlunrIndex.addDoc({ addDocument({
title: markdownData.title, title: markdownData.title,
body: markdownData.content, body: markdownData.content,
url: urlPath, url: urlPath,
@ -443,77 +289,3 @@ function recursiveParse(
} }
} }
} }
/**
* Actual logic starts here
*/
// Delete existing files
try {
fs.rmSync(contentDirectoryPath, { recursive: true })
// eslint-disable-next-line no-empty
} catch (err) {}
try {
fs.unlinkSync(mapFilePath)
// eslint-disable-next-line no-empty
} catch (err) {}
// check if it's a directory and start recursive parse function
if (!fs.lstatSync(markdownPath).isDirectory())
throw Error("Invalid markdown path")
if (!fs.lstatSync(markdownPath + "/posts").isDirectory())
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
if (!fs.lstatSync(markdownPath + "/unsearchable").isDirectory())
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
if (!fs.lstatSync(markdownPath + "/series").isDirectory())
throw Error(`Cannot find directory: ${markdownPath + "/posts"}`)
recursiveParse("posts", markdownPath + "/posts")
recursiveParse("unsearchable", markdownPath + "/unsearchable")
recursiveParse("series", markdownPath + "/series")
// sort dates
let dateKeys: string[] = []
for (const dateKey in map.date) {
dateKeys.push(dateKey)
}
dateKeys = dateKeys.sort()
const TmpDate = map.date
map.date = {}
dateKeys.forEach((sortedDateKey) => {
map.date[sortedDateKey] = TmpDate[sortedDateKey]
})
// fill meta data
for (const tag in map.tags) {
map.meta.tags.push(tag)
}
// sort series post
for (const seriesURL in seriesMap) {
seriesMap[seriesURL].sort((a, b) => {
if (a.index < b.index) {
return -1
}
if (a.index > b.index) {
return 1
}
return 0
})
}
for (const seriesURL in seriesMap) {
map.series[seriesURL].length = seriesMap[seriesURL].length
map.series[seriesURL].order = seriesMap[seriesURL].map((item) => item.url)
}
fs.writeFileSync(mapFilePath, JSON.stringify(map))
fs.writeFileSync(outPath + "/search.json", JSON.stringify(elasticlunrIndex))

22
source/generate/searchIndex.ts Normal file
View file

@ -0,0 +1,22 @@
/**
* @file generate index for searching
*/
import fs from "fs"
import elasticlunr from "elasticlunr"
import { outPath } from "./config"
const elasticlunrIndex = elasticlunr(function () {
this.addField("title" as never)
this.addField("body" as never)
this.setRef("url" as never)
})
export function addDocument(doc: unknown) {
elasticlunrIndex.addDoc(doc)
}
export function saveIndex() {
fs.writeFileSync(outPath + "/search.json", JSON.stringify(elasticlunrIndex))
}

56
source/generate/types.ts Normal file
View file

@ -0,0 +1,56 @@
export interface Map {
// key: YYYY-MM-DD
// value: url
date: {
[key: string]: string[]
}
// key: tag name
// value: url
tags: {
[key: string]: string[]
}
// list of all meta data
meta: {
tags: string[]
}
// searchable, non-series posts
// must have a post date
// tag is not required
posts: {
[key: string]: {
title: string
date: string
tags: string[]
preview: string
}
}
// series posts have "previous post" and "next post" button so they need to be ordered
series: {
[key: string]: {
title: string
length: number
order: string[] // url order
tags: string[]
}
}
// urls of unsearchable posts
// it is here to quickly check if a post exists or not
unsearchable: {
[key: string]: {
title: string
}
}
}
export interface SeriesMap {
// key: url
[key: string]: {
index: number
url: string
}[]
}

45
source/generate/util.ts Normal file
View file

@ -0,0 +1,45 @@
import fs from "fs"
import { relative } from "path"
import { markdownPath } from "./config"
// converts file path to url
export function path2URL(pathToConvert: string): string {
return `/${relative(markdownPath, pathToConvert)}`
.replace(/\.[^/.]+$/, "") // remove the file extension
.replace(/ /g, "-") // replace all space with a dash
}
// gets the text after the last slash
export function path2FileOrFolderName(inputPath: string): string {
// remove trailing slash
if (inputPath[-1] == "/")
inputPath = inputPath.slice(0, inputPath.length - 1)
// get the last section
return inputPath.slice(inputPath.lastIndexOf("/") + 1)
}
// gets the nth occurance of a pattern in string
// returns -1 if nothing is found
// https://stackoverflow.com/a/14482123/12979111
export function nthIndex(str: string, pat: string, n: number) {
let i = -1
while (n-- && i++ < str.length) {
i = str.indexOf(pat, i)
if (i < 0) break
}
return i
}
export function writeToJSON(JSONFilePath: string, dataToWrite: string) {
// create directory to put json content files
fs.mkdirSync(JSONFilePath.slice(0, JSONFilePath.lastIndexOf("/")), {
recursive: true,
})
// write content to json file
fs.writeFileSync(JSONFilePath, dataToWrite)
}

2
source/package.json
View file

@ -11,7 +11,7 @@
"private": true, "private": true,
"license": "MIT", "license": "MIT",
"scripts": { "scripts": {
"generate": "ts-node -O '{\"module\":\"commonjs\"}' --files ./generate.ts", "generate": "ts-node -O '{\"module\":\"commonjs\"}' --files ./generate",
"start": "yarn generate && react-scripts start", "start": "yarn generate && react-scripts start",
"quick-start": "react-scripts start", "quick-start": "react-scripts start",
"build": "yarn generate && react-scripts build" "build": "yarn generate && react-scripts build"

21
source/src/pages/PostList.tsx
View file

@ -1,15 +1,17 @@
/** PostList.tsx /**
* show posts in recent order * PostList.tsx
* show posts in recent order
*/ */
import React, { useEffect, useState } from "react" import { useEffect, useState } from "react"
import styled from "styled-components"
import { Helmet } from "react-helmet-async" import { Helmet } from "react-helmet-async"
import styled from "styled-components"
import theming from "../styles/theming"
import _map from "../data/map.json"
import PostCard from "../components/PostCard" import PostCard from "../components/PostCard"
import _map from "../data/map.json"
import theming from "../styles/theming"
import { Map } from "../types/typings" import { Map } from "../types/typings"
const map: Map = _map const map: Map = _map
@ -31,16 +33,17 @@ interface Props {
const PostList = (props: Props) => { const PostList = (props: Props) => {
const howMany = props.howMany || 0 const howMany = props.howMany || 0
const [postCards, setPostCards] = useState([] as unknown[]) const [postCards, setPostCards] = useState<JSX.Element[]>([])
useEffect(() => { useEffect(() => {
let postCount = 0 let postCount = 0
const _postCards = [] as unknown[] const _postCards = [] as JSX.Element[]
for (const date in map.date) { for (const date in map.date) {
if (postCount >= howMany) break if (postCount >= howMany) break
const length = map.date[date].length const length = map.date[date].length
for (let i = 0; i < length; i++) { for (let i = 0; i < length; i++) {
if (postCount >= howMany) break if (postCount >= howMany) break

2
source/tsconfig.json
View file

@ -17,5 +17,5 @@
"noEmit": true, "noEmit": true,
"jsx": "react-jsx" "jsx": "react-jsx"
}, },
"include": ["src/**/*", "types/**/*", "generate.ts"] "include": ["src/**/*", "types/**/*", "generate/**/*"]
} }