diff --git a/source/generate/config.ts b/source/generate/config.ts new file mode 100644 index 0000000..f848e9c --- /dev/null +++ b/source/generate/config.ts @@ -0,0 +1,5 @@ +export const markdownPath = "./markdown" // where it will look for markdown documents +export const outPath = "./src/data" // path to the json database + +export const contentDirectoryPath = `${outPath}/content` +export const mapFilePath = `${outPath}/map.json` diff --git a/source/generate/index.ts b/source/generate/index.ts new file mode 100644 index 0000000..1d708be --- /dev/null +++ b/source/generate/index.ts @@ -0,0 +1,99 @@ +/** + * @file Read markdown files and write their content and metadata to json files which can then be imported by React. + * - File and directory names starting with an underscore (_) are ignored. + * - Symbolic links are not supported. + * - The filename-to-URL converter isn't perfect. Some non-URL-friendly filenames might cause problems. + * - series must start with a number followed by an underscore + */ + +import fs from "fs" // read and write files + +import { Map, SeriesMap } from "./types" +import { recursiveParse } from "./recursiveParse" +import { contentDirectoryPath, mapFilePath, markdownPath } from "./config" +import { saveIndex } from "./searchIndex" + +// searchable data that will be converted to JSON string +export const map: Map = { + date: {}, + tags: {}, + meta: { + tags: [], + }, + posts: {}, + series: {}, + unsearchable: {}, +} +export const seriesMap: SeriesMap = {} + +/** + * Delete existing files + */ + +try { + fs.rmSync(contentDirectoryPath, { recursive: true }) + // eslint-disable-next-line no-empty +} catch (err) {} + +try { + fs.unlinkSync(mapFilePath) + // eslint-disable-next-line no-empty +} catch (err) {} + +// check if it's a directory and start recursive parse function +if (!fs.lstatSync(markdownPath).isDirectory()) + throw Error("Invalid markdown path") + +if (!fs.lstatSync(markdownPath + "/posts").isDirectory()) + throw Error(`Cannot find directory: ${markdownPath + "/posts"}`) + +if (!fs.lstatSync(markdownPath + "/unsearchable").isDirectory()) + throw Error(`Cannot find directory: ${markdownPath + "/posts"}`) + +if (!fs.lstatSync(markdownPath + "/series").isDirectory()) + throw Error(`Cannot find directory: ${markdownPath + "/posts"}`) + +recursiveParse("posts", markdownPath + "/posts") +recursiveParse("unsearchable", markdownPath + "/unsearchable") +recursiveParse("series", markdownPath + "/series") + +// sort dates +let dateKeys: string[] = [] +for (const dateKey in map.date) { + dateKeys.push(dateKey) +} + +dateKeys = dateKeys.sort() + +const TmpDate = map.date +map.date = {} + +dateKeys.forEach((sortedDateKey) => { + map.date[sortedDateKey] = TmpDate[sortedDateKey] +}) + +// fill meta data +for (const tag in map.tags) { + map.meta.tags.push(tag) +} + +// sort series post +for (const seriesURL in seriesMap) { + seriesMap[seriesURL].sort((a, b) => { + if (a.index < b.index) { + return -1 + } + if (a.index > b.index) { + return 1 + } + return 0 + }) +} + +for (const seriesURL in seriesMap) { + map.series[seriesURL].length = seriesMap[seriesURL].length + map.series[seriesURL].order = seriesMap[seriesURL].map((item) => item.url) +} + +fs.writeFileSync(mapFilePath, JSON.stringify(map)) +saveIndex() diff --git a/source/generate/parseMarkdown.ts b/source/generate/parseMarkdown.ts new file mode 100644 index 0000000..37d03fd --- /dev/null +++ b/source/generate/parseMarkdown.ts @@ -0,0 +1,33 @@ +import markdownIt from "markdown-it" // rendering markdown +import markdownItTexMath from "markdown-it-texmath" // rendering mathematical expression +import markdownItAnchor from "markdown-it-anchor" // markdown anchor +import hljs from "highlight.js" // code block highlighting +import katex from "katex" // rendering mathematical expression +import { nthIndex } from "./util" + +const md = markdownIt({ + // https://github.com/highlightjs/highlight.js/blob/main/SUPPORTED_LANGUAGES.md + highlight: function (str, lang) { + if (lang && hljs.getLanguage(lang)) { + try { + return hljs.highlight(str, { language: lang }).value + // eslint-disable-next-line no-empty + } catch (error) {} + } + + return "" // use external default escaping + }, + html: true, +}) + .use(markdownItTexMath, { + engine: katex, + delimiters: "dollars", + katexOptions: { macros: { "\\RR": "\\mathbb{R}" } }, + }) + .use(markdownItAnchor, {}) + +export function parseMarkdown(markdownRaw: string): string { + return ( + md.render(markdownRaw.slice(nthIndex(markdownRaw, "---", 2) + 3)) || "" + ) +} diff --git a/source/generate.ts b/source/generate/recursiveParse.ts similarity index 54% rename from source/generate.ts rename to source/generate/recursiveParse.ts index dca1988..df7eae4 100644 --- a/source/generate.ts +++ b/source/generate/recursiveParse.ts @@ -1,169 +1,18 @@ -/** - * @file Read markdown files and write their content and metadata to json files which can then be imported by React. - * - File and directory names starting with an underscore (_) are ignored. - * - Symbolic links are not supported. - * - The filename-to-URL converter isn't perfect. Some non-URL-friendly filenames might cause problems. - * - series must start with a number followed by an underscore - */ - -import fs from "fs" // read and write files -import path from "path" // get relative path -import elasticlunr from "elasticlunr" // search index generation +import fs from "fs" import readTimeEstimate from "read-time-estimate" // post read time estimation import matter from "gray-matter" // parse markdown metadata import toc from "markdown-toc" // table of contents generation -import markdownIt from "markdown-it" // rendering markdown -import markdownItTexMath from "markdown-it-texmath" // rendering mathematical expression -import markdownItAnchor from "markdown-it-anchor" // markdown anchor -import hljs from "highlight.js" // code block highlighting -import katex from "katex" // rendering mathematical expression import { JSDOM } from "jsdom" // HTML DOM parsing -const markdownPath = "./markdown" // where it will look for markdown documents -const outPath = "./src/data" // path to the json database +import { nthIndex, path2FileOrFolderName, path2URL, writeToJSON } from "./util" +import { parseMarkdown } from "./parseMarkdown" -const contentDirectoryPath = `${outPath}/content` -const mapFilePath = `${outPath}/map.json` - -interface Map { - // key: YYYY-MM-DD - // value: url - date: { - [key: string]: string[] - } - - // key: tag name - // value: url - tags: { - [key: string]: string[] - } - - // list of all meta data - meta: { - tags: string[] - } - - // searchable, non-series posts - // must have a post date - // tag is not required - posts: { - [key: string]: { - title: string - date: string - tags: string[] - preview: string - } - } - - // series posts have "previous post" and "next post" button so they need to be ordered - series: { - [key: string]: { - title: string - length: number - order: string[] // url order - tags: string[] - } - } - - // urls of unsearchable posts - // it is here to quickly check if a post exists or not - unsearchable: { - [key: string]: { - title: string - } - } -} - -interface SeriesMap { - // key: url - [key: string]: { - index: number - url: string - }[] -} - -// searchable data that will be converted to JSON string -const map: Map = { - date: {}, - tags: {}, - meta: { - tags: [], - }, - posts: {}, - series: {}, - unsearchable: {}, -} -const seriesMap: SeriesMap = {} -const elasticlunrIndex = elasticlunr(function () { - this.addField("title" as never) - this.addField("body" as never) - this.setRef("url" as never) -}) - -const md = markdownIt({ - // https://github.com/highlightjs/highlight.js/blob/main/SUPPORTED_LANGUAGES.md - highlight: function (str, lang) { - if (lang && hljs.getLanguage(lang)) { - try { - return hljs.highlight(str, { language: lang }).value - // eslint-disable-next-line no-empty - } catch (error) {} - } - - return "" // use external default escaping - }, - html: true, -}) - .use(markdownItTexMath, { - engine: katex, - delimiters: "dollars", - katexOptions: { macros: { "\\RR": "\\mathbb{R}" } }, - }) - .use(markdownItAnchor, {}) - -// converts file path to url -function path2URL(pathToConvert: string): string { - return `/${path.relative(markdownPath, pathToConvert)}` - .replace(/\.[^/.]+$/, "") // remove the file extension - .replace(/ /g, "-") // replace all space with a dash -} - -// gets the text after the last slash -function path2FileOrFolderName(inputPath: string): string { - // remove trailing slash - if (inputPath[-1] == "/") - inputPath = inputPath.slice(0, inputPath.length - 1) - - // get the last section - return inputPath.slice(inputPath.lastIndexOf("/") + 1) -} - -// gets the nth occurance of a pattern in string -// returns -1 if nothing is found -// https://stackoverflow.com/a/14482123/12979111 -function nthIndex(str: string, pat: string, n: number) { - let i = -1 - - while (n-- && i++ < str.length) { - i = str.indexOf(pat, i) - if (i < 0) break - } - - return i -} - -function writeToJSON(JSONFilePath: string, dataToWrite: string) { - // create directory to put json content files - fs.mkdirSync(JSONFilePath.slice(0, JSONFilePath.lastIndexOf("/")), { - recursive: true, - }) - - // write content to json file - fs.writeFileSync(JSONFilePath, dataToWrite) -} +import { contentDirectoryPath } from "./config" +import { addDocument } from "./searchIndex" +import { map, seriesMap } from "." // A recursive function that calls itself for every files and directories that it finds -function recursiveParse( +export function recursiveParse( mode: "posts" | "series" | "unsearchable", fileOrFolderPath: string ) { @@ -211,10 +60,7 @@ function recursiveParse( if (!markdownData.title) throw Error(`Title is not defined in file: ${fileOrFolderPath}`) - const dom = new JSDOM( - md.render(markdownRaw.slice(nthIndex(markdownRaw, "---", 2) + 3)) || - "" - ) + const dom = new JSDOM(parseMarkdown(markdownRaw)) // add .hljs to all block codes dom.window.document.querySelectorAll("pre > code").forEach((item) => { @@ -296,7 +142,7 @@ function recursiveParse( } map.posts[urlPath] = postData - elasticlunrIndex.addDoc({ + addDocument({ title: markdownData.title, body: markdownData.content, url: urlPath, @@ -322,7 +168,7 @@ function recursiveParse( title: markdownData.title, } - elasticlunrIndex.addDoc({ + addDocument({ title: markdownData.title, body: markdownData.content, url: urlPath, @@ -404,7 +250,7 @@ function recursiveParse( map.series[urlPath] = { ...postData, order: [], length: 0 } } else { map.posts[urlPath] = postData - elasticlunrIndex.addDoc({ + addDocument({ title: markdownData.title, body: markdownData.content, url: urlPath, @@ -443,77 +289,3 @@ function recursiveParse( } } } - -/** - * Actual logic starts here - */ - -// Delete existing files - -try { - fs.rmSync(contentDirectoryPath, { recursive: true }) - // eslint-disable-next-line no-empty -} catch (err) {} - -try { - fs.unlinkSync(mapFilePath) - // eslint-disable-next-line no-empty -} catch (err) {} - -// check if it's a directory and start recursive parse function -if (!fs.lstatSync(markdownPath).isDirectory()) - throw Error("Invalid markdown path") - -if (!fs.lstatSync(markdownPath + "/posts").isDirectory()) - throw Error(`Cannot find directory: ${markdownPath + "/posts"}`) - -if (!fs.lstatSync(markdownPath + "/unsearchable").isDirectory()) - throw Error(`Cannot find directory: ${markdownPath + "/posts"}`) - -if (!fs.lstatSync(markdownPath + "/series").isDirectory()) - throw Error(`Cannot find directory: ${markdownPath + "/posts"}`) - -recursiveParse("posts", markdownPath + "/posts") -recursiveParse("unsearchable", markdownPath + "/unsearchable") -recursiveParse("series", markdownPath + "/series") - -// sort dates -let dateKeys: string[] = [] -for (const dateKey in map.date) { - dateKeys.push(dateKey) -} - -dateKeys = dateKeys.sort() - -const TmpDate = map.date -map.date = {} - -dateKeys.forEach((sortedDateKey) => { - map.date[sortedDateKey] = TmpDate[sortedDateKey] -}) - -// fill meta data -for (const tag in map.tags) { - map.meta.tags.push(tag) -} - -// sort series post -for (const seriesURL in seriesMap) { - seriesMap[seriesURL].sort((a, b) => { - if (a.index < b.index) { - return -1 - } - if (a.index > b.index) { - return 1 - } - return 0 - }) -} - -for (const seriesURL in seriesMap) { - map.series[seriesURL].length = seriesMap[seriesURL].length - map.series[seriesURL].order = seriesMap[seriesURL].map((item) => item.url) -} - -fs.writeFileSync(mapFilePath, JSON.stringify(map)) -fs.writeFileSync(outPath + "/search.json", JSON.stringify(elasticlunrIndex)) diff --git a/source/generate/searchIndex.ts b/source/generate/searchIndex.ts new file mode 100644 index 0000000..2fea42b --- /dev/null +++ b/source/generate/searchIndex.ts @@ -0,0 +1,22 @@ +/** + * @file generate index for searching + */ + +import fs from "fs" +import elasticlunr from "elasticlunr" + +import { outPath } from "./config" + +const elasticlunrIndex = elasticlunr(function () { + this.addField("title" as never) + this.addField("body" as never) + this.setRef("url" as never) +}) + +export function addDocument(doc: unknown) { + elasticlunrIndex.addDoc(doc) +} + +export function saveIndex() { + fs.writeFileSync(outPath + "/search.json", JSON.stringify(elasticlunrIndex)) +} diff --git a/source/generate/types.ts b/source/generate/types.ts new file mode 100644 index 0000000..06bbe3f --- /dev/null +++ b/source/generate/types.ts @@ -0,0 +1,56 @@ +export interface Map { + // key: YYYY-MM-DD + // value: url + date: { + [key: string]: string[] + } + + // key: tag name + // value: url + tags: { + [key: string]: string[] + } + + // list of all meta data + meta: { + tags: string[] + } + + // searchable, non-series posts + // must have a post date + // tag is not required + posts: { + [key: string]: { + title: string + date: string + tags: string[] + preview: string + } + } + + // series posts have "previous post" and "next post" button so they need to be ordered + series: { + [key: string]: { + title: string + length: number + order: string[] // url order + tags: string[] + } + } + + // urls of unsearchable posts + // it is here to quickly check if a post exists or not + unsearchable: { + [key: string]: { + title: string + } + } +} + +export interface SeriesMap { + // key: url + [key: string]: { + index: number + url: string + }[] +} diff --git a/source/generate/util.ts b/source/generate/util.ts new file mode 100644 index 0000000..d83b50d --- /dev/null +++ b/source/generate/util.ts @@ -0,0 +1,45 @@ +import fs from "fs" +import { relative } from "path" + +import { markdownPath } from "./config" + +// converts file path to url +export function path2URL(pathToConvert: string): string { + return `/${relative(markdownPath, pathToConvert)}` + .replace(/\.[^/.]+$/, "") // remove the file extension + .replace(/ /g, "-") // replace all space with a dash +} + +// gets the text after the last slash +export function path2FileOrFolderName(inputPath: string): string { + // remove trailing slash + if (inputPath[-1] == "/") + inputPath = inputPath.slice(0, inputPath.length - 1) + + // get the last section + return inputPath.slice(inputPath.lastIndexOf("/") + 1) +} + +// gets the nth occurance of a pattern in string +// returns -1 if nothing is found +// https://stackoverflow.com/a/14482123/12979111 +export function nthIndex(str: string, pat: string, n: number) { + let i = -1 + + while (n-- && i++ < str.length) { + i = str.indexOf(pat, i) + if (i < 0) break + } + + return i +} + +export function writeToJSON(JSONFilePath: string, dataToWrite: string) { + // create directory to put json content files + fs.mkdirSync(JSONFilePath.slice(0, JSONFilePath.lastIndexOf("/")), { + recursive: true, + }) + + // write content to json file + fs.writeFileSync(JSONFilePath, dataToWrite) +} diff --git a/source/package.json b/source/package.json index bfbbebf..78d1b26 100644 --- a/source/package.json +++ b/source/package.json @@ -11,7 +11,7 @@ "private": true, "license": "MIT", "scripts": { - "generate": "ts-node -O '{\"module\":\"commonjs\"}' --files ./generate.ts", + "generate": "ts-node -O '{\"module\":\"commonjs\"}' --files ./generate", "start": "yarn generate && react-scripts start", "quick-start": "react-scripts start", "build": "yarn generate && react-scripts build" diff --git a/source/src/pages/PostList.tsx b/source/src/pages/PostList.tsx index 465289d..def7c3e 100644 --- a/source/src/pages/PostList.tsx +++ b/source/src/pages/PostList.tsx @@ -1,15 +1,17 @@ -/** PostList.tsx - * show posts in recent order +/** + * PostList.tsx + * show posts in recent order */ -import React, { useEffect, useState } from "react" -import styled from "styled-components" +import { useEffect, useState } from "react" import { Helmet } from "react-helmet-async" - -import theming from "../styles/theming" -import _map from "../data/map.json" +import styled from "styled-components" import PostCard from "../components/PostCard" + +import _map from "../data/map.json" +import theming from "../styles/theming" + import { Map } from "../types/typings" const map: Map = _map @@ -31,16 +33,17 @@ interface Props { const PostList = (props: Props) => { const howMany = props.howMany || 0 - const [postCards, setPostCards] = useState([] as unknown[]) + const [postCards, setPostCards] = useState([]) useEffect(() => { let postCount = 0 - const _postCards = [] as unknown[] + const _postCards = [] as JSX.Element[] for (const date in map.date) { if (postCount >= howMany) break const length = map.date[date].length + for (let i = 0; i < length; i++) { if (postCount >= howMany) break diff --git a/source/tsconfig.json b/source/tsconfig.json index 3f8eebe..778292f 100644 --- a/source/tsconfig.json +++ b/source/tsconfig.json @@ -17,5 +17,5 @@ "noEmit": true, "jsx": "react-jsx" }, - "include": ["src/**/*", "types/**/*", "generate.ts"] + "include": ["src/**/*", "types/**/*", "generate/**/*"] }