fix watch-mode batching

This commit is contained in:
Jacky Zhao 2023-07-24 00:04:01 -07:00
parent 569ff1a801
commit 041a4ce7bc
14 changed files with 91 additions and 77 deletions

View file

@ -4,7 +4,6 @@ draft: true
## high priority ## high priority
- back button doesn't work sometimes
- images in same folder are broken on shortest path mode - images in same folder are broken on shortest path mode
- https://help.obsidian.md/Editing+and+formatting/Tags#Nested+tags nested tags?? and big tag listing - https://help.obsidian.md/Editing+and+formatting/Tags#Nested+tags nested tags?? and big tag listing
- watch mode for config/source code - watch mode for config/source code

View file

@ -10,7 +10,7 @@ import { parseMarkdown } from "./processors/parse"
import { filterContent } from "./processors/filter" import { filterContent } from "./processors/filter"
import { emitContent } from "./processors/emit" import { emitContent } from "./processors/emit"
import cfg from "../quartz.config" import cfg from "../quartz.config"
import { FilePath } from "./path" import { FilePath, slugifyFilePath } from "./path"
import chokidar from "chokidar" import chokidar from "chokidar"
import { ProcessedContent } from "./plugins/vfile" import { ProcessedContent } from "./plugins/vfile"
import WebSocket, { WebSocketServer } from "ws" import WebSocket, { WebSocketServer } from "ws"
@ -20,6 +20,7 @@ async function buildQuartz(argv: Argv, version: string) {
const ctx: BuildCtx = { const ctx: BuildCtx = {
argv, argv,
cfg, cfg,
allSlugs: [],
} }
console.log(chalk.bgGreen.black(`\n Quartz v${version} \n`)) console.log(chalk.bgGreen.black(`\n Quartz v${version} \n`))
@ -51,6 +52,8 @@ async function buildQuartz(argv: Argv, version: string) {
) )
const filePaths = fps.map((fp) => `${argv.directory}${path.sep}${fp}` as FilePath) const filePaths = fps.map((fp) => `${argv.directory}${path.sep}${fp}` as FilePath)
ctx.allSlugs = fps.map((fp) => slugifyFilePath(fp as FilePath))
const parsedFiles = await parseMarkdown(ctx, filePaths) const parsedFiles = await parseMarkdown(ctx, filePaths)
const filteredContent = filterContent(ctx, parsedFiles) const filteredContent = filterContent(ctx, parsedFiles)
await emitContent(ctx, filteredContent) await emitContent(ctx, filteredContent)
@ -74,30 +77,54 @@ async function startServing(ctx: BuildCtx, initialContent: ProcessedContent[]) {
contentMap.set(vfile.data.filePath!, content) contentMap.set(vfile.data.filePath!, content)
} }
async function rebuild(fp: string, action: "add" | "change" | "unlink") { let timeoutId: ReturnType<typeof setTimeout> | null = null
const perf = new PerfTimer() let toRebuild: Set<FilePath> = new Set()
let toRemove: Set<FilePath> = new Set()
async function rebuild(fp: string, action: "add" | "change" | "delete") {
if (!ignored(fp)) { if (!ignored(fp)) {
console.log(chalk.yellow(`Detected change in ${fp}, rebuilding...`)) const filePath = `${argv.directory}${path.sep}${fp}` as FilePath
const fullPath = `${argv.directory}${path.sep}${fp}` as FilePath if (action === "add" || action === "change") {
toRebuild.add(filePath)
try { } else if (action === "delete") {
if (action === "add" || action === "change") { toRemove.add(filePath)
const [parsedContent] = await parseMarkdown(ctx, [fullPath])
contentMap.set(fullPath, parsedContent)
} else if (action === "unlink") {
contentMap.delete(fullPath)
}
await rimraf(argv.output)
const parsedFiles = [...contentMap.values()]
const filteredContent = filterContent(ctx, parsedFiles)
await emitContent(ctx, filteredContent)
console.log(chalk.green(`Done rebuilding in ${perf.timeSince()}`))
} catch {
console.log(chalk.yellow(`Rebuild failed. Waiting on a change to fix the error...`))
} }
connections.forEach((conn) => conn.send("rebuild")) if (timeoutId) {
clearTimeout(timeoutId)
}
timeoutId = setTimeout(async () => {
const perf = new PerfTimer()
console.log(chalk.yellow("Detected change, rebuilding..."))
try {
const filesToRebuild = [...toRebuild].filter((fp) => !toRemove.has(fp))
ctx.allSlugs = [...new Set([...contentMap.keys(), ...toRebuild])]
.filter((fp) => !toRemove.has(fp))
.map((fp) => slugifyFilePath(path.relative(argv.directory, fp) as FilePath))
const parsedContent = await parseMarkdown(ctx, filesToRebuild)
for (const content of parsedContent) {
const [_tree, vfile] = content
contentMap.set(vfile.data.filePath!, content)
}
for (const fp of toRemove) {
contentMap.delete(fp)
}
await rimraf(argv.output)
const parsedFiles = [...contentMap.values()]
const filteredContent = filterContent(ctx, parsedFiles)
await emitContent(ctx, filteredContent)
console.log(chalk.green(`Done rebuilding in ${perf.timeSince()}`))
} catch {
console.log(chalk.yellow(`Rebuild failed. Waiting on a change to fix the error...`))
}
connections.forEach((conn) => conn.send("rebuild"))
toRebuild.clear()
toRemove.clear()
}, 250)
} }
} }
@ -110,7 +137,7 @@ async function startServing(ctx: BuildCtx, initialContent: ProcessedContent[]) {
watcher watcher
.on("add", (fp) => rebuild(fp, "add")) .on("add", (fp) => rebuild(fp, "add"))
.on("change", (fp) => rebuild(fp, "change")) .on("change", (fp) => rebuild(fp, "change"))
.on("unlink", (fp) => rebuild(fp, "unlink")) .on("unlink", (fp) => rebuild(fp, "delete"))
const server = http.createServer(async (req, res) => { const server = http.createServer(async (req, res) => {
await serveHandler(req, res, { await serveHandler(req, res, {

View file

@ -1,4 +1,5 @@
import { QuartzConfig } from "./cfg" import { QuartzConfig } from "./cfg"
import { ServerSlug } from "./path"
export interface Argv { export interface Argv {
directory: string directory: string
@ -11,4 +12,5 @@ export interface Argv {
export interface BuildCtx { export interface BuildCtx {
argv: Argv argv: Argv
cfg: QuartzConfig cfg: QuartzConfig
allSlugs: ServerSlug[]
} }

View file

@ -20,10 +20,10 @@ type ComponentResources = {
afterDOMLoaded: string[] afterDOMLoaded: string[]
} }
function getComponentResources(plugins: PluginTypes): ComponentResources { function getComponentResources(ctx: BuildCtx): ComponentResources {
const allComponents: Set<QuartzComponent> = new Set() const allComponents: Set<QuartzComponent> = new Set()
for (const emitter of plugins.emitters) { for (const emitter of ctx.cfg.plugins.emitters) {
const components = emitter.getQuartzComponents() const components = emitter.getQuartzComponents(ctx)
for (const component of components) { for (const component of components) {
allComponents.add(component) allComponents.add(component)
} }
@ -127,7 +127,7 @@ export const ComponentResources: QuartzEmitterPlugin = () => ({
}, },
async emit(ctx, _content, resources, emit): Promise<FilePath[]> { async emit(ctx, _content, resources, emit): Promise<FilePath[]> {
// component specific scripts and styles // component specific scripts and styles
const componentResources = getComponentResources(ctx.cfg.plugins) const componentResources = getComponentResources(ctx)
// important that this goes *after* component scripts // important that this goes *after* component scripts
// as the "nav" event gets triggered here and we should make sure // as the "nav" event gets triggered here and we should make sure
// that everyone else had the chance to register a listener for it // that everyone else had the chance to register a listener for it

View file

@ -2,7 +2,7 @@ import { QuartzFilterPlugin } from "../types"
export const RemoveDrafts: QuartzFilterPlugin<{}> = () => ({ export const RemoveDrafts: QuartzFilterPlugin<{}> = () => ({
name: "RemoveDrafts", name: "RemoveDrafts",
shouldPublish([_tree, vfile]) { shouldPublish(_ctx, [_tree, vfile]) {
const draftFlag: boolean = vfile.data?.frontmatter?.draft ?? false const draftFlag: boolean = vfile.data?.frontmatter?.draft ?? false
return !draftFlag return !draftFlag
}, },

View file

@ -2,7 +2,7 @@ import { QuartzFilterPlugin } from "../types"
export const ExplicitPublish: QuartzFilterPlugin = () => ({ export const ExplicitPublish: QuartzFilterPlugin = () => ({
name: "ExplicitPublish", name: "ExplicitPublish",
shouldPublish([_tree, vfile]) { shouldPublish(_ctx, [_tree, vfile]) {
const publishFlag: boolean = vfile.data?.frontmatter?.publish ?? false const publishFlag: boolean = vfile.data?.frontmatter?.publish ?? false
return publishFlag return publishFlag
}, },

View file

@ -1,15 +1,15 @@
import { StaticResources } from "../resources" import { StaticResources } from "../resources"
import { PluginTypes } from "./types"
import { FilePath, ServerSlug } from "../path" import { FilePath, ServerSlug } from "../path"
import { BuildCtx } from "../ctx"
export function getStaticResourcesFromPlugins(plugins: PluginTypes) { export function getStaticResourcesFromPlugins(ctx: BuildCtx) {
const staticResources: StaticResources = { const staticResources: StaticResources = {
css: [], css: [],
js: [], js: [],
} }
for (const transformer of plugins.transformers) { for (const transformer of ctx.cfg.plugins.transformers) {
const res = transformer.externalResources ? transformer.externalResources() : {} const res = transformer.externalResources ? transformer.externalResources(ctx) : {}
if (res?.js) { if (res?.js) {
staticResources.js.push(...res.js) staticResources.js.push(...res.js)
} }
@ -29,7 +29,6 @@ declare module "vfile" {
// inserted in processors.ts // inserted in processors.ts
interface DataMap { interface DataMap {
slug: ServerSlug slug: ServerSlug
allSlugs: ServerSlug[]
filePath: FilePath filePath: FilePath
} }
} }

View file

@ -29,7 +29,7 @@ export const CrawlLinks: QuartzTransformerPlugin<Partial<Options> | undefined> =
const opts = { ...defaultOptions, ...userOpts } const opts = { ...defaultOptions, ...userOpts }
return { return {
name: "LinkProcessing", name: "LinkProcessing",
htmlPlugins() { htmlPlugins(ctx) {
return [ return [
() => { () => {
return (tree, file) => { return (tree, file) => {
@ -40,11 +40,8 @@ export const CrawlLinks: QuartzTransformerPlugin<Partial<Options> | undefined> =
if (opts.markdownLinkResolution === "relative") { if (opts.markdownLinkResolution === "relative") {
return targetSlug as RelativeURL return targetSlug as RelativeURL
} else if (opts.markdownLinkResolution === "shortest") { } else if (opts.markdownLinkResolution === "shortest") {
// https://forum.obsidian.md/t/settings-new-link-format-what-is-shortest-path-when-possible/6748/5
const allSlugs = file.data.allSlugs!
// if the file name is unique, then it's just the filename // if the file name is unique, then it's just the filename
const matchingFileNames = allSlugs.filter((slug) => { const matchingFileNames = ctx.allSlugs.filter((slug) => {
const parts = slug.split(path.posix.sep) const parts = slug.split(path.posix.sep)
const fileName = parts.at(-1) const fileName = parts.at(-1)
return targetCanonical === fileName return targetCanonical === fileName

View file

@ -119,7 +119,7 @@ export const ObsidianFlavoredMarkdown: QuartzTransformerPlugin<Partial<Options>
const opts = { ...defaultOptions, ...userOpts } const opts = { ...defaultOptions, ...userOpts }
return { return {
name: "ObsidianFlavoredMarkdown", name: "ObsidianFlavoredMarkdown",
textTransform(src) { textTransform(_ctx, src) {
// pre-transform wikilinks (fix anchors to things that may contain illegal syntax e.g. codeblocks, latex) // pre-transform wikilinks (fix anchors to things that may contain illegal syntax e.g. codeblocks, latex)
if (opts.wikilinks) { if (opts.wikilinks) {
src = src.toString() src = src.toString()

View file

@ -1,7 +1,6 @@
import { PluggableList } from "unified" import { PluggableList } from "unified"
import { StaticResources } from "../resources" import { StaticResources } from "../resources"
import { ProcessedContent } from "./vfile" import { ProcessedContent } from "./vfile"
import { GlobalConfiguration } from "../cfg"
import { QuartzComponent } from "../components/types" import { QuartzComponent } from "../components/types"
import { FilePath, ServerSlug } from "../path" import { FilePath, ServerSlug } from "../path"
import { BuildCtx } from "../ctx" import { BuildCtx } from "../ctx"
@ -18,10 +17,10 @@ export type QuartzTransformerPlugin<Options extends OptionType = undefined> = (
) => QuartzTransformerPluginInstance ) => QuartzTransformerPluginInstance
export type QuartzTransformerPluginInstance = { export type QuartzTransformerPluginInstance = {
name: string name: string
textTransform?: (src: string | Buffer) => string | Buffer textTransform?: (ctx: BuildCtx, src: string | Buffer) => string | Buffer
markdownPlugins?: () => PluggableList markdownPlugins?: (ctx: BuildCtx) => PluggableList
htmlPlugins?: () => PluggableList htmlPlugins?: (ctx: BuildCtx) => PluggableList
externalResources?: () => Partial<StaticResources> externalResources?: (ctx: BuildCtx) => Partial<StaticResources>
} }
export type QuartzFilterPlugin<Options extends OptionType = undefined> = ( export type QuartzFilterPlugin<Options extends OptionType = undefined> = (
@ -29,7 +28,7 @@ export type QuartzFilterPlugin<Options extends OptionType = undefined> = (
) => QuartzFilterPluginInstance ) => QuartzFilterPluginInstance
export type QuartzFilterPluginInstance = { export type QuartzFilterPluginInstance = {
name: string name: string
shouldPublish(content: ProcessedContent): boolean shouldPublish(ctx: BuildCtx, content: ProcessedContent): boolean
} }
export type QuartzEmitterPlugin<Options extends OptionType = undefined> = ( export type QuartzEmitterPlugin<Options extends OptionType = undefined> = (
@ -43,7 +42,7 @@ export type QuartzEmitterPluginInstance = {
resources: StaticResources, resources: StaticResources,
emitCallback: EmitCallback, emitCallback: EmitCallback,
): Promise<FilePath[]> ): Promise<FilePath[]>
getQuartzComponents(): QuartzComponent[] getQuartzComponents(ctx: BuildCtx): QuartzComponent[]
} }
export interface EmitOptions { export interface EmitOptions {

View file

@ -24,7 +24,7 @@ export async function emitContent(ctx: BuildCtx, content: ProcessedContent[]) {
} }
let emittedFiles = 0 let emittedFiles = 0
const staticResources = getStaticResourcesFromPlugins(cfg.plugins) const staticResources = getStaticResourcesFromPlugins(ctx)
for (const emitter of cfg.plugins.emitters) { for (const emitter of cfg.plugins.emitters) {
try { try {
const emitted = await emitter.emit(ctx, content, staticResources, emit) const emitted = await emitter.emit(ctx, content, staticResources, emit)

View file

@ -1,16 +1,13 @@
import { BuildCtx } from "../ctx" import { BuildCtx } from "../ctx"
import { PerfTimer } from "../perf" import { PerfTimer } from "../perf"
import { QuartzFilterPluginInstance } from "../plugins/types"
import { ProcessedContent } from "../plugins/vfile" import { ProcessedContent } from "../plugins/vfile"
export function filterContent( export function filterContent(ctx: BuildCtx, content: ProcessedContent[]): ProcessedContent[] {
{ cfg, argv }: BuildCtx, const { cfg, argv } = ctx
content: ProcessedContent[],
): ProcessedContent[] {
const perf = new PerfTimer() const perf = new PerfTimer()
const initialLength = content.length const initialLength = content.length
for (const plugin of cfg.plugins.filters) { for (const plugin of cfg.plugins.filters) {
const updatedContent = content.filter(plugin.shouldPublish) const updatedContent = content.filter((item) => plugin.shouldPublish(ctx, item))
if (argv.verbose) { if (argv.verbose) {
const diff = content.filter((x) => !updatedContent.includes(x)) const diff = content.filter((x) => !updatedContent.includes(x))

View file

@ -7,23 +7,24 @@ import { Root as HTMLRoot } from "hast"
import { ProcessedContent } from "../plugins/vfile" import { ProcessedContent } from "../plugins/vfile"
import { PerfTimer } from "../perf" import { PerfTimer } from "../perf"
import { read } from "to-vfile" import { read } from "to-vfile"
import { FilePath, QUARTZ, ServerSlug, slugifyFilePath } from "../path" import { FilePath, QUARTZ, slugifyFilePath } from "../path"
import path from "path" import path from "path"
import os from "os" import os from "os"
import workerpool, { Promise as WorkerPromise } from "workerpool" import workerpool, { Promise as WorkerPromise } from "workerpool"
import { QuartzTransformerPluginInstance } from "../plugins/types"
import { QuartzLogger } from "../log" import { QuartzLogger } from "../log"
import { trace } from "../trace" import { trace } from "../trace"
import { BuildCtx } from "../ctx" import { BuildCtx } from "../ctx"
export type QuartzProcessor = Processor<MDRoot, HTMLRoot, void> export type QuartzProcessor = Processor<MDRoot, HTMLRoot, void>
export function createProcessor(transformers: QuartzTransformerPluginInstance[]): QuartzProcessor { export function createProcessor(ctx: BuildCtx): QuartzProcessor {
const transformers = ctx.cfg.plugins.transformers
// base Markdown -> MD AST // base Markdown -> MD AST
let processor = unified().use(remarkParse) let processor = unified().use(remarkParse)
// MD AST -> MD AST transforms // MD AST -> MD AST transforms
for (const plugin of transformers.filter((p) => p.markdownPlugins)) { for (const plugin of transformers.filter((p) => p.markdownPlugins)) {
processor = processor.use(plugin.markdownPlugins!()) processor = processor.use(plugin.markdownPlugins!(ctx))
} }
// MD AST -> HTML AST // MD AST -> HTML AST
@ -31,7 +32,7 @@ export function createProcessor(transformers: QuartzTransformerPluginInstance[])
// HTML AST -> HTML AST transforms // HTML AST -> HTML AST transforms
for (const plugin of transformers.filter((p) => p.htmlPlugins)) { for (const plugin of transformers.filter((p) => p.htmlPlugins)) {
processor = processor.use(plugin.htmlPlugins!()) processor = processor.use(plugin.htmlPlugins!(ctx))
} }
return processor return processor
@ -73,7 +74,8 @@ async function transpileWorkerScript() {
}) })
} }
export function createFileParser({ argv, cfg }: BuildCtx, fps: FilePath[], allSlugs: ServerSlug[]) { export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
const { argv, cfg } = ctx
return async (processor: QuartzProcessor) => { return async (processor: QuartzProcessor) => {
const res: ProcessedContent[] = [] const res: ProcessedContent[] = []
for (const fp of fps) { for (const fp of fps) {
@ -85,12 +87,11 @@ export function createFileParser({ argv, cfg }: BuildCtx, fps: FilePath[], allSl
// Text -> Text transforms // Text -> Text transforms
for (const plugin of cfg.plugins.transformers.filter((p) => p.textTransform)) { for (const plugin of cfg.plugins.transformers.filter((p) => p.textTransform)) {
file.value = plugin.textTransform!(file.value) file.value = plugin.textTransform!(ctx, file.value)
} }
// base data properties that plugins may use // base data properties that plugins may use
file.data.slug = slugifyFilePath(path.relative(argv.directory, file.path) as FilePath) file.data.slug = slugifyFilePath(path.relative(argv.directory, file.path) as FilePath)
file.data.allSlugs = allSlugs
file.data.filePath = fp file.data.filePath = fp
const ast = processor.parse(file) const ast = processor.parse(file)
@ -111,24 +112,19 @@ export function createFileParser({ argv, cfg }: BuildCtx, fps: FilePath[], allSl
} }
export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<ProcessedContent[]> { export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<ProcessedContent[]> {
const { argv, cfg } = ctx const { argv } = ctx
const perf = new PerfTimer() const perf = new PerfTimer()
const log = new QuartzLogger(argv.verbose) const log = new QuartzLogger(argv.verbose)
const CHUNK_SIZE = 128 const CHUNK_SIZE = 128
let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism() let concurrency = fps.length < CHUNK_SIZE ? 1 : os.availableParallelism()
// get all slugs ahead of time as each thread needs a copy
const allSlugs = fps.map((fp) =>
slugifyFilePath(path.relative(argv.directory, path.resolve(fp)) as FilePath),
)
let res: ProcessedContent[] = [] let res: ProcessedContent[] = []
log.start(`Parsing input files using ${concurrency} threads`) log.start(`Parsing input files using ${concurrency} threads`)
if (concurrency === 1) { if (concurrency === 1) {
try { try {
const processor = createProcessor(cfg.plugins.transformers) const processor = createProcessor(ctx)
const parse = createFileParser(ctx, fps, allSlugs) const parse = createFileParser(ctx, fps)
res = await parse(processor) res = await parse(processor)
} catch (error) { } catch (error) {
log.end() log.end()
@ -144,7 +140,7 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
const childPromises: WorkerPromise<ProcessedContent[]>[] = [] const childPromises: WorkerPromise<ProcessedContent[]>[] = []
for (const chunk of chunks(fps, CHUNK_SIZE)) { for (const chunk of chunks(fps, CHUNK_SIZE)) {
childPromises.push(pool.exec("parseFiles", [argv, chunk, allSlugs])) childPromises.push(pool.exec("parseFiles", [argv, chunk, ctx.allSlugs]))
} }
const results: ProcessedContent[][] = await WorkerPromise.all(childPromises) const results: ProcessedContent[][] = await WorkerPromise.all(childPromises)

View file

@ -3,16 +3,14 @@ import { Argv, BuildCtx } from "./ctx"
import { FilePath, ServerSlug } from "./path" import { FilePath, ServerSlug } from "./path"
import { createFileParser, createProcessor } from "./processors/parse" import { createFileParser, createProcessor } from "./processors/parse"
const transformers = cfg.plugins.transformers
const processor = createProcessor(transformers)
// only called from worker thread // only called from worker thread
export async function parseFiles(argv: Argv, fps: FilePath[], allSlugs: ServerSlug[]) { export async function parseFiles(argv: Argv, fps: FilePath[], allSlugs: ServerSlug[]) {
const ctx: BuildCtx = { const ctx: BuildCtx = {
cfg, cfg,
argv, argv,
allSlugs,
} }
const processor = createProcessor(ctx)
const parse = createFileParser(ctx, fps, allSlugs) const parse = createFileParser(ctx, fps)
return parse(processor) return parse(processor)
} }