From cea6834fef54da59fc1692d1db0221b93793238f Mon Sep 17 00:00:00 2001 From: Jacky Zhao Date: Wed, 9 Aug 2023 00:26:33 -0700 Subject: [PATCH] profiling, better concurrency heuristics --- .gitignore | 1 + content/advanced/making plugins.md | 2 +- package.json | 3 ++- quartz/processors/parse.ts | 9 +++++---- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 5177d6632..fd96fec90 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .gitignore node_modules public +prof tsconfig.tsbuildinfo .obsidian .quartz-cache diff --git a/content/advanced/making plugins.md b/content/advanced/making plugins.md index a73069e6d..d2db67e41 100644 --- a/content/advanced/making plugins.md +++ b/content/advanced/making plugins.md @@ -2,7 +2,7 @@ title: Making your own plugins --- -This part of the documentation will assume you have some basic coding knowledge and will include code snippets that describe the interface of what Quartz plugins should look like. +This part of the documentation will assume you have some basic coding knowledge in TypeScript and will include code snippets that describe the interface of what Quartz plugins should look like. ![[quartz-transform-pipeline.png]] diff --git a/package.json b/package.json index 0d5e7a03c..29b46f206 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,8 @@ "scripts": { "check": "tsc --noEmit && npx prettier . --check", "format": "npx prettier . --write", - "test": "tsx ./quartz/path.test.ts" + "test": "tsx ./quartz/path.test.ts", + "profile": "0x -D prof ./quartz/bootstrap-cli.mjs build --concurrency=1" }, "keywords": [ "site generator", diff --git a/quartz/processors/parse.ts b/quartz/processors/parse.ts index 69df69b3d..299d59afe 100644 --- a/quartz/processors/parse.ts +++ b/quartz/processors/parse.ts @@ -9,7 +9,6 @@ import { PerfTimer } from "../perf" import { read } from "to-vfile" import { FilePath, QUARTZ, slugifyFilePath } from "../path" import path from "path" -import os from "os" import workerpool, { Promise as WorkerPromise } from "workerpool" import { QuartzLogger } from "../log" import { trace } from "../trace" @@ -82,6 +81,7 @@ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) { const res: ProcessedContent[] = [] for (const fp of fps) { try { + const perf = new PerfTimer() const file = await read(fp) // strip leading and trailing whitespace @@ -101,7 +101,7 @@ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) { res.push([newAst, file]) if (argv.verbose) { - console.log(`[process] ${fp} -> ${file.data.slug}`) + console.log(`[process] ${fp} -> ${file.data.slug} (${perf.timeSince()})`) } } catch (err) { trace(`\nFailed to process \`${fp}\``, err as Error) @@ -112,14 +112,15 @@ export function createFileParser(ctx: BuildCtx, fps: FilePath[]) { } } +const clamp = (num: number, min: number, max: number) => Math.min(Math.max(Math.round(num), min), max); export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise { const { argv } = ctx const perf = new PerfTimer() const log = new QuartzLogger(argv.verbose) + // rough heuristics: 128 gives enough time for v8 to JIT and optimize parsing code paths const CHUNK_SIZE = 128 - let concurrency = - ctx.argv.concurrency ?? (fps.length < CHUNK_SIZE ? 1 : os.availableParallelism()) + const concurrency = ctx.argv.concurrency ?? clamp(fps.length / CHUNK_SIZE, 1, 4) let res: ProcessedContent[] = [] log.start(`Parsing input files using ${concurrency} threads`)