feat: dynamically fetch indices

This commit is contained in:
Jacky Zhao 2022-02-15 19:39:14 -05:00
parent 4587b13360
commit fcd5d2807d
10 changed files with 205 additions and 175 deletions

View file

@ -12,7 +12,7 @@ jobs:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- name: Build Link Index - name: Build Link Index
uses: jackyzha0/hugo-obsidian@v2.7 uses: jackyzha0/hugo-obsidian@v2.8
with: with:
index: true index: true
input: content input: content

4
.gitignore vendored
View file

@ -3,5 +3,5 @@ public
resources resources
.idea .idea
content/.obsidian content/.obsidian
data/linkIndex.yaml static/linkIndex.json
data/contentIndex.yaml static/contentIndex.json

View file

@ -4,4 +4,4 @@ help: ## Show all Makefile targets
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
serve: ## serve serve: ## serve
hugo-obsidian -input=content -output=data -index -root=. && hugo server hugo-obsidian -input=content -output=static -index -root=. && hugo server

View file

@ -1,5 +1,5 @@
--- ---
title: 🪴 Quartz 3 title: 🪴 Quartz 3.1
--- ---
Host your second brain and [digital garden](https://jzhao.xyz/posts/digital-gardening) for free. Quartz features Host your second brain and [digital garden](https://jzhao.xyz/posts/digital-gardening) for free. Quartz features
1. Extremely fast full-text search by pressing `/` 1. Extremely fast full-text search by pressing `/`

View file

@ -5,7 +5,7 @@ description:
Here is the page description. This is an example Quartz site that details installation, Here is the page description. This is an example Quartz site that details installation,
setup, customization, and troubleshooting for Quartz itself. setup, customization, and troubleshooting for Quartz itself.
page_title: page_title:
"🪴 Quartz 3" "🪴 Quartz 3.1"
links: links:
- link_name: Twitter - link_name: Twitter
link: https://twitter.com/_jzhao link: https://twitter.com/_jzhao

View file

@ -3,8 +3,9 @@
{{$url := urls.Parse .Site.BaseURL }} {{$url := urls.Parse .Site.BaseURL }}
{{$host := strings.TrimRight "/" $url.Path }} {{$host := strings.TrimRight "/" $url.Path }}
{{$curPage := strings.TrimPrefix $host (strings.TrimRight "/" .Page.RelPermalink) }} {{$curPage := strings.TrimPrefix $host (strings.TrimRight "/" .Page.RelPermalink) }}
{{$inbound := index $.Site.Data.linkIndex.index.backlinks $curPage}} {{$linkIndex := getJSON "/static/linkIndex.json"}}
{{$contentTable := $.Site.Data.contentIndex}} {{$inbound := index $linkIndex.index.backlinks $curPage}}
{{$contentTable := getJSON "/static/contentIndex.json"}}
{{if $inbound}} {{if $inbound}}
{{$cleanedInbound := apply (apply $inbound "index" "." "source") "replace" "." " " "-"}} {{$cleanedInbound := apply (apply $inbound "index" "." "source") "replace" "." " " "-"}}
{{- range $cleanedInbound | uniq -}} {{- range $cleanedInbound | uniq -}}

View file

@ -11,6 +11,8 @@
} }
</style> </style>
<script> <script>
async function run() {
const { index, links, content } = await fetchData()
const curPage = {{ strings.TrimRight "/" .Page.Permalink }}.replace({{strings.TrimRight "/" .Site.BaseURL }}, "") const curPage = {{ strings.TrimRight "/" .Page.Permalink }}.replace({{strings.TrimRight "/" .Site.BaseURL }}, "")
const pathColors = {{$.Site.Data.graphConfig.paths}} const pathColors = {{$.Site.Data.graphConfig.paths}}
let depth = {{$.Site.Data.graphConfig.depth}} let depth = {{$.Site.Data.graphConfig.depth}}
@ -225,12 +227,15 @@
.attr("x1", d => d.source.x) .attr("x1", d => d.source.x)
.attr("y1", d => d.source.y) .attr("y1", d => d.source.y)
.attr("x2", d => d.target.x) .attr("x2", d => d.target.x)
.attr("y2", d => d.target.y); .attr("y2", d => d.target.y)
node node
.attr("cx", d => d.x) .attr("cx", d => d.x)
.attr("cy", d => d.y); .attr("cy", d => d.y)
labels labels
.attr("x", d => d.x) .attr("x", d => d.x)
.attr("y", d => d.y); .attr("y", d => d.y)
}); });
}
run()
</script> </script>

View file

@ -8,7 +8,7 @@
<!-- CSS Stylesheets and Fonts --> <!-- CSS Stylesheets and Fonts -->
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700&family=Source+Sans+Pro:wght@400;600;700&family=Fira+Code:wght@400;700&display=swap" rel="stylesheet"> <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700&family=Source+Sans+Pro:wght@400;600;700&family=Fira+Code:wght@400;700&display=swap" rel="stylesheet">
{{ $css := slice "base.scss" "darkmode.scss" "syntax.scss" "custom.scss"}} {{$css := slice "base.scss" "darkmode.scss" "syntax.scss" "custom.scss"}}
{{range $css}} {{range $css}}
{{$sass := resources.Get . | resources.ToCSS }} {{$sass := resources.Get . | resources.ToCSS }}
{{with $sass | minify}} {{with $sass | minify}}
@ -26,9 +26,24 @@
<!-- Preload page vars --> <!-- Preload page vars -->
<script> <script>
const content = {{$.Site.Data.contentIndex}} const fetchData = async () => {
const index = {{$.Site.Data.linkIndex.index}} const promises = [
const links = {{$.Site.Data.linkIndex.links}} fetch("/linkIndex.json")
.then(data => data.json())
.then(data => ({
index: data.index,
links: data.links,
})),
fetch("/contentIndex.json")
.then(data => data.json()),
]
const [{index, links}, content] = await Promise.all(promises)
return ({
index,
links,
content,
})
}
</script> </script>
</head> </head>
{{ template "_internal/google_analytics.html" . }} {{ template "_internal/google_analytics.html" . }}

View file

@ -1,5 +1,7 @@
{{if $.Site.Data.config.enableLinkPreview}} {{if $.Site.Data.config.enableLinkPreview}}
<script> <script>
async function run() {
const {content} = await fetchData()
function htmlToElement(html) { function htmlToElement(html) {
const template = document.createElement('template') const template = document.createElement('template')
html = html.trim() html = html.trim()
@ -11,7 +13,6 @@
document.addEventListener("DOMContentLoaded", () => { document.addEventListener("DOMContentLoaded", () => {
[...document.getElementsByClassName("internal-link")] [...document.getElementsByClassName("internal-link")]
.forEach(li => { .forEach(li => {
console.log(li.dataset.src.replace(pathRegex, ''))
const linkDest = content[li.dataset.src.replace(pathRegex, '')] const linkDest = content[li.dataset.src.replace(pathRegex, '')]
if (linkDest) { if (linkDest) {
const popoverElement = `<div class="popover"> const popoverElement = `<div class="popover">
@ -29,5 +30,8 @@
} }
}) })
}) })
}
run()
</script> </script>
{{end}} {{end}}

View file

@ -67,189 +67,194 @@
}; };
</script> </script>
<script> <script>
const contentIndex = new FlexSearch.Document({ async function run() {
cache: true, const contentIndex = new FlexSearch.Document({
charset: "latin:extra", cache: true,
optimize: true, charset: "latin:extra",
worker: true, optimize: true,
document: { worker: true,
index: [{ document: {
field: "content", index: [{
tokenize: "strict", field: "content",
context: { tokenize: "strict",
resolution: 5, context: {
depth: 3, resolution: 5,
bidirectional: true depth: 3,
}, bidirectional: true
suggest: true, },
}, { suggest: true,
field: "title", }, {
tokenize: "forward", field: "title",
}] tokenize: "forward",
} }]
}
})
const { content } = await fetchData()
for (const [key, value] of Object.entries(content)) {
contentIndex.add({
id: key,
title: value.title,
content: removeMarkdown(value.content),
}) })
}
for (const [key, value] of Object.entries(content)) { const highlight = (content, term) => {
contentIndex.add({ const highlightWindow = 20
id: key, const tokenizedTerm = term.split(/\s+/).filter(t => t !== "")
title: value.title, const splitText = content.split(/\s+/).filter(t => t !== "")
content: removeMarkdown(value.content), const includesCheck = (token) => tokenizedTerm.some(term => token.toLowerCase().startsWith(term.toLowerCase()))
})
const occurrencesIndices = splitText
.map(includesCheck)
// calculate best index
let bestSum = 0
let bestIndex = 0
for (let i = 0; i < Math.max(occurrencesIndices.length - highlightWindow, 0); i++) {
const window = occurrencesIndices.slice(i, i + highlightWindow)
const windowSum = window.reduce((total, cur) => total + cur, 0)
if (windowSum >= bestSum) {
bestSum = windowSum
bestIndex = i
}
} }
const highlight = (content, term) => { const startIndex = Math.max(bestIndex - highlightWindow, 0)
const highlightWindow = 20 const endIndex = Math.min(startIndex + 2 * highlightWindow, splitText.length)
const tokenizedTerm = term.split(/\s+/).filter(t => t !== "") const mappedText = splitText
const splitText = content.split(/\s+/).filter(t => t !== "") .slice(startIndex, endIndex)
const includesCheck = (token) => tokenizedTerm.some(term => token.toLowerCase().startsWith(term.toLowerCase())) .map(token => {
if (includesCheck(token)) {
const occurrencesIndices = splitText return `<span class="search-highlight">${token}</span>`
.map(includesCheck)
// calculate best index
let bestSum = 0
let bestIndex = 0
for (let i = 0; i < Math.max(occurrencesIndices.length - highlightWindow, 0); i++) {
const window = occurrencesIndices.slice(i, i + highlightWindow)
const windowSum = window.reduce((total, cur) => total + cur, 0)
if (windowSum >= bestSum) {
bestSum = windowSum
bestIndex = i
}
} }
return token
})
.join(" ")
.replaceAll('</span> <span class="search-highlight">', " ")
return `${startIndex === 0 ? "" : "..."}${mappedText}${endIndex === splitText.length ? "" : "..."}`
}
const startIndex = Math.max(bestIndex - highlightWindow, 0) const resultToHTML = ({url, title, content, term}) => {
const endIndex = Math.min(startIndex + 2 * highlightWindow, splitText.length) const text = removeMarkdown(content)
const mappedText = splitText const resultTitle = highlight(title, term)
.slice(startIndex, endIndex) const resultText = highlight(text, term)
.map(token => { return `<button class="result-card" id="${url}">
if (includesCheck(token)) {
return `<span class="search-highlight">${token}</span>`
}
return token
})
.join(" ")
.replaceAll('</span> <span class="search-highlight">', " ")
return `${startIndex === 0 ? "" : "..."}${mappedText}${endIndex === splitText.length ? "" : "..."}`
}
const resultToHTML = ({url, title, content, term}) => {
const text = removeMarkdown(content)
const resultTitle = highlight(title, term)
const resultText = highlight(text, term)
return `<button class="result-card" id="${url}">
<h3>${resultTitle}</h3> <h3>${resultTitle}</h3>
<p>${resultText}</p> <p>${resultText}</p>
</button>` </button>`
}
const redir = (id, term) => {
window.location.href = "{{.Site.BaseURL}}" + `${id}#:~:text=${encodeURIComponent(term)}`
}
const formatForDisplay = id => ({
id,
url: id,
title: content[id].title,
content: content[id].content
})
const source = document.getElementById('search-bar')
const results = document.getElementById("results-container")
let term
source.addEventListener("keyup", (e) => {
if (e.key === "Enter") {
const anchor = document.getElementsByClassName("result-card")[0]
redir(anchor.id, term)
} }
})
const redir = (id, term) => { source.addEventListener('input', (e) => {
window.location.href = "{{.Site.BaseURL}}" + `${id}#:~:text=${encodeURIComponent(term)}` term = e.target.value
} contentIndex.search(term, [
{
const fetch = id => ({ field: "content",
id, limit: 10,
url: id, suggest: true,
title: content[id].title, },
content: content[id].content {
}) field: "title",
limit: 5,
const source = document.getElementById('search-bar') }
const results = document.getElementById("results-container") ]).then(searchResults => {
let term const getByField = field => {
source.addEventListener("keyup", (e) => { const results = searchResults.filter(x => x.field === field)
if (e.key === "Enter") { if (results.length === 0) {
const anchor = document.getElementsByClassName("result-card")[0] return []
redir(anchor.id, term) } else {
return [...results[0].result]
} }
}) }
source.addEventListener('input', (e) => { const allIds = new Set([...getByField('title'), ...getByField('content')])
term = e.target.value const finalResults = [...allIds].map(formatForDisplay)
contentIndex.search(term, [
{
field: "content",
limit: 10,
suggest: true,
},
{
field: "title",
limit: 5,
}
]).then(searchResults => {
const getByField = field => {
const results = searchResults.filter(x => x.field === field)
if (results.length === 0) {
return []
} else {
return [...results[0].result]
}
}
const allIds = new Set([...getByField('title'), ...getByField('content')])
const finalResults = [...allIds].map(fetch)
// display // display
if (finalResults.length === 0) { if (finalResults.length === 0) {
results.innerHTML = `<button class="result-card"> results.innerHTML = `<button class="result-card">
<h3>No results.</h3> <h3>No results.</h3>
<p>Try another search term?</p> <p>Try another search term?</p>
</button>` </button>`
} else { } else {
results.innerHTML = finalResults results.innerHTML = finalResults
.map(result => resultToHTML({ .map(result => resultToHTML({
...result, ...result,
term, term,
})) }))
.join("\n") .join("\n")
const anchors = document.getElementsByClassName("result-card"); const anchors = document.getElementsByClassName("result-card");
[...anchors].forEach(anchor => { [...anchors].forEach(anchor => {
anchor.onclick = () => redir(anchor.id, term) anchor.onclick = () => redir(anchor.id, term)
})
}
}) })
}
}) })
})
const searchContainer = document.getElementById("search-container") const searchContainer = document.getElementById("search-container")
function openSearch() {
if (searchContainer.style.display === "none" || searchContainer.style.display === "") { function openSearch() {
source.value = "" if (searchContainer.style.display === "none" || searchContainer.style.display === "") {
results.innerHTML = "" source.value = ""
searchContainer.style.display = "block" results.innerHTML = ""
source.focus() searchContainer.style.display = "block"
} else { source.focus()
searchContainer.style.display = "none" } else {
} searchContainer.style.display = "none"
} }
}
function closeSearch() { function closeSearch() {
searchContainer.style.display = "none" searchContainer.style.display = "none"
}
document.addEventListener('keydown', (event) => {
if (event.key === "/") {
event.preventDefault()
openSearch()
} }
if (event.key === "Escape") {
event.preventDefault()
closeSearch()
}
})
document.addEventListener('keydown', (event) => { window.addEventListener('DOMContentLoaded', () => {
if (event.key === "/") { const searchButton = document.getElementById("search-icon")
event.preventDefault() searchButton.addEventListener('click', (evt) => {
openSearch() openSearch()
}
if (event.key === "Escape") {
event.preventDefault()
closeSearch()
}
}) })
searchButton.addEventListener('keydown', (evt) => {
window.addEventListener('DOMContentLoaded', () => { openSearch()
const searchButton = document.getElementById("search-icon")
searchButton.addEventListener('click', (evt) => {
openSearch()
})
searchButton.addEventListener('keydown', (evt) => {
openSearch()
})
searchContainer.addEventListener('click', (evt) => {
closeSearch()
})
document.getElementById("search-space").addEventListener('click', (evt) => {
evt.stopPropagation()
})
}) })
searchContainer.addEventListener('click', (evt) => {
closeSearch()
})
document.getElementById("search-space").addEventListener('click', (evt) => {
evt.stopPropagation()
})
})
}
run()
</script> </script>