feat(search): add pagefind provider support

Add Pagefind indexing and browser search adapters behind a provider switch.

This lets prebuild generate either Stork or Pagefind search artifacts and lets the existing search UI run against Pagefind while preserving scoped filters, excerpts, and result metadata.

.gitignore

@@ -92,6 +92,7 @@ dist
 /static/**/*.json
 /static/**/*.toml
 /static/**/*.st
+/static/pagefind/
 /commits-data.json
 /static/tailwind.css
 

build-lists.js

@@ -41,6 +41,12 @@ import { makeSearchableList } from '~/helpers/searchable-list.js'
 import {
     writeStorkToml
 } from '~/helpers/stork/toml.js'
+import {
+    writePagefindIndex
+} from '~/helpers/pagefind/index.js'
+import {
+    getSearchProvider
+} from '~/helpers/search/config.js'
 import {
     KindListMemoized as KindList
 } from '~/helpers/api/kind.js'
@@ -735,9 +741,15 @@ class BuildLists {
         console.log('Building XML Sitemap')
         await saveSitemap( sitemapEndpoints.map( ({ route }) => route ) )
 
-        // Save stork toml index
+        const searchProvider = getSearchProvider( process.env.PUBLIC_SEARCH_PROVIDER )
+
+        if ( searchProvider === 'stork' ) {
             console.log('Building Stork toml index')
             await writeStorkToml( sitemapEndpoints )
+        } else {
+            console.log('Building Pagefind index')
+            await writePagefindIndex( sitemapEndpoints )
+        }
 
         console.log('Total Nuxt Endpoints', this.endpointMaps.nuxt.size )
         console.log('Total Eleventy Endpoints', this.endpointMaps.eleventy.size )

components/search-stork.vue

@@ -39,7 +39,7 @@
                         hasActiveFilter( button.query ) ? 'border-opacity-50 bg-darkest' : 'border-opacity-0 neumorphic-shadow-inner'
                     ]"
                     :aria-label="`Filter list by ${button.label}`"
-                    @click="toggleFilter(button.query); queryResults(query)"
+                    @click="toggleFilter(button.query); queryResults()"
                 >{{ button.label }}</button>
             </div>
         </div>
@@ -151,22 +151,16 @@
                         </div>
 
                         <div
-                            v-if="listing.storkResult"
+                            v-if="listing.resultExcerptsMarkup?.length"
                             class="text-xs leading-5 font-light"
                         >
                             <div
-                                v-for="(excerpt, excerptIndex) in listing.storkResult.excerpts"
+                                v-for="(excerptMarkup, excerptIndex) in listing.resultExcerptsMarkup"
                                 :key="`excerpt-${ excerptIndex }`"
                                 class="result-excerpt space-y-3"
-                            >
+                                v-html="excerptMarkup"
-                                <div
-                                    v-for="(range, rangeIndex) in makeHighlightedMarkup( excerpt )"
-                                    :key="`range-${ rangeIndex }`"
-
-                                    v-html="range"
                             />
                         </div>
-                        </div>
                         <!-- listing.lastUpdated: {{ listing.lastUpdated }} -->
                         <template v-if="listing.lastUpdated">
                             <small
@@ -291,9 +285,18 @@ import {
 import {
     getIconForListing
 } from '~/helpers/app-derived.js'
+import {
+    PagefindClient,
+    mapPagefindDataToListing
+} from '~/helpers/pagefind/browser.js'
+import {
+    getSearchProvider
+} from '~/helpers/search/config.js'
+import {
+    SearchFilters
+} from '~/helpers/search/filters.js'
 import {
     StorkClient,
-    StorkFilters,
     makeHighlightedMarkup,
     makeHighlightedResultTitle
 } from '~/helpers/stork/browser.js'
@@ -304,7 +307,7 @@ import RelativeTime from '~/components/relative-time.vue'
 import ListSummary from '~/components/list-summary.vue'
 import ListEndButtons from '~/components/list-end-buttons.vue'
 
-let storkClient = null
+const searchProvider = getSearchProvider( import.meta.env.PUBLIC_SEARCH_PROVIDER )
 
 export default {
     components: {
@@ -347,15 +350,27 @@ export default {
             hasStartedAnyQuery: false,
             listingsResults: [],
             waitingForQuery: false,
-            isSSR: import.meta.env.SSR
+            isSSR: import.meta.env.SSR,
+            searchClient: null,
+            searchFilters: null,
+            lastQueryId: 0
         }
     },
     computed: {
-        storkQuery () {
+        activeSearchProvider () {
+            return searchProvider
+        },
+        activeQuery () {
             return [
                 this.userTextQuery.trim(),
                 ...this.filterQueryList
-            ].join(' ')
+            ].filter( Boolean ).join(' ')
+        },
+        pagefindFilters () {
+            const filters = new SearchFilters()
+            filters.setFromStringArray( this.filterQueryList )
+
+            return filters.asPagefindFilters
         },
         appList () {
             return this.kindPage.items
@@ -390,7 +405,7 @@ export default {
             return this.baseFilters.length > 0
         },
         hasSearchInputText () {
-            return this.userTextQuery.length > 0
+            return this.userTextQuery.trim().length > 0
         },
         hasAnyUserFilters () {
             return this.userFilters.length > 0
@@ -402,7 +417,7 @@ export default {
             return !this.hasAnyUserTerms
         },
         inputTerms () {
-            return this.userTextQuery.trim().split(' ')
+            return this.userTextQuery.trim().split(' ').filter( Boolean )
         },
         userFilters () {
             // console.log('filterQueryList', )
@@ -442,22 +457,23 @@ export default {
         }
     },
     mounted () {
-        // Setup stork client
+        this.searchClient = this.makeSearchClient()
-        storkClient = new StorkClient()
 
-        // Store filter instance
+        this.searchFilters = new SearchFilters()
-        this.storkFilters = new StorkFilters()
-
-        // Add initial filters
-        this.storkFilters.setFromStringArray( this.baseFilters )
 
+        this.searchFilters.setFromStringArray( this.baseFilters )
+        this.filterQueryList = this.searchFilters.list
     },
     methods: {
-        makeHighlightedMarkup,
-        makeHighlightedResultTitle,
-
         getIconForListing,
 
+        makeSearchClient () {
+            if ( this.activeSearchProvider === 'stork' ) {
+                return new StorkClient()
+            }
+
+            return new PagefindClient()
+        },
         getSearchLinks (app) {
             return app?.searchLinks || []
         },
@@ -466,10 +482,8 @@ export default {
             return this.filterQueryList.includes( filter )
         },
         toggleFilter ( newFilterQuery ) {
-
+            this.searchFilters.toggleFilter( newFilterQuery )
-            this.storkFilters.toggleFilter( newFilterQuery )
+            this.filterQueryList = this.searchFilters.list
-
-            this.filterQueryList = this.storkFilters.list
         },
         scrollInputToTop () {
             scrollIntoView(this.$refs['search-container'], {
@@ -477,15 +491,63 @@ export default {
                 behavior: 'smooth'
             })
         },
+        mapStorkResultToListing ( result ) {
+            return {
+                name: makeHighlightedResultTitle( result ),
+                text: '',
+                endpoint: result.entry.url,
+                slug: result.entry.url,
+                category: {
+                    slug: 'uncategorized'
+                },
+                lastUpdated: null,
+                resultExcerptsMarkup: ( result.excerpts || [] ).flatMap( excerpt => {
+                    return makeHighlightedMarkup( excerpt )
+                } )
+            }
+        },
+        async runPagefindQuery () {
+            const pagefindQuery = await this.searchClient.lazyQuery( this.userTextQuery, {
+                filters: this.pagefindFilters,
+                sort: this.hasSearchInputText ? {} : {
+                    updated: 'desc'
+                }
+            } )
+
+            if ( pagefindQuery === null ) {
+                return null
+            }
+
+            const resultData = await Promise.all( ( pagefindQuery.results || [] ).map( async result => {
+                return await result.data()
+            } ) )
+
+            return resultData.map( data => {
+                return mapPagefindDataToListing( data, {
+                    highlightTerms: this.inputTerms
+                } )
+            } )
+        },
+        async runStorkQuery () {
+            const storkQuery = await this.searchClient.lazyQuery( this.activeQuery, this.activeQuery.split(' ') )
+
+            if ( storkQuery === null ) {
+                return null
+            }
+
+            return ( storkQuery.results || [] ).map( result => {
+                return this.mapStorkResultToListing( result )
+            } )
+        },
 
         // Called on input and when a filter is toggled
-        async queryResults ( rawQuery ) {
+        async queryResults ( rawQuery = this.userTextQuery ) {
+            const queryId = ++this.lastQueryId
 
-            console.log( 'query', this.storkQuery )
+            if ( this.activeQuery.trim().length === 0 ) {
-
+                this.waitingForQuery = false
-            // If our query is empty
+                return
-            // then bail
+            }
-            if ( this.storkQuery.trim().length === 0 ) return
 
             this.waitingForQuery = true
 
@@ -494,36 +556,22 @@ export default {
             // Declare that at least one query has been made
             this.hasStartedAnyQuery = true
 
-            // console.log('rawQuery', rawQuery)
+            const results = this.activeSearchProvider === 'stork'
+                ? await this.runStorkQuery()
+                : await this.runPagefindQuery()
 
-            const requiredTerms = this.storkQuery.split(' ')
+            if ( queryId !== this.lastQueryId ) {
-
-            const storkQuery = await storkClient.lazyQuery( this.storkQuery, requiredTerms )
-
-            // If the query response is empty
-            // then return
-            if ( storkQuery === null ) {
                 return
             }
 
-            // console.log( 'storkQuery', storkQuery )
+            if ( results === null ) {
-
-            this.listingsResults = storkQuery.results.map( result => {
-                return {
-                    name: makeHighlightedResultTitle( result ),
-                    endpoint: result.entry.url,
-                    slug: '',
-                    category: {
-                        slug: 'uncategorized'
-                    },
-                    storkResult: result
-                }
-            })
-
-            // Switch from loading state and reveal the results
                 this.waitingForQuery = false
+                return
+            }
 
-            // console.log('this.listingsResults', this.listingsResults)
+            this.listingsResults = results
+
+            this.waitingForQuery = false
         },
 
         handleSearchInput ( event ) {

helpers/pagefind/browser.js

@@ -0,0 +1,168 @@
+import {
+    isNonEmptyString
+} from '~/helpers/check-types.js'
+import {
+    pagefindBundleRelativeURL,
+    pagefindScriptURL
+} from '~/helpers/pagefind/config.js'
+
+function escapeHtml ( text = '' ) {
+    return text
+        .replaceAll('&', '&')
+        .replaceAll('<', '&lt;')
+        .replaceAll('>', '&gt;')
+        .replaceAll('"', '&quot;')
+        .replaceAll("'", '&#39;')
+}
+
+function escapeRegExp ( value ) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+export function makeHighlightedPagefindTitle ( title, terms = [] ) {
+    const highlightedTerms = terms
+        .filter( term => isNonEmptyString( term ) )
+        .map( term => term.trim() )
+        .filter( term => term.length > 0 )
+        .sort( ( a, b ) => b.length - a.length )
+
+    const titleMarkup = escapeHtml( title || '' )
+
+    if ( highlightedTerms.length === 0 ) {
+        return titleMarkup
+    }
+
+    const pattern = highlightedTerms.map( escapeRegExp ).join('|')
+
+    return titleMarkup.replace(
+        new RegExp(`(${ pattern })`, 'gi'),
+        '<span class="stork-highlighted-text font-bold text-white bg-green-800 rounded px-1">$1</span>'
+    )
+}
+
+export function normalizePagefindExcerptMarkup ( excerptMarkup = '' ) {
+    if ( !isNonEmptyString( excerptMarkup ) ) return ''
+
+    return excerptMarkup
+        .replace(/<mark[^>]*>/g, '<span class="stork-highlighted-text font-bold text-white bg-green-800 rounded px-1">')
+        .replace(/<\/mark>/g, '</span>')
+}
+
+export function mapPagefindDataToListing ( resultData, {
+    highlightTerms = []
+} = {} ) {
+    const lastUpdatedTimestamp = Number( resultData.meta?.lastUpdatedTimestamp || 0 )
+
+    return {
+        name: makeHighlightedPagefindTitle( resultData.meta?.title || resultData.url, highlightTerms ),
+        text: resultData.meta?.text || resultData.url,
+        endpoint: resultData.url,
+        slug: resultData.meta?.slug || resultData.url,
+        category: {
+            slug: resultData.meta?.categorySlug || 'uncategorized'
+        },
+        lastUpdated: lastUpdatedTimestamp > 0 ? {
+            timestamp: lastUpdatedTimestamp
+        } : null,
+        resultExcerptsMarkup: isNonEmptyString( resultData.excerpt ) ? [
+            normalizePagefindExcerptMarkup( resultData.excerpt )
+        ] : []
+    }
+}
+
+export class PagefindClient {
+    constructor ( options = {} ) {
+        this.bundlePath = options.bundlePath || pagefindBundleRelativeURL
+        this.pagefind = options.pagefind || null
+        this.debounceMs = options.debounceMs || 100
+        this.cancelCurrentQuery = null
+    }
+
+    setupState = 'not-setup'
+
+    get isSetup () {
+        return this.setupState === 'complete'
+    }
+
+    waitForSetup () {
+        return new Promise( resolve => {
+            if ( this.isSetup ) resolve()
+
+            const timer = setInterval( () => {
+                if ( this.isSetup ) {
+                    clearInterval( timer )
+                    resolve()
+                }
+            }, 50 )
+        })
+    }
+
+    async loadPagefindScript () {
+        if ( this.pagefind ) return
+
+        const pagefindModule = await import(/* @vite-ignore */ pagefindScriptURL)
+        this.pagefind = pagefindModule.default || pagefindModule
+
+        this.pagefind.options({
+            bundlePath: this.bundlePath
+        })
+    }
+
+    async setup () {
+        if ( this.setupState !== 'not-setup' ) {
+            await this.waitForSetup()
+            return
+        }
+
+        this.setupState = 'pending'
+
+        await this.loadPagefindScript()
+
+        if ( typeof this.pagefind.init === 'function' ) {
+            await this.pagefind.init()
+        }
+
+        this.setupState = 'complete'
+    }
+
+    async lazyQuery ( query, {
+        filters = {},
+        sort = {}
+    } = {} ) {
+        const searchOptions = {}
+
+        if ( Object.keys( filters ).length > 0 ) {
+            searchOptions.filters = filters
+        }
+
+        if ( Object.keys( sort ).length > 0 ) {
+            searchOptions.sort = sort
+        }
+
+        const trimmedQuery = query.trim()
+
+        const result = await new Promise( async ( resolve, reject ) => {
+            if ( this.cancelCurrentQuery !== null ) {
+                this.cancelCurrentQuery()
+            }
+
+            this.cancelCurrentQuery = () => { reject({ message: `Cancelled previous query for ${ trimmedQuery }`, canceled: true }) }
+
+            if ( !this.isSetup ) await this.setup()
+
+            if ( trimmedQuery.length === 0 ) {
+                resolve( await this.pagefind.search( null, searchOptions ) )
+                return
+            }
+
+            resolve( await this.pagefind.debouncedSearch( trimmedQuery, searchOptions, this.debounceMs ) )
+        }).catch( err => {
+            console.log('Query rejected', err)
+            return null
+        })
+
+        this.cancelCurrentQuery = null
+
+        return result
+    }
+}

helpers/pagefind/config.js

@@ -0,0 +1,5 @@
+export const pagefindLanguage = 'en'
+export const pagefindOutputPath = './static/pagefind'
+export const pagefindBundleRelativeURL = '/pagefind/'
+export const pagefindScriptURL = `${ pagefindBundleRelativeURL }pagefind.js`
+export const sitemapEndpointsPath = './static/sitemap-endpoints.json'

helpers/pagefind/index.js

@@ -0,0 +1,181 @@
+import fs from 'fs-extra'
+import * as pagefind from 'pagefind'
+
+import {
+    isNonEmptyArray,
+    isNonEmptyString
+} from '~/helpers/check-types.js'
+import {
+    getRouteType
+} from '~/helpers/app-derived.js'
+import {
+    getAppCategory
+} from '~/helpers/categories.js'
+import {
+    pagefindLanguage,
+    pagefindOutputPath
+} from '~/helpers/pagefind/config.js'
+
+function getSearchListing ( sitemapEntry ) {
+    return sitemapEntry.payload.app || sitemapEntry.payload.listing || sitemapEntry.payload.video || null
+}
+
+function pushContentPart ( parts, value ) {
+    if ( !isNonEmptyString( value ) ) return
+
+    parts.push( value.trim() )
+}
+
+function pushListContentPart ( parts, values ) {
+    if ( !isNonEmptyArray( values ) ) return
+
+    pushContentPart( parts, values.join(', ') )
+}
+
+function normalizeFilterValue ( value ) {
+    if ( !isNonEmptyString( value ) ) return null
+
+    return value.replaceAll('-', '_').trim()
+}
+
+export function shouldIndexSitemapEntry ( sitemapEntry ) {
+    return getSearchListing( sitemapEntry ) !== null
+}
+
+export function makePagefindTitle ( sitemapEntry ) {
+    const listing = getSearchListing( sitemapEntry )
+    const routeType = getRouteType( sitemapEntry.route )
+
+    let title = listing?.name || sitemapEntry.route
+
+    if ( routeType === 'benchmarks' ) {
+        title = `${ title } Benchmarks`
+    }
+
+    return title
+}
+
+export function makePagefindContent ( sitemapEntry ) {
+    const listing = getSearchListing( sitemapEntry )
+    const routeType = getRouteType( sitemapEntry.route )
+    const parts = []
+
+    pushContentPart( parts, makePagefindTitle( sitemapEntry ) )
+    pushContentPart( parts, listing?.text )
+    pushContentPart( parts, listing?.content )
+    pushContentPart( parts, listing?.description )
+    pushListContentPart( parts, listing?.aliases )
+    pushListContentPart( parts, listing?.tags )
+    pushListContentPart( parts, listing?.timestamps?.map( timestamp => timestamp.fullText ) )
+    pushListContentPart( parts, listing?.appLinks?.map( appLink => appLink.name ) )
+    pushContentPart( parts, listing?.category?.label )
+    pushContentPart( parts, listing?.status )
+
+    if ( routeType === 'benchmarks' ) {
+        pushContentPart( parts, 'Benchmarks')
+        pushContentPart( parts, 'Apple Silicon App Tested')
+    }
+
+    return parts.join('\n\n')
+}
+
+export function makePagefindFilters ( sitemapEntry ) {
+    const listing = getSearchListing( sitemapEntry )
+    const routeType = getRouteType( sitemapEntry.route )
+    const filters = {
+        type: [ normalizeFilterValue( routeType ) ]
+    }
+
+    const status = normalizeFilterValue( listing?.status )
+
+    if ( status !== null ) {
+        filters.status = [ status ]
+    }
+
+    if ( listing?.category?.slug ) {
+        filters.category = [ getAppCategory( listing ).snakeSlug ]
+    }
+
+    return filters
+}
+
+export function mapSitemapEntryToPagefindRecord ( sitemapEntry ) {
+    if ( !shouldIndexSitemapEntry( sitemapEntry ) ) return null
+
+    const listing = getSearchListing( sitemapEntry )
+    const routeType = getRouteType( sitemapEntry.route )
+    const lastUpdatedTimestamp = String( listing?.lastUpdated?.timestamp || 0 )
+
+    return {
+        url: sitemapEntry.route,
+        content: makePagefindContent( sitemapEntry ),
+        language: pagefindLanguage,
+        meta: {
+            title: makePagefindTitle( sitemapEntry ),
+            text: listing?.text || '',
+            slug: listing?.slug || sitemapEntry.route,
+            categorySlug: listing?.category?.slug || 'uncategorized',
+            routeType,
+            lastUpdatedTimestamp
+        },
+        filters: makePagefindFilters( sitemapEntry ),
+        sort: {
+            updated: lastUpdatedTimestamp
+        }
+    }
+}
+
+export async function writePagefindIndex ( sitemapEndpoints, {
+    outputPath = pagefindOutputPath
+} = {} ) {
+    await fs.remove( outputPath )
+
+    const {
+        errors,
+        index
+    } = await pagefind.createIndex({
+        forceLanguage: pagefindLanguage
+    })
+
+    if ( errors.length > 0 ) {
+        throw new Error(`Pagefind createIndex errors: ${ errors.join(', ') }`)
+    }
+
+    if ( !index ) {
+        throw new Error('Pagefind index was not created')
+    }
+
+    let recordCount = 0
+
+    try {
+        for ( const sitemapEntry of sitemapEndpoints ) {
+            const record = mapSitemapEntryToPagefindRecord( sitemapEntry )
+
+            if ( record === null ) continue
+
+            const response = await index.addCustomRecord( record )
+
+            if ( response.errors.length > 0 ) {
+                throw new Error(`Pagefind addCustomRecord errors for ${ sitemapEntry.route }: ${ response.errors.join(', ') }`)
+            }
+
+            recordCount += 1
+        }
+
+        const writeResponse = await index.writeFiles({
+            outputPath
+        })
+
+        if ( writeResponse.errors.length > 0 ) {
+            throw new Error(`Pagefind writeFiles errors: ${ writeResponse.errors.join(', ') }`)
+        }
+
+        return {
+            outputPath,
+            recordCount
+        }
+    } finally {
+        await index.deleteIndex().catch( () => null )
+        await pagefind.close().catch( () => null )
+    }
+}

helpers/search/config.js

@@ -0,0 +1,18 @@
+export const defaultSearchProvider = 'pagefind'
+
+export const supportedSearchProviders = new Set([
+    'pagefind',
+    'stork'
+])
+
+export function getSearchProvider ( rawProvider = defaultSearchProvider ) {
+    const provider = ( rawProvider || defaultSearchProvider ).toLowerCase()
+
+    if ( supportedSearchProviders.has( provider ) ) {
+        return provider
+    }
+
+    console.warn(`Unknown search provider "${ provider }", falling back to "${ defaultSearchProvider }"`)
+
+    return defaultSearchProvider
+}

package.json

@@ -40,6 +40,8 @@
     "setup-stork": "pnpm run \"/^download-stork:.*/\" --parallel",
     "build-stork-index": "./$npm_package_config_stork_executable build --input $npm_package_config_stork_toml --output $npm_package_config_stork_index",
     "build-stork-index-js": "pnpm exec vite-node scripts/build-stork-index.js",
+    "build-pagefind-index": "pnpm exec vite-node scripts/build-pagefind-index.js",
+    "build-search-index": "pnpm exec vite-node scripts/build-search-index.js",
     "stork-search": "./$npm_package_config_stork_executable search --index $npm_package_config_stork_index --query $1",
     "stork-index": "pnpm setup-stork && pnpm build-stork-index",
     "stork-netlify": "chmod +x scripts/stork-netlify.sh && ./scripts/stork-netlify.sh",
@@ -57,7 +59,7 @@
     "vercel-build": "pnpm exec vite-node scripts/vercel-build.js",
     "netlify-prebuild:download-sitemaps": "pnpm exec vite-node scripts/download-sitemaps.js",
     "netlify-prebuild:test-prebuild-functions": "pnpm test-prebuild && pnpm test-api-client && pnpm test-listings",
-    "netlify-prebuild": "pnpm run \"/^netlify-prebuild:.*/\" && pnpm stork-index",
+    "netlify-prebuild": "pnpm run \"/^netlify-prebuild:.*/\" && pnpm build-search-index",
     "netlify-postbuild:test-postbuild-functions": "vitest test/main.test.ts",
     "netlify-postbuild:test-circular-deps": "madge --circular --extensions js,mjs,ts,vue,astro ./*",
     "netlify-build": "pnpm run netlify-prebuild && pnpm generate-astro && pnpm run \"/^netlify-postbuild:.*/\"",
@@ -107,6 +109,7 @@
     "node-html-parser": "^2.0.0",
     "observe-element-in-viewport": "0.0.15",
     "ofetch": "^1.0.0",
+    "pagefind": "1.4.0",
     "plist": "^3.0.1",
     "pretty-bytes": "^5.5.0",
     "rollup-plugin-node-polyfills": "^0.2.1",

pnpm-lock.yaml

@@ -137,6 +137,9 @@ importers:
       ofetch:
         specifier: ^1.0.0
         version: 1.0.0
+      pagefind:
+        specifier: 1.4.0
+        version: 1.4.0
       plist:
         specifier: ^3.0.1
         version: 3.0.1
@@ -1254,6 +1257,36 @@ packages:
   '@oslojs/encoding@1.1.0':
     resolution: {integrity: sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==}
 
+  '@pagefind/darwin-arm64@1.4.0':
+    resolution: {integrity: sha512-2vMqkbv3lbx1Awea90gTaBsvpzgRs7MuSgKDxW0m9oV1GPZCZbZBJg/qL83GIUEN2BFlY46dtUZi54pwH+/pTQ==}
+    cpu: [arm64]
+    os: [darwin]
+
+  '@pagefind/darwin-x64@1.4.0':
+    resolution: {integrity: sha512-e7JPIS6L9/cJfow+/IAqknsGqEPjJnVXGjpGm25bnq+NPdoD3c/7fAwr1OXkG4Ocjx6ZGSCijXEV4ryMcH2E3A==}
+    cpu: [x64]
+    os: [darwin]
+
+  '@pagefind/freebsd-x64@1.4.0':
+    resolution: {integrity: sha512-WcJVypXSZ+9HpiqZjFXMUobfFfZZ6NzIYtkhQ9eOhZrQpeY5uQFqNWLCk7w9RkMUwBv1HAMDW3YJQl/8OqsV0Q==}
+    cpu: [x64]
+    os: [freebsd]
+
+  '@pagefind/linux-arm64@1.4.0':
+    resolution: {integrity: sha512-PIt8dkqt4W06KGmQjONw7EZbhDF+uXI7i0XtRLN1vjCUxM9vGPdtJc2mUyVPevjomrGz5M86M8bqTr6cgDp1Uw==}
+    cpu: [arm64]
+    os: [linux]
+
+  '@pagefind/linux-x64@1.4.0':
+    resolution: {integrity: sha512-z4oddcWwQ0UHrTHR8psLnVlz6USGJ/eOlDPTDYZ4cI8TK8PgwRUPQZp9D2iJPNIPcS6Qx/E4TebjuGJOyK8Mmg==}
+    cpu: [x64]
+    os: [linux]
+
+  '@pagefind/windows-x64@1.4.0':
+    resolution: {integrity: sha512-NkT+YAdgS2FPCn8mIA9bQhiBs+xmniMGq1LFPDhcFn0+2yIUEiIG06t7bsZlhdjknEQRTSdT7YitP6fC5qwP0g==}
+    cpu: [x64]
+    os: [win32]
+
   '@parcel/watcher-android-arm64@2.5.6':
     resolution: {integrity: sha512-YQxSS34tPF/6ZG7r/Ih9xy+kP/WwediEUsqmtf0cuCV5TPPKw/PQHRhueUo6JdeFJaqV3pyjm0GdYjZotbRt/A==}
     engines: {node: '>= 10.0.0'}
@@ -5808,6 +5841,10 @@ packages:
   package-manager-detector@1.6.0:
     resolution: {integrity: sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA==}
 
+  pagefind@1.4.0:
+    resolution: {integrity: sha512-z2kY1mQlL4J8q5EIsQkLzQjilovKzfNVhX8De6oyE6uHpfFtyBaqUpcl/XzJC/4fjD8vBDyh1zolimIcVrCn9g==}
+    hasBin: true
+
   parent-module@1.0.1:
     resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}
     engines: {node: '>=6'}
@@ -9217,6 +9254,24 @@ snapshots:
 
   '@oslojs/encoding@1.1.0': {}
 
+  '@pagefind/darwin-arm64@1.4.0':
+    optional: true
+
+  '@pagefind/darwin-x64@1.4.0':
+    optional: true
+
+  '@pagefind/freebsd-x64@1.4.0':
+    optional: true
+
+  '@pagefind/linux-arm64@1.4.0':
+    optional: true
+
+  '@pagefind/linux-x64@1.4.0':
+    optional: true
+
+  '@pagefind/windows-x64@1.4.0':
+    optional: true
+
   '@parcel/watcher-android-arm64@2.5.6':
     optional: true
 
@@ -14522,6 +14577,15 @@ snapshots:
 
   package-manager-detector@1.6.0: {}
 
+  pagefind@1.4.0:
+    optionalDependencies:
+      '@pagefind/darwin-arm64': 1.4.0
+      '@pagefind/darwin-x64': 1.4.0
+      '@pagefind/freebsd-x64': 1.4.0
+      '@pagefind/linux-arm64': 1.4.0
+      '@pagefind/linux-x64': 1.4.0
+      '@pagefind/windows-x64': 1.4.0
+
   parent-module@1.0.1:
     dependencies:
       callsites: 3.1.0

scripts/build-pagefind-index.js

@@ -0,0 +1,43 @@
+import fs from 'fs-extra'
+import axios from 'axios'
+import 'dotenv/config.js'
+
+import {
+    sitemapEndpointsPath
+} from '~/helpers/pagefind/config.js'
+import {
+    writePagefindIndex
+} from '~/helpers/pagefind/index.js'
+
+async function loadSitemapEndpoints () {
+    if ( await fs.pathExists( sitemapEndpointsPath ) ) {
+        return await fs.readJson( sitemapEndpointsPath )
+    }
+
+    if ( !process.env.PUBLIC_API_DOMAIN ) {
+        throw new Error(`Missing ${ sitemapEndpointsPath } and PUBLIC_API_DOMAIN is not set`)
+    }
+
+    const apiUrl = new URL( process.env.PUBLIC_API_DOMAIN )
+    apiUrl.pathname = sitemapEndpointsPath.replace(/^\.?\/?static\//, '/')
+
+    const response = await axios.get( apiUrl.toString() )
+    await fs.outputJson( sitemapEndpointsPath, response.data, { spaces: 2 } )
+
+    return response.data
+}
+
+;(async () => {
+    const sitemapEndpoints = await loadSitemapEndpoints()
+    const {
+        outputPath,
+        recordCount
+    } = await writePagefindIndex( sitemapEndpoints )
+
+    console.log(`Built Pagefind index with ${ recordCount } records at ${ outputPath }`)
+
+    process.exit()
+})().catch( error => {
+    console.error( error )
+    process.exit(1)
+})

scripts/build-search-index.js

@@ -0,0 +1,18 @@
+import { execSync } from 'child_process'
+import 'dotenv/config.js'
+
+import {
+    getSearchProvider
+} from '~/helpers/search/config.js'
+
+const searchProvider = getSearchProvider( process.env.PUBLIC_SEARCH_PROVIDER )
+
+console.log(`Building search index for provider: ${ searchProvider }`)
+
+if ( searchProvider === 'stork' ) {
+    execSync( 'pnpm stork-index', { stdio: 'inherit' } )
+    process.exit()
+}
+
+execSync( 'pnpm build-pagefind-index', { stdio: 'inherit' } )
+process.exit()

test/prebuild/pagefind.test.js

@@ -0,0 +1,64 @@
+import { describe, it, expect } from 'vitest'
+
+import {
+    mapSitemapEntryToPagefindRecord,
+    shouldIndexSitemapEntry
+} from '~/helpers/pagefind/index.js'
+
+describe('Pagefind records', () => {
+    it('should skip sitemap entries without searchable payloads', () => {
+        expect( shouldIndexSitemapEntry({
+            route: '/games',
+            payload: {}
+        }) ).toBe( false )
+    })
+
+    it('should map benchmark entries to Pagefind records', () => {
+        const record = mapSitemapEntryToPagefindRecord({
+            route: '/app/example/benchmarks',
+            payload: {
+                app: {
+                    name: 'Example App',
+                    text: '✅ Native support',
+                    content: 'Runs fast on Apple Silicon',
+                    aliases: [ 'Example' ],
+                    tags: [ 'Utilities' ],
+                    status: 'no-in-progress',
+                    slug: 'example',
+                    category: {
+                        slug: 'system-tools',
+                        label: 'System Tools'
+                    },
+                    lastUpdated: {
+                        timestamp: 1234567890
+                    }
+                }
+            }
+        })
+
+        expect( record ).toMatchObject({
+            url: '/app/example/benchmarks',
+            language: 'en',
+            meta: {
+                title: 'Example App Benchmarks',
+                text: '✅ Native support',
+                slug: 'example',
+                categorySlug: 'system-tools',
+                routeType: 'benchmarks',
+                lastUpdatedTimestamp: '1234567890'
+            },
+            filters: {
+                status: [ 'no_in_progress' ],
+                category: [ 'system_tools' ],
+                type: [ 'benchmarks' ]
+            },
+            sort: {
+                updated: '1234567890'
+            }
+        })
+
+        expect( record.content ).toContain('Example App Benchmarks')
+        expect( record.content ).toContain('Runs fast on Apple Silicon')
+        expect( record.content ).toContain('Apple Silicon App Tested')
+    })
+})