Build lists before generating site

build-lists.js

@@ -0,0 +1,250 @@
+import { promises as fs } from 'fs'
+import dotenv from 'dotenv'
+
+import buildAppList from './helpers/build-app-list.js'
+import buildGamesList from './helpers/build-game-list.js'
+import buildHomebrewList from './helpers/build-homebrew-list.js'
+import buildVideoList from './helpers/build-video-list.js'
+
+import { buildVideoPayload, buildAppBenchmarkPayload } from './helpers/build-payload.js'
+
+import { categories } from './helpers/categories.js'
+import { getAppEndpoint, getVideoEndpoint } from './helpers/app-derived.js'
+
+// Setup dotenv
+dotenv.config()
+
+
+class BuildLists {
+
+    constructor () {
+        // this.apps = new Set()
+        // this.games = new Set()
+        // this.homebrewFormulae = new Set()
+
+        this.lists = {}
+
+        this.nuxtEndpointsSet = new Set()
+        this.eleventyEndpointsSet = new Set()
+
+        // Videos contains app and game data
+        // so it goes during the second pass
+        // this.videos = new Set()
+    }
+
+    listsOptions = [
+        {
+            name: 'apps',
+            path: '/static/app-list.json',
+            buildMethod: buildAppList,
+        },
+        {
+            name: 'games',
+            path: '/static/game-list.json',
+            buildMethod: buildGamesList,
+        },
+        {
+            name: 'homebrewFormulae',
+            path: '/static/homebrew-list.json',
+            buildMethod: buildHomebrewList,
+        },
+
+        // Always goes after initial lists
+        {
+            name: 'videos',
+            path: '/static/video-list.json',
+            buildMethod: async () => {
+
+                // const videoList = await buildVideoList( buildArgs )
+
+                // const extraVideos = []
+
+                // const multiplier = 12
+
+                // for (let i = 0; i < multiplier; i++) {
+                //     videoList.forEach( video => {
+                //         extraVideos.push({
+                //             ...video,
+                //             slug: video.slug + '-' + i,
+                //         })
+                //     })
+                // }
+
+                // return [
+                //     ...videoList,
+                //     ...extraVideos
+                // ].slice(0, 10 * 1000)
+
+                // return await this.saveList(videoListOptions, allVideoAppsList)
+
+                return await buildVideoList( this.getAllVideoAppsList() )
+            },
+        }
+    ]
+
+    getAllVideoAppsList = () => {
+        // return new Set([
+        //     ...this.lists.apps,
+        //     ...this.lists.games,
+        // ])
+
+        return [
+            ...Array.from(this.lists.apps),
+            ...Array.from(this.lists.games),
+        ]
+    }
+
+    saveToJson = async function ( content, path ) {
+
+        // Write the list to JSON
+        await fs.writeFile(path, JSON.stringify(content))
+
+        return
+    }
+
+    saveList = async function ( listOptions ) {
+
+        if (this.lists[listOptions.name].size === 0) throw new Error('Trying to save empty list')
+
+        // Make the relative path for our new JSON file
+        const listFullPath = `.${listOptions.path}`
+
+        // console.log('listFullPath', listFullPath)
+
+        // Write the list to JSON
+        await fs.writeFile(listFullPath, JSON.stringify(this.lists[listOptions.name]))
+
+        // Read back the JSON we just wrote to ensure it exists
+        const savedListJSON = await fs.readFile(listFullPath, 'utf-8')
+
+        // console.log('savedListJSON', savedListJSON)
+
+        const savedList = JSON.parse(savedListJSON)
+
+        // Import the created JSON File
+        return savedList
+    }
+
+    async buildLists () {
+        console.log('Build Lists started')
+
+
+        for (const listOptions of this.listsOptions) {
+
+            const methodName = `Building ${listOptions.path}`
+            console.time(methodName)
+
+            // Run the build method to get the lists
+            this.lists[listOptions.name] = await listOptions.buildMethod()
+
+
+            console.timeEnd(methodName)
+        }
+
+        console.log('Build Lists finished')
+
+        return
+    }
+
+    storeAppLists = async function () {
+
+        if (Object.keys(this.listsOptions).length === 0) throw new Error('Trying to store empty lists')
+
+        for ( const listOptionsKey in this.listsOptions ) {
+            await this.saveList(this.listsOptions[listOptionsKey])
+        }
+
+        return
+    }
+
+    async build () {
+
+        await this.buildLists()
+
+        await this.storeAppLists()
+
+        // console.log('appList', appList)
+
+        // Break out lists
+        // const [
+        //     appList,
+        //     gameList,
+        //     _,//homebrewList,
+
+        //     videoList
+        // ] = savedLists
+
+        // console.log('appList', appList)
+
+        const allVideoAppsList = this.getAllVideoAppsList()
+
+        // console.log('allVideoAppsList', allVideoAppsList[0])
+        // console.log('videoList', videoList[0])
+
+        // const allEndpointsSet = new Set()
+
+        // Add list based routes
+        for ( const listKey in this.lists ) {
+
+            this.lists[listKey].forEach( app => {
+
+                const isVideo = (app.category === undefined)
+                const isApp = (app.endpoint.includes('/app/'))
+                const isGame = (app.category === 'games')
+
+                if (isVideo) {
+                    this.eleventyEndpointsSet.add({
+                        route: getVideoEndpoint(app),
+                        payload: buildVideoPayload( app, allVideoAppsList, this.lists.videos )
+                    })
+
+                    return
+                }
+
+                // Add benchmark endpoints for apps and games
+                if ( isApp || isGame ) {
+                    this.nuxtEndpointsSet.add({
+                        route: `${getAppEndpoint(app)}/benchmarks`,
+                        payload: buildAppBenchmarkPayload( app, allVideoAppsList, this.lists.videos )
+                    })
+                }
+
+                this.nuxtEndpointsSet.add({
+                    route: getAppEndpoint(app),
+                    payload: { app }
+                })
+
+                return
+            })
+
+        }
+
+
+        Object.keys(categories).forEach( slug => {
+            this.nuxtEndpointsSet.add({
+                route: '/kind/' + slug,
+                // payload: appList
+            })
+        })
+
+
+        // Save Nuxt Endpoints
+        await this.saveToJson(Array.from(this.nuxtEndpointsSet), './static/nuxt-endpoints.json')
+
+        // Save Eleventy Endpoints
+        await this.saveToJson(Array.from(this.eleventyEndpointsSet), './static/eleventy-endpoints.json')
+
+        return
+    }
+}
+
+
+const listBuilder = new BuildLists()
+
+listBuilder.build()
+
+// export default async function () {
+//     const listBuilder = new BuildLists()
+
+//     return await listBuilder.build()
+// }

nuxt.config.js

@@ -13,104 +13,7 @@ import { categories } from './helpers/categories.js'
 import { getAppEndpoint, getVideoEndpoint } from './helpers/app-derived.js'
 
 
-const listsOptions = [
-    {
-        buildMethod: buildAppList,
-        path: '/static/app-list.json',
-    },
-    {
-        buildMethod: buildGamesList,
-        path: '/static/game-list.json',
-    },
-    {
-        buildMethod: buildHomebrewList,
-        path: '/static/homebrew-list.json',
-    }
-]
 
-const videoListOptions = {
-    buildMethod: async buildArgs => {
-        return await buildVideoList( buildArgs )
-
-        // const videoList = await buildVideoList( buildArgs )
-
-        // const extraVideos = []
-
-        // const multiplier = 12
-
-        // for (let i = 0; i < multiplier; i++) {
-        //     videoList.forEach( video => {
-        //         extraVideos.push({
-        //             ...video,
-        //             slug: video.slug + '-' + i,
-        //         })
-        //     })
-        // }
-
-        // return [
-        //     ...videoList,
-        //     ...extraVideos
-        // ].slice(0, 10 * 1000)
-    },
-    path: '/static/video-list.json',
-}
-
-
-const saveList = async function ( list, buildArgs = null ) {
-    const methodName = `Building ${list.path}`
-    console.time(methodName)
-
-    // Run the build method
-    const builtList = await list.buildMethod(buildArgs)
-
-    // Make the relative path for our new JSON file
-    const listFullPath = `.${list.path}`
-
-    // console.log('listFullPath', listFullPath)
-
-    // Write the list to JSON
-    await fs.writeFile(listFullPath, JSON.stringify(builtList))
-
-    // Read back the JSON we just wrote to ensure it exists
-    const savedListJSON = await fs.readFile(listFullPath, 'utf-8')
-
-    // console.log('savedListJSON', savedListJSON)
-
-    const savedList = JSON.parse(savedListJSON)
-
-
-    console.timeEnd(methodName)
-
-    // Import the created JSON File
-    return savedList
-}
-
-
-const storeAppLists = async function (builder) {
-
-    console.log('Build Lists started')
-
-    const savedLists = await Promise.all(listsOptions.map(saveList))
-        // Build and save list of videos based on app lists
-        .then(async lists => {
-            const [
-                appList,
-                gameList
-            ] = lists
-
-            // Build a video app list with apps and games only
-            const allVideoAppsList = [
-                ...appList,
-                ...gameList
-            ].flat(1)
-
-            return await saveList(videoListOptions, allVideoAppsList)
-        })
-
-    console.log('Build Lists finished')
-
-    return savedLists
-}
 
 
 export default {
@@ -125,12 +28,12 @@ export default {
     * https://nuxtjs.org/api/configuration-hooks/
     */
     hooks: {
-        build: {
+        // build: {
-            before: storeAppLists
+        //     before: storeAppLists
-        },
+        // },
-        generate: {
+        // generate: {
-            before: storeAppLists
+        //     before: storeAppLists
-        }
+        // }
     },
 
     generate: {
@@ -142,102 +45,9 @@ export default {
             ]
         },
         routes() {
-            return Promise.all([
+            return fs.readFile('./static/nuxt-endpoints.json', 'utf-8')
-                ...listsOptions,
+                .then( endpointsJson => {
-                videoListOptions
+                    return JSON.parse(endpointsJson)
-            ].map(async list => {
-                // Read saved lists
-
-                const methodName = `Reading ${list.path}`
-                console.time(methodName)
-
-                const listPath = `.${list.path}`
-
-                // Read JSON to ensure it exists
-                const savedListJSON = await fs.readFile(listPath, 'utf-8')
-
-                // Parse the saved JSON into a variable
-                const savedList = JSON.parse(savedListJSON)
-
-                console.timeEnd(methodName)
-
-                // Pass on the variable
-                return savedList
-            }))
-                .then(( lists ) => {
-                    // console.log('appList', appList)
-
-                    // Break out lists
-                    const [
-                        appList,
-                        gameList,
-                        _,//homebrewList,
-
-                        videoList
-                    ] = lists
-
-                    const allVideoAppsList = [
-                        ...appList,
-                        ...gameList
-                    ]
-
-                    // console.log('allVideoAppsList', allVideoAppsList[0])
-                    // console.log('videoList', videoList[0])
-
-                    const [
-                        appRoutes,
-                        gameRoutes,
-                        homebrewRoutes,
-
-                        // videoRoutes
-                    ] = lists.map((list, listI) => {
-                        return list.map( app => {
-
-                            const isVideo = (app.category === undefined)
-
-                            if (isVideo) {
-                                return {
-                                    route: getVideoEndpoint(app),
-                                    payload: {}//buildVideoPayload(app, allVideoAppsList, videoList)
-                                }
-                            }
-
-                            return {
-                                route: getAppEndpoint(app),
-                                payload: { app }
-                            }
-                        })
-                    })
-
-                    // Build routes for app types that support benchmark endpoints
-                    const benchmarkRoutes = [
-                        ...appRoutes,
-                        ...gameRoutes,
-                    ].flat(1).map( ({ route, payload: { app } }) => ({
-                        route: `${route}/benchmarks`,
-                        payload: buildAppBenchmarkPayload( app, allVideoAppsList, videoList )
-                    }))
-
-                    // console.log('homebrewRoutes', homebrewRoutes)
-
-                    const categoryRoutes = Object.keys(categories).map( slug => ({
-                        route: '/kind/' + slug,
-                        // payload: appList
-                    }))
-
-                    // Merge endpoints into set to ensure no duplicates
-                    const allEndpointsSet = new Set([
-                        ...appRoutes,
-                        ...gameRoutes,
-                        ...homebrewRoutes,
-
-                        // Non-app routes
-                        // ...videoRoutes,
-                        ...categoryRoutes,
-                        ...benchmarkRoutes
-                    ])
-
-                    return Array.from(allEndpointsSet)
                 })
         }
     },

package.json

@@ -8,7 +8,8 @@
     "dev": "nuxt",
     "build": "nuxt build",
     "start": "nuxt start",
-    "generate": "rm -f ./static/app-list.json && npm run clone-readme && npm run generate-nuxt && npm run generate-eleventy",
+    "generate": "npm run clone-readme && npm run build-lists && npm run generate-nuxt && npm run generate-eleventy",
+    "build-lists": "node -r esm build-lists.js",
     "generate-nuxt": "nuxt generate",
     "generate-eleventy": "node -r esm node_modules/.bin/eleventy",
     "lint": "eslint --ext .js,.vue --ignore-path .gitignore .",
@@ -34,6 +35,7 @@
     "@nuxtjs/tailwindcss": "^3.3.4",
     "autoprefixer": "^8.6.4",
     "babel-eslint": "^8.2.1",
+    "dotenv": "^8.2.0",
     "eslint": "^5.16.0",
     "eslint-config-prettier": "^3.1.0",
     "eslint-loader": "^2.0.0",