mirror of
https://github.com/ThatGuySam/doesitarm.git
synced 2026-05-18 06:44:46 -07:00
464 lines
15 KiB
JavaScript
464 lines
15 KiB
JavaScript
import { dirname } from 'path'
|
|
|
|
import fs from 'fs-extra'
|
|
import dotenv from 'dotenv'
|
|
import { PromisePool } from '@supercharge/promise-pool'
|
|
|
|
import buildAppList from './helpers/build-app-list.js'
|
|
import buildGamesList from './helpers/build-game-list.js'
|
|
import buildHomebrewList from './helpers/build-homebrew-list.js'
|
|
import buildVideoList from './helpers/build-video-list.js'
|
|
import buildDeviceList from './helpers/build-device-list.js'
|
|
|
|
import { videosRelatedToApp } from './helpers/related.js'
|
|
import { buildVideoPayload, buildAppBenchmarkPayload } from './helpers/build-payload.js'
|
|
|
|
import { categories, getAppCategory } from './helpers/categories.js'
|
|
import {
|
|
getAppType,
|
|
getAppEndpoint,
|
|
getVideoEndpoint,
|
|
isVideo
|
|
} from './helpers/app-derived.js'
|
|
import { makeSearchableList } from './helpers/searchable-list.js'
|
|
|
|
// Setup dotenv
|
|
dotenv.config()
|
|
|
|
const commandArguments = process.argv
|
|
const cliOptions = {
|
|
withApi: commandArguments.includes('--with-api'),
|
|
noLists: commandArguments.includes('--no-lists'),
|
|
}
|
|
|
|
|
|
|
|
class BuildLists {
|
|
|
|
constructor () {
|
|
// Where our lists are stored
|
|
this.lists = {}
|
|
|
|
this.endpointMaps = {
|
|
// Where Nuxt Routes and Payloads get stored
|
|
nuxt: new Map(),
|
|
|
|
// Where Eleventy Endpoints get stored
|
|
eleventy: new Map()
|
|
}
|
|
|
|
this.allVideoAppsList = new Set()
|
|
}
|
|
|
|
listsOptions = [
|
|
|
|
// Mixed sources will(theoretically) go here
|
|
// then be read by follow main build methods
|
|
|
|
|
|
// Main build methods
|
|
{
|
|
name: 'app',
|
|
path: '/static/app-list.json',
|
|
buildMethod: buildAppList,
|
|
},
|
|
{
|
|
name: 'game',
|
|
path: '/static/game-list.json',
|
|
buildMethod: buildGamesList,
|
|
},
|
|
{
|
|
name: 'homebrew',
|
|
path: '/static/homebrew-list.json',
|
|
buildMethod: buildHomebrewList,
|
|
},
|
|
{
|
|
name: 'device',
|
|
path: '/static/device-list.json',
|
|
buildMethod: buildDeviceList,
|
|
},
|
|
|
|
// Secondary Derivative built lists
|
|
// Always goes after initial lists
|
|
// since it depend on them
|
|
{
|
|
name: 'video',
|
|
path: '/static/video-list.json',
|
|
buildMethod: async () => {
|
|
|
|
// console.log('this.getAllVideoAppsList()', this.getAllVideoAppsList())
|
|
|
|
return await buildVideoList( this.getAllVideoAppsList() )
|
|
|
|
|
|
// const videoList = await buildVideoList( this.getAllVideoAppsList() )
|
|
|
|
// const extraVideos = []
|
|
|
|
// const multiplier = 12
|
|
|
|
// for (let i = 0; i < multiplier; i++) {
|
|
// videoList.forEach( video => {
|
|
// extraVideos.push({
|
|
// ...video,
|
|
// slug: video.slug + '-' + i,
|
|
// })
|
|
// })
|
|
// }
|
|
|
|
// return new Set([
|
|
// ...videoList,
|
|
// ...extraVideos
|
|
// ].slice(0, 10 * 1000))
|
|
},
|
|
beforeSave: videoListSet => {
|
|
this.allVideoAppsList = this.getAllVideoAppsList()
|
|
|
|
return Array.from(videoListSet).map( video => {
|
|
return {
|
|
...video,
|
|
payload: buildVideoPayload( video, this.allVideoAppsList, this.lists.video )
|
|
}
|
|
})
|
|
}
|
|
}
|
|
]
|
|
|
|
shouldHaveRelatedVideos ( app ) {
|
|
const appType = getAppType( app )
|
|
|
|
return appType === 'app' || appType === 'formula'
|
|
}
|
|
|
|
getAllVideoAppsList = () => {
|
|
return new Set([
|
|
...this.lists.app,
|
|
...this.lists.game,
|
|
])
|
|
}
|
|
|
|
bundles = []
|
|
async getSavedAppBundles ({ keepBundlesInMemory = true }) {
|
|
if ( !keepBundlesInMemory ) {
|
|
return await fs.readJson('./static/app-bundles.json')
|
|
}
|
|
|
|
// From here we get and store bundles
|
|
|
|
console.log('this.bundles.length', this.bundles.length)
|
|
|
|
// Check if any bundles are already in memory
|
|
if ( this.bundles.length > 0 ) {
|
|
return this.bundles
|
|
}
|
|
|
|
this.bundles = await fs.readJson('./static/app-bundles.json')
|
|
|
|
return this.bundles
|
|
}
|
|
|
|
// Load the bundles from files
|
|
// so that we don't have to keep them in memory
|
|
async findAppBundle ( needleBundleIdentifier ) {
|
|
const bundles = await this.getSavedAppBundles()
|
|
|
|
return bundles.find( ([
|
|
storedAppBundleIdentifier,
|
|
// versions
|
|
]) => storedAppBundleIdentifier === needleBundleIdentifier )
|
|
}
|
|
|
|
async getAppBundles ( app ) {
|
|
console.log('app', app)
|
|
return await Promise.all( app.bundleIds.map( async bundleIdentifier => {
|
|
return await this.findAppBundle( bundleIdentifier )
|
|
} ) )
|
|
}
|
|
|
|
saveToJson = async function ( content, path ) {
|
|
|
|
// Write the list to JSON
|
|
await fs.writeFile(path, JSON.stringify(content))
|
|
|
|
return
|
|
}
|
|
|
|
saveList = async function ( listOptions ) {
|
|
|
|
if (this.lists[listOptions.name].size === 0) throw new Error('Trying to save empty list')
|
|
|
|
// Make the relative path for our new JSON file
|
|
const listFullPath = `.${listOptions.path}`
|
|
|
|
const hasSaveMethod = listOptions.hasOwnProperty('beforeSave')
|
|
const saveMethod = hasSaveMethod ? listOptions.beforeSave : listSet => Array.from( listSet )
|
|
|
|
// console.log('listFullPath', listFullPath)
|
|
|
|
const saveableList = saveMethod( this.lists[listOptions.name] )
|
|
|
|
// console.log('saveableList', typeof saveableList)
|
|
|
|
// Stringify one at a time to allow for large lists
|
|
const saveableListJSON = '[' + saveableList.map(el => JSON.stringify(el)).join(',') + ']'
|
|
|
|
// Write the list to JSON
|
|
await fs.writeFile(listFullPath, saveableListJSON)
|
|
|
|
return
|
|
}
|
|
|
|
// Run all listsOprions methods
|
|
// and store them to this.lists
|
|
async buildLists () {
|
|
console.log( 'Build Lists started', cliOptions )
|
|
|
|
|
|
for ( const listOptions of this.listsOptions ) {
|
|
|
|
const methodName = `Building ${listOptions.name}`
|
|
console.time(methodName)
|
|
|
|
const builtList = await listOptions.buildMethod()
|
|
|
|
// Run the build method to get the lists
|
|
this.lists[listOptions.name] = new Set( builtList )
|
|
|
|
console.timeEnd(methodName)
|
|
console.log(`Finished ${listOptions.name} list with ${this.lists[listOptions.name].size} items`)
|
|
}
|
|
|
|
console.log('Build Lists finished')
|
|
|
|
return
|
|
}
|
|
|
|
saveApiEndpoints = async function ( listOptions ) {
|
|
|
|
const poolSize = 1000
|
|
|
|
await PromisePool
|
|
.withConcurrency( poolSize )
|
|
.for( Array.from( this.lists[listOptions.name] ) )
|
|
.process(async ( listEntry, index, pool ) => {
|
|
// console.log('listEntry', listEntry)
|
|
|
|
const {
|
|
// name,
|
|
// aliases,
|
|
// status,
|
|
// bundleIds,
|
|
endpoint,
|
|
|
|
} = listEntry
|
|
|
|
const endpointPath = `./static/api${endpoint}.json`
|
|
const endpointDirectory = dirname(endpointPath)
|
|
|
|
// Add related videos
|
|
if ( this.shouldHaveRelatedVideos( listEntry ) ) {
|
|
listEntry.relatedVideos = videosRelatedToApp( listEntry, this.lists.video ).map(video => {
|
|
// console.log('video', video)
|
|
return {
|
|
...video,
|
|
endpoint: `${getAppEndpoint(listEntry)}/benchmarks#${video.id}`
|
|
}
|
|
})
|
|
}
|
|
|
|
// Add App Bundles
|
|
if ( !!listEntry?.bundleIds ) {
|
|
listEntry.bundles = await this.getAppBundles( listEntry )
|
|
}
|
|
|
|
// console.log(`Saving endpoint "${endpoint}" to "${endpointPath}"`)
|
|
|
|
// Ensure the directory exists
|
|
await fs.ensureDir( endpointDirectory )
|
|
|
|
// Write the endpoint to JSON
|
|
await this.saveToJson( listEntry, endpointPath )
|
|
})
|
|
}
|
|
|
|
// Save app lists to JSON
|
|
saveAppLists = async function () {
|
|
|
|
if (Object.keys(this.listsOptions).length === 0) throw new Error('Trying to store empty lists')
|
|
|
|
console.log('Save lists started')
|
|
|
|
for ( const listOptionsKey in this.listsOptions ) {
|
|
|
|
const methodName = `Saving ${this.listsOptions[listOptionsKey].path}`
|
|
console.time(methodName)
|
|
|
|
const listOptions = this.listsOptions[listOptionsKey]
|
|
|
|
if ( !cliOptions.noLists ) {
|
|
await this.saveList( listOptions )
|
|
}
|
|
|
|
const searchableList = makeSearchableList( this.lists[listOptions.name] )
|
|
|
|
// console.log('searchableList', searchableList)
|
|
|
|
// Save a searchable list
|
|
await this.saveToJson( Array.from(searchableList), `./static/${listOptions.name}-list-searchable.json` )
|
|
|
|
console.timeEnd(methodName)
|
|
|
|
if ( cliOptions.withApi ) {
|
|
console.log('\n', `-- Starting individual /${ listOptions.name } endpoints`)
|
|
|
|
const endpointMethodName = `Finished individual /${ listOptions.name } endpoints`
|
|
console.time(endpointMethodName)
|
|
|
|
await this.saveApiEndpoints( listOptions )
|
|
|
|
console.timeEnd(endpointMethodName)
|
|
console.log( '\n\n' )
|
|
}
|
|
}
|
|
|
|
console.log('Save lists finished')
|
|
|
|
return
|
|
}
|
|
|
|
async build () {
|
|
|
|
await this.buildLists()
|
|
|
|
await this.saveAppLists()
|
|
|
|
|
|
|
|
// console.log('appList', appList)
|
|
|
|
// console.log('this.allVideoAppsList', this.allVideoAppsList.length, this.allVideoAppsList[0])
|
|
|
|
// Add list based routes
|
|
for ( const listKey in this.lists ) {
|
|
|
|
this.lists[listKey].forEach( app => {
|
|
|
|
// const isVideo = (app.category === undefined)
|
|
const appType = getAppType( app )
|
|
|
|
if ( isVideo( app ) ) {
|
|
this.endpointMaps.eleventy.set(
|
|
getVideoEndpoint(app),
|
|
buildVideoPayload( app, this.allVideoAppsList, this.lists.video )
|
|
)
|
|
|
|
// this.endpointMaps.nuxt.set( getVideoEndpoint(app), buildVideoPayload( app, this.allVideoAppsList, this.lists.video ) )
|
|
|
|
return
|
|
}
|
|
|
|
// if ( isGame ) { console.log() }
|
|
|
|
// Add benchmark endpoints for apps and games
|
|
if ( appType === 'app' || appType === 'game' ) {
|
|
const payload = buildAppBenchmarkPayload( app, this.allVideoAppsList, this.lists.video )
|
|
|
|
// Only add a benchmarks endpoint if it has any videos
|
|
if ( payload.allVideos.length > 0 ) {
|
|
// this.endpointMaps.nuxt.add({
|
|
// route: `${getAppEndpoint(app)}/benchmarks`,
|
|
// payload: buildAppBenchmarkPayload( app, this.allVideoAppsList, this.lists.video )
|
|
// })
|
|
|
|
this.endpointMaps.nuxt.set( `${getAppEndpoint(app)}/benchmarks`, buildAppBenchmarkPayload( app, this.allVideoAppsList, this.lists.video ) )
|
|
}
|
|
}
|
|
|
|
// Add standard app endpoint
|
|
if ( this.shouldHaveRelatedVideos( app ) ) {
|
|
|
|
const relatedVideos = videosRelatedToApp( app, this.lists.video ).map(video => {
|
|
// console.log('video', video)
|
|
return {
|
|
...video,
|
|
endpoint: `${getAppEndpoint(app)}/benchmarks#${video.id}`
|
|
}
|
|
})
|
|
|
|
// Add app or formula endpoint
|
|
this.endpointMaps.eleventy.set( getAppEndpoint(app), {
|
|
app,
|
|
relatedVideos
|
|
} )
|
|
|
|
} else if ( appType === 'device' ) {
|
|
// Add device endpoint
|
|
// console.log('Added to nuxt endpoints', app.endpoint )
|
|
this.endpointMaps.nuxt.set( app.endpoint , { listing: app } )
|
|
|
|
} else {
|
|
// Add game or other endpoint
|
|
// console.log('Added to nuxt endpoints', app.endpoint )
|
|
this.endpointMaps.nuxt.set( getAppEndpoint(app), { app } )
|
|
}
|
|
|
|
return
|
|
})
|
|
|
|
}
|
|
|
|
|
|
// Create endpoints for categories
|
|
Object.keys(categories).forEach( slug => {
|
|
// this.endpointMaps.nuxt.add({
|
|
// route: '/kind/' + slug,
|
|
// // payload: appList
|
|
// })
|
|
this.endpointMaps.nuxt.set( '/kind/' + slug, {} )
|
|
})
|
|
|
|
|
|
// Save Nuxt Endpoints
|
|
// await this.saveToJson(Array.from(this.endpointMaps.nuxt), './static/nuxt-endpoints.json')
|
|
|
|
// // Save Eleventy Endpoints
|
|
// await this.saveToJson(Array.from(this.endpointMaps.eleventy), './static/eleventy-endpoints.json')
|
|
|
|
// console.log('this.endpointMaps.eleventy /app/chrome', this.endpointMaps.eleventy.get( '/app/chrome' ))
|
|
|
|
|
|
// Filter eleventy endpoints
|
|
// this.endpointMaps.eleventy = new Set([
|
|
// ['/app/chrome', this.endpointMaps.eleventy.get( '/app/chrome' )]
|
|
// ])
|
|
|
|
if ( !cliOptions.withApi ) {
|
|
for ( const [ endpointSetName, endpointSet ] of Object.entries(this.endpointMaps) ) {
|
|
// Save Endpoints
|
|
await this.saveToJson(Array.from( endpointSet , ([route, payload]) => ({ route, payload })), `./static/${endpointSetName}-endpoints.json`)
|
|
}
|
|
|
|
// Save sitemap endpoints
|
|
await this.saveToJson(Object.values(this.endpointMaps).map( endpointSet => {
|
|
return Array.from( endpointSet , ([route, payload]) => ({ route, payload }) )
|
|
} ).flat(1), './static/sitemap-endpoints.json')
|
|
}
|
|
|
|
console.log('Total Nuxt Endpoints', this.endpointMaps.nuxt.size )
|
|
console.log('Total Eleventy Endpoints', this.endpointMaps.eleventy.size )
|
|
console.log('Total Endpoints', this.endpointMaps.nuxt.size + this.endpointMaps.eleventy.size )
|
|
|
|
return
|
|
}
|
|
}
|
|
|
|
|
|
const listBuilder = new BuildLists()
|
|
|
|
listBuilder.build()
|
|
|
|
// export default async function () {
|
|
// const listBuilder = new BuildLists()
|
|
|
|
// return await listBuilder.build()
|
|
// }
|