doesitarm/helpers/build-video-list.js

import fs from 'fs-extra'
import axios from 'axios'
import { PromisePool } from '@supercharge/promise-pool'

import { fuzzyMatchesWholeWord } from './matching.js'
import { byTimeThenNull } from './sort-list.js'
import { getVideoEndpoint } from './app-derived.js'
import parseDate from './parse-date'
import { makeSlug } from './slug.js'
import { youtubeVideoPath } from '~/helpers/api/youtube/build.js'


const inTimestamps = ( name, video ) => {
    // If this is empty
    // then reutrn false
    if ( video.timestamps.length === 0 ) return false

    for ( const timestamp of video.timestamps ) {
        if ( fuzzyMatchesWholeWord(name, timestamp.fullText ) ) return true
    }

    return false
}

const videoFeaturesApp = function (app, video) {
    const appFuzzyName = app.name.toLowerCase()

    if ( fuzzyMatchesWholeWord(appFuzzyName, video.title ) ) return true

    if ( inTimestamps(appFuzzyName, video) ) return true

    if ( fuzzyMatchesWholeWord(appFuzzyName, video.description ) ) return true

    return false
}

const generateVideoTags = function ( video ) {
    const tags = {
        'benchmark': {
            relatedWords: [
                'benchmarks',
                'comparison',
                'speed test',
                'bench mark',
                'bench marks'
            ]
        },
        'performance': {
            relatedWords: [
                'speed'
            ]
        }
    }

    const videoTags = new Set()

    video.tags.forEach( tag => {
        videoTags.add(tag)
    })

    // Match tags against video titles and descriptions
    for (const tagKey in tags) {

        // Skip if this video already has this tag
        // then skip it
        if (videoTags.has(tagKey)) continue

        const matchingWords = [
            tagKey,
            ...tags[tagKey].relatedWords
        ]

        for (const tagWord of matchingWords) {

            // Skip if this video already has this tag
            // then stop this loop
            if (videoTags.has(tagKey)) break

            // Check title
            if (fuzzyMatchesWholeWord( tagWord, video.title )) {
                videoTags.add(tagKey)

                // console.log(video.title, 'has', tagKey, 'tag')

                // We're done since the tag matched for the title
                continue
            }

            // Check description
            if (fuzzyMatchesWholeWord( tagWord, video.description )) {
                videoTags.add(tagKey)

                // console.log(video.title, 'has', tagKey, 'tag')
            }
        }
    }

    return videoTags
}

const makeThumbnailData = function ( thumbnails, widthLimit = null ) {

    const thumbnailEntries = Object.entries( thumbnails )
    const srcsetArray = []

    let maxWidth = 0

    thumbnailEntries.forEach(([thumbnailKey, thumbnail]) => {
        if ( widthLimit !== null &&  widthLimit < thumbnail.width) return

        // If this width is more than known maxWidth
        // then set maxWidth
        if (thumbnail.width > maxWidth) maxWidth = thumbnail.width

        // Add this width to our srcset
        srcsetArray.push(`${thumbnail.url} ${thumbnail.width}w`)
    })


    const sizes = `(max-width: ${maxWidth}px) 100vw, ${maxWidth}px`
    const srcset = srcsetArray.join(', ')
    const src = thumbnails.default.url

    // console.log('srcsetArray', srcsetArray)

    return {
        sizes,
        srcset,
        src
    }
}

async function handleFetchedVideo ( fetchedVideo, videoId, applist ) {

    // Skip private videos
    if (fetchedVideo.title === 'Private video') return

    // Skip deleted videos
    if (fetchedVideo.title === 'Deleted video') return

    // Build video slug
    const slug = makeSlug( `${fetchedVideo.title}-i-${videoId}` )

    const apps = []
    // Generate new tag set based on api data
    const tags = generateVideoTags(fetchedVideo)

    for ( const app of applist ) {
        if (videoFeaturesApp(app, fetchedVideo)) {
            apps.push(app.slug)

            tags.add(app.category.slug)
        }
    }

    // console.log('fetchedVideo.rawData.snippet', fetchedVideo.rawData.snippet)

    const lastUpdated = {
        raw: fetchedVideo.rawData.snippet.publishedAt,
        timestamp: parseDate(fetchedVideo.rawData.snippet.publishedAt).timestamp,
    }

    // console.log('fetchedVideo.thumbnails', fetchedVideo.thumbnails)

    return {
        name: fetchedVideo.title,
        id: videoId,
        lastUpdated,
        apps,
        slug,
        channel: {
            name: fetchedVideo.rawData.snippet.channelTitle,
            id: fetchedVideo.rawData.snippet.channelId
        },
        // Convert tags set into array
        tags: Array.from(tags),
        timestamps: fetchedVideo.timestamps,
        // thumbnails: fetchedVideo.rawData.snippet.thumbnails,
        thumbnail: makeThumbnailData( fetchedVideo.rawData.snippet.thumbnails, 700 ),
        endpoint: getVideoEndpoint({
            slug
        })
    }
}

export default async function ( applist ) {

    // const videosJsonUrl = process.env.VIDEO_SOURCE || `${process.env.VFUNCTIONS_URL}/videos.json`

    // Fetch Commits
    // const response = await axios.get( videosJsonUrl )
    // Extract commit from response data
    const fetchedVideos = await fs.readJson( youtubeVideoPath )//response.data

    const videos = []

    await PromisePool
        .withConcurrency(1000)
        .for( Object.entries( fetchedVideos ) )
        .process(async ( [ videoId, fetchedVideo ], index, pool ) => {
            const mappedVideo = await handleFetchedVideo ( fetchedVideo, videoId, applist )

            // Skip if this video is not an object
            if ( Object( mappedVideo ) !== mappedVideo ) return

            videos.push( mappedVideo )
        })

    // console.log('videos', videos)

    // publishedAt

    return videos.sort(byTimeThenNull)
}