Fetch youtube videos from playlists

helpers/api/youtube/build.js

@@ -0,0 +1,172 @@
+import fs from 'fs-extra'
+import { google } from 'googleapis'
+
+import { playlists, benchmarksPlaylistId } from './playlists.js'
+
+
+const youtubeVideoPath = './static/api/youtube-videos.json'
+
+
+async function getPlaylistsItems ( { playlistId } = {} ) {
+    const perPage = 50
+
+    // Setup Youtube API V3 Service instance
+    const service = google.youtube('v3')
+
+    // Fetch data from the Youtube API
+    const { errors = null, data = null } = await service.playlistItems.list({
+        key: process.env.GOOGLE_API_KEY,
+        part: 'snippet,contentDetails',
+        playlistId,
+        maxResults: perPage
+    }).catch(({ errors }) => {
+
+        console.log('Error fetching playlist', errors)
+
+        return {
+            errors
+        }
+    })
+
+    // Send an error response if something went wrong
+    if (errors !== null) {
+        throw new Error(errors)
+
+        return
+    }
+
+    const items = data.items
+
+    // If there are more results then push them to our playlist
+    if (data.nextPageToken !== null) {
+
+        // Store the token for page #2 into our variable
+        let pageToken = data.nextPageToken
+
+        while (pageToken !== null) {
+            // Fetch data from the Youtube API
+            const youtubePageResponse = await service.playlistItems.list({
+                key: process.env.GOOGLE_API_KEY,
+                part: 'snippet,contentDetails',
+                playlistId,
+                maxResults: perPage,
+                pageToken: pageToken
+            })
+
+            // Add the videos from this page on to our total items list
+            youtubePageResponse.data.items.forEach(item => items.push(item))
+
+            // Now that we're done set up the next page token or empty out the pageToken variable so our loop will stop
+            pageToken = ('nextPageToken' in youtubePageResponse.data) ? youtubePageResponse.data.nextPageToken : null
+        }
+    }
+
+    console.log(`Fetched ${items.length} videos from https://www.youtube.com/playlist?list=${ playlistId }`)
+
+    return items
+}
+
+async function getYouTubeVideos ( options = {} ) {
+
+    const {
+        // requestsDelay = 3600,
+    } = options
+
+    // Fetch all videos from playlists
+    const playlistSets = []
+
+    for ( const playlistToFetch of playlists ) {
+
+        // console.log('playlistJsonUrl', playlistJsonUrl)
+
+        const playlistItems = await getPlaylistsItems({
+            playlistId: playlistToFetch.id
+        })
+        // console.log('playlistItems', playlistItems.length)
+
+        playlistSets.push( playlistItems )
+    }
+
+    // Pull benchmarksPlaylist out of playlist sets
+    // benchmarksPlaylistId
+    const benchmarksVideoIds = playlistSets.find( playlist => {
+        // Skip empty playlists
+        if (playlist.length === 0) return false
+
+        // Get this playlist's ID from first video
+        // and check against benchmarksPlaylistId
+        return playlist[0].snippet.playlistId === benchmarksPlaylistId
+    }).map( video => video.contentDetails.videoId)
+
+    // Creat an object to store playlist items
+    const playlistItems = {}
+
+
+    // Loop through the sets and store all the videos into a single array
+    for (const playlistSet of playlistSets) {
+        for (const playlistItem of playlistSet) {
+            // If we've already stored this video
+            // then skip
+            if (playlistItems.hasOwnProperty(playlistItem.contentDetails.videoId)) continue
+
+            const tags = []
+
+            // If this video is in the benchmarks playlist
+            // then add the benchmark tag
+            if (benchmarksVideoIds.includes(playlistItem.contentDetails.videoId)) {
+                tags.push('benchmark')
+            }
+
+            // Store newly found video
+            playlistItems[playlistItem.contentDetails.videoId] = {
+                title: playlistItem.snippet.title,
+                description: playlistItem.snippet.description,
+                timestamps: [],
+                rawData: playlistItem,
+                tags
+            }
+        }
+    }
+
+
+    // Loop through playlist items and store timestamp data
+    for (const videoId in playlistItems) {
+        // console.log('playlistItem', playlistItem)
+        // If the description is empty
+        // then skip
+        if (playlistItems[videoId].description.trim().length === 0) continue
+
+        // Break up the description by line breaks
+        const descriptionLines = playlistItems[videoId].description.split(/\r?\n/)
+
+        // console.log('descriptionLines', descriptionLines)
+
+        for (const line of descriptionLines) {
+            // https://stackoverflow.com/a/11067610/1397641
+            const matches = line.match(/(?:([0-5]?[0-9]):)?([0-5]?[0-9]):([0-5][0-9])/)
+
+            // If there are no timestamps on this line
+            // then skip
+            if (matches === null) continue
+
+            playlistItems[videoId].timestamps.push({
+                time: matches[0],
+                fullText: line
+            })
+        }
+    }
+
+    return playlistItems
+}
+
+
+export async function saveYouTubeVideos () {
+    //
+    const youtubeVideos = await getYouTubeVideos()
+
+    //
+
+    // Save to JSON
+    await fs.outputJson( youtubeVideoPath, youtubeVideos )
+
+}