From 976ab06cc3c90c262d7f9d2b653357d56dcf9cbe Mon Sep 17 00:00:00 2001 From: Sam Carlton Date: Mon, 6 Jun 2022 11:48:25 -0500 Subject: [PATCH] Fetch youtube videos from playlists --- build-lists.js | 5 +- helpers/api/youtube/build.js | 172 ++++++++++++++++++++++++++++ helpers/api/youtube/playlists.js | 185 +++++++++++++++++++++++++++++++ 3 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 helpers/api/youtube/build.js create mode 100644 helpers/api/youtube/playlists.js diff --git a/build-lists.js b/build-lists.js index b0a3c3b..8a01791 100644 --- a/build-lists.js +++ b/build-lists.js @@ -5,8 +5,9 @@ import dotenv from 'dotenv' import semver from 'semver' import { PromisePool } from '@supercharge/promise-pool' import memoize from 'fast-memoize' -import has from 'just-has' +// import has from 'just-has' +import { saveYouTubeVideos } from '~/helpers/api/youtube/build.js' import buildAppList from '~/helpers/build-app-list.js' import buildGamesList from '~/helpers/build-game-list.js' import buildHomebrewList from '~/helpers/build-homebrew-list.js' @@ -574,6 +575,8 @@ class BuildLists { async build () { + await saveYouTubeVideos() + // Pull in and layer data from all sources await this.buildLists() diff --git a/helpers/api/youtube/build.js b/helpers/api/youtube/build.js new file mode 100644 index 0000000..f6515e5 --- /dev/null +++ b/helpers/api/youtube/build.js @@ -0,0 +1,172 @@ +import fs from 'fs-extra' +import { google } from 'googleapis' + +import { playlists, benchmarksPlaylistId } from './playlists.js' + + +const youtubeVideoPath = './static/api/youtube-videos.json' + + +async function getPlaylistsItems ( { playlistId } = {} ) { + const perPage = 50 + + // Setup Youtube API V3 Service instance + const service = google.youtube('v3') + + // Fetch data from the Youtube API + const { errors = null, data = null } = await service.playlistItems.list({ + key: process.env.GOOGLE_API_KEY, + part: 'snippet,contentDetails', + playlistId, + maxResults: perPage + }).catch(({ errors }) => { + + console.log('Error fetching playlist', errors) + + return { + errors + } + }) + + // Send an error response if something went wrong + if (errors !== null) { + throw new Error(errors) + + return + } + + const items = data.items + + // If there are more results then push them to our playlist + if (data.nextPageToken !== null) { + + // Store the token for page #2 into our variable + let pageToken = data.nextPageToken + + while (pageToken !== null) { + // Fetch data from the Youtube API + const youtubePageResponse = await service.playlistItems.list({ + key: process.env.GOOGLE_API_KEY, + part: 'snippet,contentDetails', + playlistId, + maxResults: perPage, + pageToken: pageToken + }) + + // Add the videos from this page on to our total items list + youtubePageResponse.data.items.forEach(item => items.push(item)) + + // Now that we're done set up the next page token or empty out the pageToken variable so our loop will stop + pageToken = ('nextPageToken' in youtubePageResponse.data) ? youtubePageResponse.data.nextPageToken : null + } + } + + console.log(`Fetched ${items.length} videos from https://www.youtube.com/playlist?list=${ playlistId }`) + + return items +} + +async function getYouTubeVideos ( options = {} ) { + + const { + // requestsDelay = 3600, + } = options + + // Fetch all videos from playlists + const playlistSets = [] + + for ( const playlistToFetch of playlists ) { + + // console.log('playlistJsonUrl', playlistJsonUrl) + + const playlistItems = await getPlaylistsItems({ + playlistId: playlistToFetch.id + }) + // console.log('playlistItems', playlistItems.length) + + playlistSets.push( playlistItems ) + } + + // Pull benchmarksPlaylist out of playlist sets + // benchmarksPlaylistId + const benchmarksVideoIds = playlistSets.find( playlist => { + // Skip empty playlists + if (playlist.length === 0) return false + + // Get this playlist's ID from first video + // and check against benchmarksPlaylistId + return playlist[0].snippet.playlistId === benchmarksPlaylistId + }).map( video => video.contentDetails.videoId) + + // Creat an object to store playlist items + const playlistItems = {} + + + // Loop through the sets and store all the videos into a single array + for (const playlistSet of playlistSets) { + for (const playlistItem of playlistSet) { + // If we've already stored this video + // then skip + if (playlistItems.hasOwnProperty(playlistItem.contentDetails.videoId)) continue + + const tags = [] + + // If this video is in the benchmarks playlist + // then add the benchmark tag + if (benchmarksVideoIds.includes(playlistItem.contentDetails.videoId)) { + tags.push('benchmark') + } + + // Store newly found video + playlistItems[playlistItem.contentDetails.videoId] = { + title: playlistItem.snippet.title, + description: playlistItem.snippet.description, + timestamps: [], + rawData: playlistItem, + tags + } + } + } + + + // Loop through playlist items and store timestamp data + for (const videoId in playlistItems) { + // console.log('playlistItem', playlistItem) + // If the description is empty + // then skip + if (playlistItems[videoId].description.trim().length === 0) continue + + // Break up the description by line breaks + const descriptionLines = playlistItems[videoId].description.split(/\r?\n/) + + // console.log('descriptionLines', descriptionLines) + + for (const line of descriptionLines) { + // https://stackoverflow.com/a/11067610/1397641 + const matches = line.match(/(?:([0-5]?[0-9]):)?([0-5]?[0-9]):([0-5][0-9])/) + + // If there are no timestamps on this line + // then skip + if (matches === null) continue + + playlistItems[videoId].timestamps.push({ + time: matches[0], + fullText: line + }) + } + } + + return playlistItems +} + + +export async function saveYouTubeVideos () { + // + const youtubeVideos = await getYouTubeVideos() + + // + + // Save to JSON + await fs.outputJson( youtubeVideoPath, youtubeVideos ) + +} diff --git a/helpers/api/youtube/playlists.js b/helpers/api/youtube/playlists.js new file mode 100644 index 0000000..4ed5e24 --- /dev/null +++ b/helpers/api/youtube/playlists.js @@ -0,0 +1,185 @@ + + +export const benchmarksPlaylistId = 'PLaa9cZC07ZPFqjoYLZRR3kbbnJRHhlmXq' + + + +export const playlists = [ + // Awais Mirza - Apple Silicon Mac Software Testing + // https://www.youtube.com/playlist?list=PLz5rnvLVJX5XF8cXAOQuarPIeP8Xr7b1_ + { + id: 'PLz5rnvLVJX5XF8cXAOQuarPIeP8Xr7b1_' + }, + // Andrew Tsai - M1 Apple Silicon Game Benchmarks + // https://www.youtube.com/playlist?list=PLFbqxkNlqrHNK0i4WN99Jc8g-qSbKoZEy + { + id: 'PLFbqxkNlqrHNK0i4WN99Jc8g-qSbKoZEy' + }, + // Max Tech - Apple Silicon Macs Explained + // https://www.youtube.com/playlist?list=PLo11Rczpzuj05que94HF80LWD217ToJht + { + id: 'PLo11Rczpzuj05que94HF80LWD217ToJht' + }, + // MrMacRight - Gaming Performance Tests + // https://www.youtube.com/playlist?list=PL9H5Z-IdZ8M0tUdSfS_rmdc8CRR_FD83e + { + id: 'PL9H5Z-IdZ8M0tUdSfS_rmdc8CRR_FD83e' + }, + // Created Labs - New 2020 M1 MacBook + // https://www.youtube.com/playlist?list=PLcbQnQIwz6qSt-qsD3jCJaEw9fK8yXarA + { + id: 'PLcbQnQIwz6qSt-qsD3jCJaEw9fK8yXarA' + }, + // DaVinci Resolve + Apple M1 Tests - Learn Color Grading + // https://www.youtube.com/playlist?list=PLRYMmqUFQ_cfETgJ_9tSHT1eD8c94y-qg + { + id: 'PLRYMmqUFQ_cfETgJ_9tSHT1eD8c94y-qg' + }, + // Apple Silicon Macs — M1 & Beyond! - Rene Ritchie + // https://www.youtube.com/playlist?list=PL3XJJi5sAjD1sCicSKd0irf5TnQ1KJvPm + { + id: 'PL3XJJi5sAjD1sCicSKd0irf5TnQ1KJvPm' + }, + // Apple M1 Silicon Benchmarks - Tonyisgaming + // https://www.youtube.com/playlist?list=PLgz-h_Uy9AwOctqRcm6hEYEqTO9yIawoO + { + id: 'PLgz-h_Uy9AwOctqRcm6hEYEqTO9yIawoO' + }, + // M1 - Jerry Schulze + // https://www.youtube.com/playlist?list=PLFf7t8YdWQOgVmQdiEey2c_7ygDeZGBvf + { + id: 'PLFf7t8YdWQOgVmQdiEey2c_7ygDeZGBvf' + }, + // Apple Silicon - DevChannel + // https://www.youtube.com/playlist?list=PLEi3_qsqIyclc-3NqbEYlIvXEdcLXy1qX + { + id: 'PLEi3_qsqIyclc-3NqbEYlIvXEdcLXy1qX' + }, + // Michael P. Schmidt + // https://www.youtube.com/playlist?list=PLsT75DpPtn2N0RvXGdkjnwAYM0m9EMpb7 + { + id: 'PLsT75DpPtn2N0RvXGdkjnwAYM0m9EMpb7' + }, + // Ben G. Kaiser + // https://www.youtube.com/playlist?list=PL_9qmWdi19yCOCzAdTfFxpZtXonyfWTYJ + { + id: 'PL_9qmWdi19yCOCzAdTfFxpZtXonyfWTYJ' + }, + // Constant Geekery + // https://youtube.com/playlist?list=PLQncOO7KICBIH-1Jv3fhOOU9b3-j6XoED + { + id: 'PLQncOO7KICBIH-1Jv3fhOOU9b3-j6XoED' + }, + // Alexander Ziskind + // https://youtube.com/playlist?list=PLPwbI_iIX3aR88msMh-cHoJiBqS6YMUUH + { + id: 'PLPwbI_iIX3aR88msMh-cHoJiBqS6YMUUH' + }, + // Execute Automation + // https://youtube.com/playlist?list=PL6tu16kXT9Pqwg2H8G3mROh5g7LISl8wT + { + id: 'PL6tu16kXT9Pqwg2H8G3mROh5g7LISl8wT' + }, + // Portland CNC + // https://youtube.com/playlist?list=PLlQPaN85gB1kRnc8RUnV-TEOXU_2iap8S + { + id: 'PLlQPaN85gB1kRnc8RUnV-TEOXU_2iap8S' + }, + // Ben Designs + // https://youtube.com/playlist?list=PLQgB1FZhNI7XAf9-2Gb88o6MxxbqDQIgj + { + id: 'PLQgB1FZhNI7XAf9-2Gb88o6MxxbqDQIgj' + }, + // Ben Aqua + // https://youtube.com/playlist?list=PLvswiYwf1PZR_V1cXgKu0fKqN690LXRal + { + id: 'PLvswiYwf1PZR_V1cXgKu0fKqN690LXRal' + }, + // Tech Gear Talk + // https://youtube.com/playlist?list=PL9Y8hNm43poLPdLAfFikiO_c4ZvpB4Wi8 + { + id: 'PL9Y8hNm43poLPdLAfFikiO_c4ZvpB4Wi8' + }, + // c0pist + // https://youtube.com/playlist?list=PLmEmhiFYCJygSGcPGNRgiJ9T1AKxJjERI + { + id: 'PLmEmhiFYCJygSGcPGNRgiJ9T1AKxJjERI' + }, + // BilValentine + // https://youtube.com/playlist?list=PLj__zPOcS7bkamYx2oe9p6YOn-63Imexd + { + id: 'PLj__zPOcS7bkamYx2oe9p6YOn-63Imexd' + }, + // Techkhamun + // https://youtube.com/playlist?list=PLprAMxVeEzkb4-19ncZ6GNWWL4QMxZIQK + { + id: 'PLprAMxVeEzkb4-19ncZ6GNWWL4QMxZIQK' + }, + // iCave + // https://youtube.com/playlist?list=PLXHx58X9BZxJfSmttXQJv1zmNi0f7ANhq + { + id: 'PLXHx58X9BZxJfSmttXQJv1zmNi0f7ANhq' + }, + // Douglas Hewitt + // https://youtube.com/playlist?list=PLugJ1ygKKMlI4WwyCbdLJOOHUZkFwvYbK + { + id: 'PLugJ1ygKKMlI4WwyCbdLJOOHUZkFwvYbK' + }, + // Painfully Honest Tech + // https://youtube.com/playlist?list=PLWrpcd0vRa5StCaJoGqT4NsmvOPjF-VZu + { + id: 'PLWrpcd0vRa5StCaJoGqT4NsmvOPjF-VZu' + }, + // IrixGuy + // https://youtube.com/playlist?list=PLzbToXWOz_ZiL5rUDKGaTQS5-eeWfMq7T + { + id: 'PLzbToXWOz_ZiL5rUDKGaTQS5-eeWfMq7T' + }, + // sand0m1ze gaming + // https://youtube.com/playlist?list=PLPPMLgyyaMusoENI6yomC6DsYUymf2ey5 + { + id: 'PLPPMLgyyaMusoENI6yomC6DsYUymf2ey5' + }, + // The Dev + // https://youtube.com/playlist?list=PLtOOsLeNlKexKL9yzwp7vpbXaaynmfuQb + { + id: 'PLtOOsLeNlKexKL9yzwp7vpbXaaynmfuQb' + }, + // Luke Barousse - Data Science + // https://youtube.com/playlist?list=PL_CkpxkuPiT9eGORxdWVWn0J58AUki_0W + { + id: 'PL_CkpxkuPiT9eGORxdWVWn0J58AUki_0W' + }, + // AudioMap + // https://youtube.com/playlist?list=PL2xBqm1csOKoBkzu4GCnlHe_KiTGjzlNL + { + id: 'PL2xBqm1csOKoBkzu4GCnlHe_KiTGjzlNL' + }, + // Mark Payne + // https://youtube.com/playlist?list=PL_NaeOyKxj28b_iVaiXfsyPaKwVeht-mG + { + id: 'PL_NaeOyKxj28b_iVaiXfsyPaKwVeht-mG' + }, + // White Sea Studio + // https://youtube.com/playlist?list=PLil8shFjsBGaDDxkXhXPYa937D3LcMJDx + { + id: 'PLil8shFjsBGaDDxkXhXPYa937D3LcMJDx' + }, + // Pete Herro + // https://youtube.com/playlist?list=PLy7gjSbRTE7M8QuYN6Uxp81whm2gAfwK9 + { + id: 'PLy7gjSbRTE7M8QuYN6Uxp81whm2gAfwK9' + }, + + + // My Personal Benchmarks Playlist + // https://www.youtube.com/playlist?list=PLaa9cZC07ZPFqjoYLZRR3kbbnJRHhlmXq + { + id: benchmarksPlaylistId + }, + // My Personal Playlist (For odds and ends) + // https://www.youtube.com/playlist?list=PLaa9cZC07ZPGM1f6A3F72qXNtPbVLl4V7 + { + id: 'PLaa9cZC07ZPGM1f6A3F72qXNtPbVLl4V7' + }, +]