mirror of
https://github.com/ThatGuySam/doesitarm.git
synced 2026-05-15 06:35:20 -07:00
Fetch youtube videos from playlists
This commit is contained in:
Sam Carlton
parent
ffe5b5bbde
commit
976ab06cc3
3 changed files with 361 additions and 1 deletions
|
|
@ -5,8 +5,9 @@ import dotenv from 'dotenv'
|
|||
import semver from 'semver'
|
||||
import { PromisePool } from '@supercharge/promise-pool'
|
||||
import memoize from 'fast-memoize'
|
||||
import has from 'just-has'
|
||||
// import has from 'just-has'
|
||||
|
||||
import { saveYouTubeVideos } from '~/helpers/api/youtube/build.js'
|
||||
import buildAppList from '~/helpers/build-app-list.js'
|
||||
import buildGamesList from '~/helpers/build-game-list.js'
|
||||
import buildHomebrewList from '~/helpers/build-homebrew-list.js'
|
||||
|
|
@ -574,6 +575,8 @@ class BuildLists {
|
|||
|
||||
async build () {
|
||||
|
||||
await saveYouTubeVideos()
|
||||
|
||||
// Pull in and layer data from all sources
|
||||
await this.buildLists()
|
||||
|
||||
|
|
|
|||
172
helpers/api/youtube/build.js
Normal file
172
helpers/api/youtube/build.js
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
import fs from 'fs-extra'
|
||||
import { google } from 'googleapis'
|
||||
|
||||
import { playlists, benchmarksPlaylistId } from './playlists.js'
|
||||
|
||||
|
||||
const youtubeVideoPath = './static/api/youtube-videos.json'
|
||||
|
||||
|
||||
async function getPlaylistsItems ( { playlistId } = {} ) {
|
||||
const perPage = 50
|
||||
|
||||
// Setup Youtube API V3 Service instance
|
||||
const service = google.youtube('v3')
|
||||
|
||||
// Fetch data from the Youtube API
|
||||
const { errors = null, data = null } = await service.playlistItems.list({
|
||||
key: process.env.GOOGLE_API_KEY,
|
||||
part: 'snippet,contentDetails',
|
||||
playlistId,
|
||||
maxResults: perPage
|
||||
}).catch(({ errors }) => {
|
||||
|
||||
console.log('Error fetching playlist', errors)
|
||||
|
||||
return {
|
||||
errors
|
||||
}
|
||||
})
|
||||
|
||||
// Send an error response if something went wrong
|
||||
if (errors !== null) {
|
||||
throw new Error(errors)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
const items = data.items
|
||||
|
||||
// If there are more results then push them to our playlist
|
||||
if (data.nextPageToken !== null) {
|
||||
|
||||
// Store the token for page #2 into our variable
|
||||
let pageToken = data.nextPageToken
|
||||
|
||||
while (pageToken !== null) {
|
||||
// Fetch data from the Youtube API
|
||||
const youtubePageResponse = await service.playlistItems.list({
|
||||
key: process.env.GOOGLE_API_KEY,
|
||||
part: 'snippet,contentDetails',
|
||||
playlistId,
|
||||
maxResults: perPage,
|
||||
pageToken: pageToken
|
||||
})
|
||||
|
||||
// Add the videos from this page on to our total items list
|
||||
youtubePageResponse.data.items.forEach(item => items.push(item))
|
||||
|
||||
// Now that we're done set up the next page token or empty out the pageToken variable so our loop will stop
|
||||
pageToken = ('nextPageToken' in youtubePageResponse.data) ? youtubePageResponse.data.nextPageToken : null
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Fetched ${items.length} videos from https://www.youtube.com/playlist?list=${ playlistId }`)
|
||||
|
||||
return items
|
||||
}
|
||||
|
||||
async function getYouTubeVideos ( options = {} ) {
|
||||
|
||||
const {
|
||||
// requestsDelay = 3600,
|
||||
} = options
|
||||
|
||||
// Fetch all videos from playlists
|
||||
const playlistSets = []
|
||||
|
||||
for ( const playlistToFetch of playlists ) {
|
||||
|
||||
// console.log('playlistJsonUrl', playlistJsonUrl)
|
||||
|
||||
const playlistItems = await getPlaylistsItems({
|
||||
playlistId: playlistToFetch.id
|
||||
})
|
||||
// console.log('playlistItems', playlistItems.length)
|
||||
|
||||
playlistSets.push( playlistItems )
|
||||
}
|
||||
|
||||
// Pull benchmarksPlaylist out of playlist sets
|
||||
// benchmarksPlaylistId
|
||||
const benchmarksVideoIds = playlistSets.find( playlist => {
|
||||
// Skip empty playlists
|
||||
if (playlist.length === 0) return false
|
||||
|
||||
// Get this playlist's ID from first video
|
||||
// and check against benchmarksPlaylistId
|
||||
return playlist[0].snippet.playlistId === benchmarksPlaylistId
|
||||
}).map( video => video.contentDetails.videoId)
|
||||
|
||||
// Creat an object to store playlist items
|
||||
const playlistItems = {}
|
||||
|
||||
|
||||
// Loop through the sets and store all the videos into a single array
|
||||
for (const playlistSet of playlistSets) {
|
||||
for (const playlistItem of playlistSet) {
|
||||
// If we've already stored this video
|
||||
// then skip
|
||||
if (playlistItems.hasOwnProperty(playlistItem.contentDetails.videoId)) continue
|
||||
|
||||
const tags = []
|
||||
|
||||
// If this video is in the benchmarks playlist
|
||||
// then add the benchmark tag
|
||||
if (benchmarksVideoIds.includes(playlistItem.contentDetails.videoId)) {
|
||||
tags.push('benchmark')
|
||||
}
|
||||
|
||||
// Store newly found video
|
||||
playlistItems[playlistItem.contentDetails.videoId] = {
|
||||
title: playlistItem.snippet.title,
|
||||
description: playlistItem.snippet.description,
|
||||
timestamps: [],
|
||||
rawData: playlistItem,
|
||||
tags
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Loop through playlist items and store timestamp data
|
||||
for (const videoId in playlistItems) {
|
||||
// console.log('playlistItem', playlistItem)
|
||||
// If the description is empty
|
||||
// then skip
|
||||
if (playlistItems[videoId].description.trim().length === 0) continue
|
||||
|
||||
// Break up the description by line breaks
|
||||
const descriptionLines = playlistItems[videoId].description.split(/\r?\n/)
|
||||
|
||||
// console.log('descriptionLines', descriptionLines)
|
||||
|
||||
for (const line of descriptionLines) {
|
||||
// https://stackoverflow.com/a/11067610/1397641
|
||||
const matches = line.match(/(?:([0-5]?[0-9]):)?([0-5]?[0-9]):([0-5][0-9])/)
|
||||
|
||||
// If there are no timestamps on this line
|
||||
// then skip
|
||||
if (matches === null) continue
|
||||
|
||||
playlistItems[videoId].timestamps.push({
|
||||
time: matches[0],
|
||||
fullText: line
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return playlistItems
|
||||
}
|
||||
|
||||
|
||||
export async function saveYouTubeVideos () {
|
||||
//
|
||||
const youtubeVideos = await getYouTubeVideos()
|
||||
|
||||
//
|
||||
|
||||
// Save to JSON
|
||||
await fs.outputJson( youtubeVideoPath, youtubeVideos )
|
||||
|
||||
}
|
||||
185
helpers/api/youtube/playlists.js
Normal file
185
helpers/api/youtube/playlists.js
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
|
||||
|
||||
export const benchmarksPlaylistId = 'PLaa9cZC07ZPFqjoYLZRR3kbbnJRHhlmXq'
|
||||
|
||||
|
||||
|
||||
export const playlists = [
|
||||
// Awais Mirza - Apple Silicon Mac Software Testing
|
||||
// https://www.youtube.com/playlist?list=PLz5rnvLVJX5XF8cXAOQuarPIeP8Xr7b1_
|
||||
{
|
||||
id: 'PLz5rnvLVJX5XF8cXAOQuarPIeP8Xr7b1_'
|
||||
},
|
||||
// Andrew Tsai - M1 Apple Silicon Game Benchmarks
|
||||
// https://www.youtube.com/playlist?list=PLFbqxkNlqrHNK0i4WN99Jc8g-qSbKoZEy
|
||||
{
|
||||
id: 'PLFbqxkNlqrHNK0i4WN99Jc8g-qSbKoZEy'
|
||||
},
|
||||
// Max Tech - Apple Silicon Macs Explained
|
||||
// https://www.youtube.com/playlist?list=PLo11Rczpzuj05que94HF80LWD217ToJht
|
||||
{
|
||||
id: 'PLo11Rczpzuj05que94HF80LWD217ToJht'
|
||||
},
|
||||
// MrMacRight - Gaming Performance Tests
|
||||
// https://www.youtube.com/playlist?list=PL9H5Z-IdZ8M0tUdSfS_rmdc8CRR_FD83e
|
||||
{
|
||||
id: 'PL9H5Z-IdZ8M0tUdSfS_rmdc8CRR_FD83e'
|
||||
},
|
||||
// Created Labs - New 2020 M1 MacBook
|
||||
// https://www.youtube.com/playlist?list=PLcbQnQIwz6qSt-qsD3jCJaEw9fK8yXarA
|
||||
{
|
||||
id: 'PLcbQnQIwz6qSt-qsD3jCJaEw9fK8yXarA'
|
||||
},
|
||||
// DaVinci Resolve + Apple M1 Tests - Learn Color Grading
|
||||
// https://www.youtube.com/playlist?list=PLRYMmqUFQ_cfETgJ_9tSHT1eD8c94y-qg
|
||||
{
|
||||
id: 'PLRYMmqUFQ_cfETgJ_9tSHT1eD8c94y-qg'
|
||||
},
|
||||
// Apple Silicon Macs — M1 & Beyond! - Rene Ritchie
|
||||
// https://www.youtube.com/playlist?list=PL3XJJi5sAjD1sCicSKd0irf5TnQ1KJvPm
|
||||
{
|
||||
id: 'PL3XJJi5sAjD1sCicSKd0irf5TnQ1KJvPm'
|
||||
},
|
||||
// Apple M1 Silicon Benchmarks - Tonyisgaming
|
||||
// https://www.youtube.com/playlist?list=PLgz-h_Uy9AwOctqRcm6hEYEqTO9yIawoO
|
||||
{
|
||||
id: 'PLgz-h_Uy9AwOctqRcm6hEYEqTO9yIawoO'
|
||||
},
|
||||
// M1 - Jerry Schulze
|
||||
// https://www.youtube.com/playlist?list=PLFf7t8YdWQOgVmQdiEey2c_7ygDeZGBvf
|
||||
{
|
||||
id: 'PLFf7t8YdWQOgVmQdiEey2c_7ygDeZGBvf'
|
||||
},
|
||||
// Apple Silicon - DevChannel
|
||||
// https://www.youtube.com/playlist?list=PLEi3_qsqIyclc-3NqbEYlIvXEdcLXy1qX
|
||||
{
|
||||
id: 'PLEi3_qsqIyclc-3NqbEYlIvXEdcLXy1qX'
|
||||
},
|
||||
// Michael P. Schmidt
|
||||
// https://www.youtube.com/playlist?list=PLsT75DpPtn2N0RvXGdkjnwAYM0m9EMpb7
|
||||
{
|
||||
id: 'PLsT75DpPtn2N0RvXGdkjnwAYM0m9EMpb7'
|
||||
},
|
||||
// Ben G. Kaiser
|
||||
// https://www.youtube.com/playlist?list=PL_9qmWdi19yCOCzAdTfFxpZtXonyfWTYJ
|
||||
{
|
||||
id: 'PL_9qmWdi19yCOCzAdTfFxpZtXonyfWTYJ'
|
||||
},
|
||||
// Constant Geekery
|
||||
// https://youtube.com/playlist?list=PLQncOO7KICBIH-1Jv3fhOOU9b3-j6XoED
|
||||
{
|
||||
id: 'PLQncOO7KICBIH-1Jv3fhOOU9b3-j6XoED'
|
||||
},
|
||||
// Alexander Ziskind
|
||||
// https://youtube.com/playlist?list=PLPwbI_iIX3aR88msMh-cHoJiBqS6YMUUH
|
||||
{
|
||||
id: 'PLPwbI_iIX3aR88msMh-cHoJiBqS6YMUUH'
|
||||
},
|
||||
// Execute Automation
|
||||
// https://youtube.com/playlist?list=PL6tu16kXT9Pqwg2H8G3mROh5g7LISl8wT
|
||||
{
|
||||
id: 'PL6tu16kXT9Pqwg2H8G3mROh5g7LISl8wT'
|
||||
},
|
||||
// Portland CNC
|
||||
// https://youtube.com/playlist?list=PLlQPaN85gB1kRnc8RUnV-TEOXU_2iap8S
|
||||
{
|
||||
id: 'PLlQPaN85gB1kRnc8RUnV-TEOXU_2iap8S'
|
||||
},
|
||||
// Ben Designs
|
||||
// https://youtube.com/playlist?list=PLQgB1FZhNI7XAf9-2Gb88o6MxxbqDQIgj
|
||||
{
|
||||
id: 'PLQgB1FZhNI7XAf9-2Gb88o6MxxbqDQIgj'
|
||||
},
|
||||
// Ben Aqua
|
||||
// https://youtube.com/playlist?list=PLvswiYwf1PZR_V1cXgKu0fKqN690LXRal
|
||||
{
|
||||
id: 'PLvswiYwf1PZR_V1cXgKu0fKqN690LXRal'
|
||||
},
|
||||
// Tech Gear Talk
|
||||
// https://youtube.com/playlist?list=PL9Y8hNm43poLPdLAfFikiO_c4ZvpB4Wi8
|
||||
{
|
||||
id: 'PL9Y8hNm43poLPdLAfFikiO_c4ZvpB4Wi8'
|
||||
},
|
||||
// c0pist
|
||||
// https://youtube.com/playlist?list=PLmEmhiFYCJygSGcPGNRgiJ9T1AKxJjERI
|
||||
{
|
||||
id: 'PLmEmhiFYCJygSGcPGNRgiJ9T1AKxJjERI'
|
||||
},
|
||||
// BilValentine
|
||||
// https://youtube.com/playlist?list=PLj__zPOcS7bkamYx2oe9p6YOn-63Imexd
|
||||
{
|
||||
id: 'PLj__zPOcS7bkamYx2oe9p6YOn-63Imexd'
|
||||
},
|
||||
// Techkhamun
|
||||
// https://youtube.com/playlist?list=PLprAMxVeEzkb4-19ncZ6GNWWL4QMxZIQK
|
||||
{
|
||||
id: 'PLprAMxVeEzkb4-19ncZ6GNWWL4QMxZIQK'
|
||||
},
|
||||
// iCave
|
||||
// https://youtube.com/playlist?list=PLXHx58X9BZxJfSmttXQJv1zmNi0f7ANhq
|
||||
{
|
||||
id: 'PLXHx58X9BZxJfSmttXQJv1zmNi0f7ANhq'
|
||||
},
|
||||
// Douglas Hewitt
|
||||
// https://youtube.com/playlist?list=PLugJ1ygKKMlI4WwyCbdLJOOHUZkFwvYbK
|
||||
{
|
||||
id: 'PLugJ1ygKKMlI4WwyCbdLJOOHUZkFwvYbK'
|
||||
},
|
||||
// Painfully Honest Tech
|
||||
// https://youtube.com/playlist?list=PLWrpcd0vRa5StCaJoGqT4NsmvOPjF-VZu
|
||||
{
|
||||
id: 'PLWrpcd0vRa5StCaJoGqT4NsmvOPjF-VZu'
|
||||
},
|
||||
// IrixGuy
|
||||
// https://youtube.com/playlist?list=PLzbToXWOz_ZiL5rUDKGaTQS5-eeWfMq7T
|
||||
{
|
||||
id: 'PLzbToXWOz_ZiL5rUDKGaTQS5-eeWfMq7T'
|
||||
},
|
||||
// sand0m1ze gaming
|
||||
// https://youtube.com/playlist?list=PLPPMLgyyaMusoENI6yomC6DsYUymf2ey5
|
||||
{
|
||||
id: 'PLPPMLgyyaMusoENI6yomC6DsYUymf2ey5'
|
||||
},
|
||||
// The Dev
|
||||
// https://youtube.com/playlist?list=PLtOOsLeNlKexKL9yzwp7vpbXaaynmfuQb
|
||||
{
|
||||
id: 'PLtOOsLeNlKexKL9yzwp7vpbXaaynmfuQb'
|
||||
},
|
||||
// Luke Barousse - Data Science
|
||||
// https://youtube.com/playlist?list=PL_CkpxkuPiT9eGORxdWVWn0J58AUki_0W
|
||||
{
|
||||
id: 'PL_CkpxkuPiT9eGORxdWVWn0J58AUki_0W'
|
||||
},
|
||||
// AudioMap
|
||||
// https://youtube.com/playlist?list=PL2xBqm1csOKoBkzu4GCnlHe_KiTGjzlNL
|
||||
{
|
||||
id: 'PL2xBqm1csOKoBkzu4GCnlHe_KiTGjzlNL'
|
||||
},
|
||||
// Mark Payne
|
||||
// https://youtube.com/playlist?list=PL_NaeOyKxj28b_iVaiXfsyPaKwVeht-mG
|
||||
{
|
||||
id: 'PL_NaeOyKxj28b_iVaiXfsyPaKwVeht-mG'
|
||||
},
|
||||
// White Sea Studio
|
||||
// https://youtube.com/playlist?list=PLil8shFjsBGaDDxkXhXPYa937D3LcMJDx
|
||||
{
|
||||
id: 'PLil8shFjsBGaDDxkXhXPYa937D3LcMJDx'
|
||||
},
|
||||
// Pete Herro
|
||||
// https://youtube.com/playlist?list=PLy7gjSbRTE7M8QuYN6Uxp81whm2gAfwK9
|
||||
{
|
||||
id: 'PLy7gjSbRTE7M8QuYN6Uxp81whm2gAfwK9'
|
||||
},
|
||||
|
||||
|
||||
// My Personal Benchmarks Playlist
|
||||
// https://www.youtube.com/playlist?list=PLaa9cZC07ZPFqjoYLZRR3kbbnJRHhlmXq
|
||||
{
|
||||
id: benchmarksPlaylistId
|
||||
},
|
||||
// My Personal Playlist (For odds and ends)
|
||||
// https://www.youtube.com/playlist?list=PLaa9cZC07ZPGM1f6A3F72qXNtPbVLl4V7
|
||||
{
|
||||
id: 'PLaa9cZC07ZPGM1f6A3F72qXNtPbVLl4V7'
|
||||
},
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue