diff --git a/package.json b/package.json index 28f1a96..735f5a5 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "stork-index": "run-s setup-stork build-stork-index", "download-stork-toml": "node -r esm -r tsconfig-paths/register scripts/download-stork-toml.js", "download-stork-executable": "node -r esm -r tsconfig-paths/register scripts/download-stork-executable.js", + "download-sitemaps": "node -r esm -r tsconfig-paths/register scripts/download-sitemaps.js", "stork-netlify": "chmod +x scripts/stork-netlify.sh && ./scripts/stork-netlify.sh", "dev-astro": "astro dev", "generate-astro": "astro build --experimental-ssr", @@ -61,7 +62,7 @@ "clone-readme": "cp ./README.md README-temp.md", "cloudflare-deploy": "npm run build-api", "vercel-build": "run-s build-lists-and-api test-postbuild-api", - "netlify-build": "run-s test-prebuild-functions stork-index generate-astro" + "netlify-build": "run-s test-prebuild-functions download-sitemaps stork-index generate-astro" }, "dependencies": { "@11ty/eleventy-assets": "^1.0.5", diff --git a/scripts/download-sitemaps.js b/scripts/download-sitemaps.js new file mode 100644 index 0000000..523231e --- /dev/null +++ b/scripts/download-sitemaps.js @@ -0,0 +1,59 @@ +import fs from 'fs-extra' +import 'dotenv/config' +import axios from 'axios' +import { parse } from 'fast-xml-parser' + +import { + sitemapLocation, + sitemapIndexFileName, +} from '~/helpers/constants.js' + + + +;(async () => { + + // Build Sitemap Index URL + const sitemapIndexUrl = new URL( `${ sitemapLocation.split('static')[1] }${ sitemapIndexFileName }`, process.env.PUBLIC_API_DOMAIN ) + + // Fetch Sitemap Index + const sitemapIndexXML = await axios.get( sitemapIndexUrl.href ).then( response => response.data ) + + // Save Sitemap Index + const sitemapIndexFilePath = `${ sitemapLocation }${ sitemapIndexFileName }` + await fs.writeFile( sitemapIndexFilePath, sitemapIndexXML ) + + // Get URLs from index + const { sitemapindex } = parse( sitemapIndexXML ) + + const { + sitemap + } = sitemapindex + + const urlEntries = Array.isArray( sitemap ) ? sitemap : [ sitemap ] + + + // Fetch each sitemap + for ( const entry of urlEntries ) { + + // Build Sitemap Index URL + const sitemapUrl = new URL( entry.loc ) + const apiSitemapUrl = new URL( sitemapUrl.pathname, process.env.PUBLIC_API_DOMAIN ) + + // sitemapUrl.origin = process.env.PUBLIC_API_DOMAIN + + // Fetch Sitemap Index + const sitemapXML = await axios.get( apiSitemapUrl.href ).then( response => response.data ) + + console.log( 'apiSitemapUrl', apiSitemapUrl ) + + + const sitemapFileName = apiSitemapUrl.pathname.split('/')[1] + const sitemapIndexFilePath = `${ sitemapLocation }${ sitemapFileName }` + + // Save file + await fs.writeFile( sitemapIndexFilePath, sitemapXML ) + } + + + process.exit() +})()