mirror of
https://github.com/ThatGuySam/doesitarm.git
synced 2026-05-18 06:44:46 -07:00
Test for non-alpanumeric characters in readme
This commit is contained in:
Sam Carlton
parent
b7471132c1
commit
4cbde7c8f5
3 changed files with 257 additions and 157 deletions
|
|
@ -16,6 +16,27 @@ import { makeSlug } from './slug.js'
|
||||||
|
|
||||||
const md = new MarkdownIt()
|
const md = new MarkdownIt()
|
||||||
|
|
||||||
|
// Find the end of our list
|
||||||
|
export function getEndOfListTokenIndex ( parsedMarkdown ) {
|
||||||
|
|
||||||
|
return parsedMarkdown.findIndex((Token) => {
|
||||||
|
// JSON.stringify(Token).includes('end-of-list')
|
||||||
|
const matches = Token.content.includes('end-of-list')
|
||||||
|
|
||||||
|
// if (matches) {
|
||||||
|
// console.log('Token', Token)
|
||||||
|
// }
|
||||||
|
|
||||||
|
return matches
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getReadmeTokenList ( parsedMarkdown ) {
|
||||||
|
const endOfListIndex = getEndOfListTokenIndex( parsedMarkdown )
|
||||||
|
|
||||||
|
return parsedMarkdown.slice(0, endOfListIndex)
|
||||||
|
}
|
||||||
|
|
||||||
const getTokenLinks = function ( childTokens ) {
|
const getTokenLinks = function ( childTokens ) {
|
||||||
|
|
||||||
const tokenList = []
|
const tokenList = []
|
||||||
|
|
@ -56,6 +77,160 @@ const getTokenLinks = function ( childTokens ) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export function buildReadmeAppList ({ readmeContent, scanListMap, commits }) {
|
||||||
|
|
||||||
|
// Parse markdown
|
||||||
|
const result = md.parse(readmeContent)
|
||||||
|
|
||||||
|
// console.log('results', result.length)
|
||||||
|
// console.log('results', result)
|
||||||
|
|
||||||
|
|
||||||
|
// Find the end of our list
|
||||||
|
// const endOfListIndex = getEndOfListIndex( result )
|
||||||
|
|
||||||
|
const appListTokens = getReadmeTokenList( result )
|
||||||
|
|
||||||
|
const appList = []
|
||||||
|
|
||||||
|
let categorySlug = 'start'
|
||||||
|
let categoryTitle = 'Start'
|
||||||
|
let isHeading = false
|
||||||
|
let isParagraph = false
|
||||||
|
|
||||||
|
for (const token of appListTokens) {
|
||||||
|
// On heading close switch off heading mode
|
||||||
|
if (token.type.includes('heading_')) isHeading = !isHeading
|
||||||
|
|
||||||
|
// On heading close switch off heading mode
|
||||||
|
if (token.type.includes('paragraph_')) isParagraph = !isParagraph
|
||||||
|
|
||||||
|
if (isHeading && token.type === 'inline') {
|
||||||
|
categoryTitle = token.content
|
||||||
|
categorySlug = makeSlug( token.content )
|
||||||
|
|
||||||
|
// appList[categorySlug] = []
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if ( isParagraph && token.type === 'inline' && token.content.includes(' - ') ) {
|
||||||
|
|
||||||
|
const [ link, text ] = token.content.split(' - ').map(string => string.trim())
|
||||||
|
|
||||||
|
const [ name, url ] = link.substring(1, link.length-1).split('](')
|
||||||
|
|
||||||
|
const bundleIds = []
|
||||||
|
let tags = []
|
||||||
|
let aliases = []
|
||||||
|
const relatedLinksMap = new Map( getTokenLinks(token.children).map( link => [ link.href, link ] ) )
|
||||||
|
|
||||||
|
// Search for this app in the scanList and remove duplicates
|
||||||
|
scanListMap.forEach( ( scannedApp, key ) => {
|
||||||
|
|
||||||
|
for ( const alias of scannedApp.aliases ) {
|
||||||
|
// console.log( key, alias, name, eitherMatches(alias, name) )
|
||||||
|
|
||||||
|
if ( eitherMatches(alias, name) ) {
|
||||||
|
// If we don't have any bundleIds yet
|
||||||
|
// Add this app's bundleId to the list
|
||||||
|
if ( !bundleIds.includes( scannedApp.bundleIds[0] ) ) { bundleIds.push(scannedApp.bundleIds[0]) }
|
||||||
|
|
||||||
|
// Merge this scanned app's tags into the matching app
|
||||||
|
tags = Array.from(new Set([
|
||||||
|
...tags,
|
||||||
|
...scannedApp.tags
|
||||||
|
]))
|
||||||
|
|
||||||
|
// Merge as set then convert to array to prevent duplicates
|
||||||
|
aliases = Array.from(new Set([
|
||||||
|
...aliases,
|
||||||
|
...scannedApp.aliases
|
||||||
|
]))
|
||||||
|
|
||||||
|
// Merge relatated links
|
||||||
|
for ( const link of scannedApp.relatedLinks ) {
|
||||||
|
|
||||||
|
relatedLinksMap.set( link.href, {
|
||||||
|
...link,
|
||||||
|
label: (link.label === 'View') ? 'App Website' : link.label
|
||||||
|
} )
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Merged ${alias} (${scannedApp.bundleIds[0]}) from scanned apps into ${name} from README`)
|
||||||
|
scanListMap.delete( key )
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
// Convert link map values into array for JSON
|
||||||
|
const relatedLinks = Array.from( relatedLinksMap.values() )
|
||||||
|
|
||||||
|
// console.log('relatedLinks', relatedLinks)
|
||||||
|
|
||||||
|
const appSlug = makeSlug( name )
|
||||||
|
|
||||||
|
const endpoint = getAppEndpoint({
|
||||||
|
category: {
|
||||||
|
slug: null
|
||||||
|
},
|
||||||
|
slug: appSlug
|
||||||
|
})// `/app/${appSlug}`
|
||||||
|
|
||||||
|
let status = 'unknown'
|
||||||
|
|
||||||
|
for (const statusKey in statuses) {
|
||||||
|
if (text.includes(statusKey)) {
|
||||||
|
status = statuses[statusKey]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const category = {
|
||||||
|
label: categoryTitle,
|
||||||
|
slug: categorySlug
|
||||||
|
}
|
||||||
|
|
||||||
|
const lastUpdatedRaw = lookForLastUpdated({ name, slug: appSlug, endpoint, category }, commits)
|
||||||
|
|
||||||
|
const lastUpdated = (lastUpdatedRaw) ? {
|
||||||
|
raw: lastUpdatedRaw,
|
||||||
|
timestamp: parseDate(lastUpdatedRaw).timestamp,
|
||||||
|
} : null
|
||||||
|
|
||||||
|
|
||||||
|
appList.push({
|
||||||
|
name,
|
||||||
|
aliases,
|
||||||
|
status,
|
||||||
|
bundleIds,
|
||||||
|
lastUpdated,
|
||||||
|
// url,
|
||||||
|
text,
|
||||||
|
slug: appSlug,
|
||||||
|
endpoint,
|
||||||
|
category,
|
||||||
|
tags,
|
||||||
|
// content: token.content,
|
||||||
|
relatedLinks,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
// if ( tags.length > 1 ) {
|
||||||
|
// console.log('tags', name, bundleIds, tags)
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
|
||||||
|
// appList[categorySlug]
|
||||||
|
|
||||||
|
|
||||||
|
// console.log('token', token)
|
||||||
|
}
|
||||||
|
|
||||||
|
return appList
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
const lookForLastUpdated = function (app, commits) {
|
const lookForLastUpdated = function (app, commits) {
|
||||||
|
|
||||||
for (const { node: commit } of commits) {
|
for (const { node: commit } of commits) {
|
||||||
|
|
@ -255,162 +430,7 @@ export default async function () {
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
// Parse markdown
|
const appList = buildReadmeAppList({ readmeContent, scanListMap, commits })
|
||||||
const result = md.parse(readmeContent)
|
|
||||||
|
|
||||||
// console.log('results', result.length)
|
|
||||||
// console.log('results', result)
|
|
||||||
|
|
||||||
|
|
||||||
// Finf the end of our list
|
|
||||||
const endOfListIndex = result.findIndex((Token) => {
|
|
||||||
// JSON.stringify(Token).includes('end-of-list')
|
|
||||||
const matches = Token.content.includes('end-of-list')
|
|
||||||
|
|
||||||
// if (matches) {
|
|
||||||
// console.log('Token', Token)
|
|
||||||
// }
|
|
||||||
|
|
||||||
return matches
|
|
||||||
})
|
|
||||||
|
|
||||||
const appListTokens = result.slice(0, endOfListIndex)
|
|
||||||
|
|
||||||
const appList = []
|
|
||||||
|
|
||||||
let categorySlug = 'start'
|
|
||||||
let categoryTitle = 'Start'
|
|
||||||
let isHeading = false
|
|
||||||
let isParagraph = false
|
|
||||||
|
|
||||||
for (const token of appListTokens) {
|
|
||||||
// On heading close switch off heading mode
|
|
||||||
if (token.type.includes('heading_')) isHeading = !isHeading
|
|
||||||
|
|
||||||
// On heading close switch off heading mode
|
|
||||||
if (token.type.includes('paragraph_')) isParagraph = !isParagraph
|
|
||||||
|
|
||||||
if (isHeading && token.type === 'inline') {
|
|
||||||
categoryTitle = token.content
|
|
||||||
categorySlug = makeSlug( token.content )
|
|
||||||
|
|
||||||
// appList[categorySlug] = []
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
if ( isParagraph && token.type === 'inline' && token.content.includes(' - ') ) {
|
|
||||||
|
|
||||||
const [ link, text ] = token.content.split(' - ').map(string => string.trim())
|
|
||||||
|
|
||||||
const [ name, url ] = link.substring(1, link.length-1).split('](')
|
|
||||||
|
|
||||||
const bundleIds = []
|
|
||||||
let tags = []
|
|
||||||
let aliases = []
|
|
||||||
const relatedLinksMap = new Map( getTokenLinks(token.children).map( link => [ link.href, link ] ) )
|
|
||||||
|
|
||||||
// Search for this app in the scanList and remove duplicates
|
|
||||||
scanListMap.forEach( ( scannedApp, key ) => {
|
|
||||||
|
|
||||||
for ( const alias of scannedApp.aliases ) {
|
|
||||||
// console.log( key, alias, name, eitherMatches(alias, name) )
|
|
||||||
|
|
||||||
if ( eitherMatches(alias, name) ) {
|
|
||||||
// If we don't have any bundleIds yet
|
|
||||||
// Add this app's bundleId to the list
|
|
||||||
if ( !bundleIds.includes( scannedApp.bundleIds[0] ) ) { bundleIds.push(scannedApp.bundleIds[0]) }
|
|
||||||
|
|
||||||
// Merge this scanned app's tags into the matching app
|
|
||||||
tags = Array.from(new Set([
|
|
||||||
...tags,
|
|
||||||
...scannedApp.tags
|
|
||||||
]))
|
|
||||||
|
|
||||||
// Merge as set then convert to array to prevent duplicates
|
|
||||||
aliases = Array.from(new Set([
|
|
||||||
...aliases,
|
|
||||||
...scannedApp.aliases
|
|
||||||
]))
|
|
||||||
|
|
||||||
// Merge relatated links
|
|
||||||
for ( const link of scannedApp.relatedLinks ) {
|
|
||||||
|
|
||||||
relatedLinksMap.set( link.href, {
|
|
||||||
...link,
|
|
||||||
label: (link.label === 'View') ? 'App Website' : link.label
|
|
||||||
} )
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Merged ${alias} (${scannedApp.bundleIds[0]}) from scanned apps into ${name} from README`)
|
|
||||||
scanListMap.delete( key )
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
// Convert link map values into array for JSON
|
|
||||||
const relatedLinks = Array.from( relatedLinksMap.values() )
|
|
||||||
|
|
||||||
// console.log('relatedLinks', relatedLinks)
|
|
||||||
|
|
||||||
const appSlug = makeSlug( name )
|
|
||||||
|
|
||||||
const endpoint = getAppEndpoint({
|
|
||||||
category: {
|
|
||||||
slug: null
|
|
||||||
},
|
|
||||||
slug: appSlug
|
|
||||||
})// `/app/${appSlug}`
|
|
||||||
|
|
||||||
let status = 'unknown'
|
|
||||||
|
|
||||||
for (const statusKey in statuses) {
|
|
||||||
if (text.includes(statusKey)) {
|
|
||||||
status = statuses[statusKey]
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const category = {
|
|
||||||
label: categoryTitle,
|
|
||||||
slug: categorySlug
|
|
||||||
}
|
|
||||||
|
|
||||||
const lastUpdatedRaw = lookForLastUpdated({ name, slug: appSlug, endpoint, category }, commits)
|
|
||||||
|
|
||||||
const lastUpdated = (lastUpdatedRaw) ? {
|
|
||||||
raw: lastUpdatedRaw,
|
|
||||||
timestamp: parseDate(lastUpdatedRaw).timestamp,
|
|
||||||
} : null
|
|
||||||
|
|
||||||
|
|
||||||
appList.push({
|
|
||||||
name,
|
|
||||||
aliases,
|
|
||||||
status,
|
|
||||||
bundleIds,
|
|
||||||
lastUpdated,
|
|
||||||
// url,
|
|
||||||
text,
|
|
||||||
slug: appSlug,
|
|
||||||
endpoint,
|
|
||||||
category,
|
|
||||||
tags,
|
|
||||||
// content: token.content,
|
|
||||||
relatedLinks,
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
// if ( tags.length > 1 ) {
|
|
||||||
// console.log('tags', name, bundleIds, tags)
|
|
||||||
// }
|
|
||||||
}
|
|
||||||
|
|
||||||
// appList[categorySlug]
|
|
||||||
|
|
||||||
|
|
||||||
// console.log('token', token)
|
|
||||||
}
|
|
||||||
|
|
||||||
// console.log('appList', appList)
|
// console.log('appList', appList)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,13 +10,14 @@
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
"test-prebuild": "ava ./test/prebuild.js --verbose",
|
||||||
"test": "ava --timeout=1m --verbose",
|
"test": "ava --timeout=1m --verbose",
|
||||||
"dev": "nuxt",
|
"dev": "nuxt",
|
||||||
"build": "nuxt build",
|
"build": "nuxt build",
|
||||||
"start": "nuxt start",
|
"start": "nuxt start",
|
||||||
"generate-dev": "npm run generate && npm test",
|
"generate-dev": "npm run generate && npm test",
|
||||||
"generate": "npm run clone-readme && npm run build-lists && npm run generate-nuxt && npm run generate-eleventy",
|
"generate": "npm run clone-readme && npm run build-lists && npm run generate-nuxt && npm run generate-eleventy",
|
||||||
"build-lists": "node -r esm build-lists.js",
|
"build-lists": "npm run test-prebuild && node -r esm build-lists.js",
|
||||||
"generate-nuxt": "NODE_OPTIONS=--max-old-space-size=60000 nuxt generate",
|
"generate-nuxt": "NODE_OPTIONS=--max-old-space-size=60000 nuxt generate",
|
||||||
"generate-eleventy": "node --max-old-space-size=60000 -r esm node_modules/.bin/eleventy --quiet",
|
"generate-eleventy": "node --max-old-space-size=60000 -r esm node_modules/.bin/eleventy --quiet",
|
||||||
"generate-postcss": "ENV=production postcss assets/css/tailwind.css --o static/tailwind.css",
|
"generate-postcss": "ENV=production postcss assets/css/tailwind.css --o static/tailwind.css",
|
||||||
|
|
|
||||||
79
test/prebuild.js
Normal file
79
test/prebuild.js
Normal file
|
|
@ -0,0 +1,79 @@
|
||||||
|
import fs from 'fs-extra'
|
||||||
|
import test from 'ava'
|
||||||
|
// import MarkdownIt from 'markdown-it'
|
||||||
|
|
||||||
|
import { buildReadmeAppList } from '../helpers/build-app-list.js'
|
||||||
|
|
||||||
|
|
||||||
|
require('dotenv').config()
|
||||||
|
|
||||||
|
// const md = new MarkdownIt()
|
||||||
|
|
||||||
|
const allowedTitleCharacters = new Set( 'ABCDEFGHIJKMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890 -.'.split('') )
|
||||||
|
|
||||||
|
|
||||||
|
function isString( maybeString ) {
|
||||||
|
return (typeof maybeString === 'string' || maybeString instanceof String)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
test.before(async t => {
|
||||||
|
const readmeFileContent = await fs.readFile('./README.md', 'utf-8')
|
||||||
|
// const readmeMarkdown = md.parse( readmeFileContent )
|
||||||
|
|
||||||
|
// t.log( 'readmeMarkdown', readmeMarkdown )
|
||||||
|
|
||||||
|
|
||||||
|
// Store sitemap urls to context
|
||||||
|
t.context.readmeFileContent = readmeFileContent
|
||||||
|
})
|
||||||
|
|
||||||
|
test('README App Titles are alphanumeric only', (t) => {
|
||||||
|
// console.log('t.context.sitemapUrls', t.context.sitemapUrls)
|
||||||
|
|
||||||
|
const readmeAppList = buildReadmeAppList({
|
||||||
|
readmeContent: t.context.readmeFileContent,
|
||||||
|
scanListMap: new Map(),
|
||||||
|
commits: []
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
// console.log('readmeAppList', readmeAppList)
|
||||||
|
t.log('readmeAppList', readmeAppList.length)
|
||||||
|
|
||||||
|
|
||||||
|
for (const readmeApp of readmeAppList) {
|
||||||
|
const cleanedAppName = readmeApp.name//.toLowerCase()
|
||||||
|
|
||||||
|
// const firstInvisbleCharacterIndex = cleanedAppName.search( invisibleCharacerRegex )
|
||||||
|
|
||||||
|
// const standardSpaceCharCode = 32
|
||||||
|
|
||||||
|
for ( const character of cleanedAppName ) {
|
||||||
|
if ( !allowedTitleCharacters.has( character ) ) {
|
||||||
|
|
||||||
|
// badCharacter = readmeApp.name[firstBadCharacterIndex]
|
||||||
|
|
||||||
|
// t.log( readmeApp )
|
||||||
|
t.fail(`README App Title ${readmeApp.name} has non-alphanumeric character ${character}(charCode ${character.charCodeAt(0)})`)
|
||||||
|
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// if ( readmeApp.name.includes('Apple Trans') ) {
|
||||||
|
// const normalNameLength = 'Apple Transporter'.length
|
||||||
|
// t.log( 'normalNameLength', normalNameLength )
|
||||||
|
// t.log( readmeApp.name, readmeApp.name.length )
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
|
||||||
|
// const urlsWithDoubleSlashes = t.context.sitemapUrls.filter( url => url.pathname.includes('//') )
|
||||||
|
|
||||||
|
// if ( urlsWithDoubleSlashes.length > 0) {
|
||||||
|
// t.fail( `${ urlsWithDoubleSlashes.length } urls with doubles slashes found including ${ urlsWithDoubleSlashes[0] }` )
|
||||||
|
// }
|
||||||
|
|
||||||
|
t.log( `${readmeAppList.length} valid alpanumeric app titles in readme` )
|
||||||
|
t.pass()
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue