fix: Remediation to deal with duplicates and removed repositories in source data

Author: SensibleWood

.gitignore

@@ -3,3 +3,4 @@ coverage/
 node_modules/
 .nyc_output/
 .vscode/
+/.env

docs/CNAME

@@ -0,0 +1 @@
+tools.openapis.org

gulpfile.js/index.js

@@ -3,6 +3,12 @@ const { src, dest } = require('gulp');
 const transform = require('gulp-transform');
 const rename = require('gulp-rename');
 
+const { argv } = require('yargs')
+  .option('metadata', {
+    describe: 'Input file that describes build metadata',
+    default: 'gulpfile.js/metadata.json',
+  });
+
 const {
   classifyTools,
   validateMetadata,
@@ -11,11 +17,14 @@ const {
   readSourceData,
   mergeSources,
   normaliseSources,
+  purgeSources,
 } = require('../lib/data');
 
+console.log(`info: Using metadata file <${argv.metadata}>`);
+
 // This is complete scan of the source data. All sources will be retrieved and processed
 // though the transformation code, rebuilding tools.yaml
-const full = () => src('gulpfile.js/metadata.json')
+const full = () => src(argv.metadata)
   .pipe(transform('utf8', validateMetadata))
   .pipe(transform('utf8', readSourceData))
   .pipe(rename('raw-sources.yaml')) // Write raw data for debug purposes
@@ -24,15 +33,18 @@ const full = () => src('gulpfile.js/metadata.json')
   .pipe(transform('utf8', normaliseSources))
   .pipe(transform('utf8', getRepositoryMetadata))
   .pipe(transform('utf8', classifyTools))
+  .pipe(transform('utf8', purgeSources))
   .pipe(rename('tools.yaml'))
   .pipe(dest('src/_data'));
 
 // This is a scan of the metadata associated with the repositories already
 // held in the repository. No new source data is retrieved from sources
-const metadata = () => src('gulpfile.js/metadata.json')
+// but existing data may be removed if not found
+const metadata = () => src(argv.metadata)
   .pipe(transform('utf8', validateMetadata))
   .pipe(transform('utf8', readLocalSourceData))
   .pipe(transform('utf8', getRepositoryMetadata))
+  .pipe(transform('utf8', purgeSources))
   .pipe(rename('tools.yaml'))
   .pipe(dest('src/_data'));
 

lib/data/index.js

@@ -6,6 +6,7 @@ const {
   readSourceData,
   mergeSources,
   normaliseSources,
+  purgeSources,
 } = require('./transform');
 
 module.exports = {
@@ -16,4 +17,5 @@ module.exports = {
   readSourceData,
   mergeSources,
   normaliseSources,
+  purgeSources,
 };

lib/data/processors/master-processor.js

@@ -1,13 +1,19 @@
 const fs = require('fs');
 const YAML = require('js-yaml');
 
+const { logger } = require('../util');
+
 module.exports = async (args) => {
   const { url } = args;
 
   if (!url) {
     throw new Error(`Mandatory parameters missing when invoking ${__filename}`);
   }
 
-  return YAML.load(fs.readFileSync(url, 'utf-8'))
+  const tools = YAML.load(fs.readFileSync(url, 'utf-8'))
     .map((tool) => ({ ...tool, foundInMaster: true }));
+
+  logger('Number of sources found in master', tools.length);
+
+  return tools;
 };

lib/data/repo/github.js

@@ -59,6 +59,7 @@ module.exports = async (
   repositoryMetadata = {},
 ) => {
   const paths = [
+    { source: '$.data.id', target: 'repositoryId' },
     { source: '$.data.description', target: 'description' },
     { source: '$.data.created_at', target: 'created' },
     { source: '$.data.updated_at', target: 'updated' },
@@ -97,6 +98,8 @@ module.exports = async (
   }
 
   const repoUrl = `https://api.github.com/repos/${organization}/${repo}`;
+  let newUrl = null;
+  let moved = false;
 
   // Get repository metadata. If a 404 is returned mark as not found, throw any other errors
   try {
@@ -105,11 +108,26 @@ module.exports = async (
     repoResponse = await http.get(
       repoUrl,
       {
-        auth: { username, password }, headers, timeout: 60000, validateStatus,
+        auth: { username, password }, headers, timeout: 60000, validateStatus, maxRedirects: 0,
       },
     );
+
+    if (repoResponse.status === 301) {
+      // The repository has moved so prepare to drop it out of the dataset or use the new URL
+      // to identify the repository
+      const targetUrl = repoResponse.data.url;
+
+      repoResponse = await http.get(targetUrl, {
+        auth: { username, password }, timeout: 60000, validateStatus,
+      });
+
+      newUrl = repoResponse.data.html_url
+        .toLowerCase();
+      moved = true;
+    }
   } catch (err) {
     if (!err.response) {
+      /* istanbul ignore if */
       if (process.env.NODE_ENV !== 'test') {
         console.error(err);
       }
@@ -142,6 +160,7 @@ module.exports = async (
     );
   } catch (err) {
     if (!err.response) {
+      /* istanbul ignore if */
       if (process.env.NODE_ENV !== 'test') {
         console.error(err);
       }
@@ -176,6 +195,10 @@ module.exports = async (
 
       return output;
     }, {
-      repositoryMetadata: getReadme(readmeResponse, url, repositoryMetadata),
+      repositoryMetadata: {
+        ...getReadme(readmeResponse, url, repositoryMetadata),
+        ...(newUrl ? { newUrl } : {}),
+        ...(moved ? { moved } : {}),
+      },
     });
 };

lib/data/transform/index.js

@@ -4,12 +4,14 @@ const mergeSources = require('./merge-sources');
 const normaliseSources = require('./normalise-sources');
 const { readLocalSourceData, readSourceData } = require('./read-source-data');
 const validateMetadata = require('./validate-metadata');
+const purgeSources = require('./purge-sources');
 
 module.exports = {
   classifyTools,
   getRepositoryMetadata,
   mergeSources,
   normaliseSources,
+  purgeSources,
   readLocalSourceData,
   readSourceData,
   validateMetadata,

lib/data/transform/merge-sources.js

@@ -12,7 +12,8 @@ module.exports = async (rawSources) => {
   logger(__filename, 'mergeSources');
   const sources = YAML.load(rawSources);
 
-  // Get properties across all sources
+  // Get properties across all sources. This is used to help define the precedence of a given key
+  // when the properties are normalised in subsequent scripts. Executed now as sources are merged
   const sourceProperties = sources
     .reduce((output, source) => Object.assign(
       output,
@@ -22,12 +23,32 @@ module.exports = async (rawSources) => {
       {},
     ), {});
 
+  // Merge all sources that have been previously moved into a single array. This will be used to
+  // prevent work being done again on data pulled back in from the source repositories
+  // Any repositories matching a value found in this array will be ignored
+  const movedSources = sources
+    .filter((source) => source.foundInMaster && source.oldLocations)
+    .reduce((output, source) => output.concat(source.oldLocations), []);
+
   // This of course removes some of the flexibility we get from the processor approach
   // Need to devise a way to discover the uri rather than using hard-coded values
   const mergedSources = sources
-    .reduce((output, source) => {
+    .map((source) => ({
+      uri: getUri(source)
+        .toLowerCase(),
+      source,
+    }))
+    .filter((source) => {
+      if (movedSources.indexOf(source.uri) !== -1) {
+        logger(`Ignoring repository as found to have moved at previous run: ${source.uri}`);
+        return false;
+      }
+
+      return true;
+    })
+    .reduce((output, filteredSource) => {
       const updatedOutput = output;
-      const uri = getUri(source).toLowerCase();
+      const { uri, source } = filteredSource;
 
       if (!updatedOutput[uri]) {
         updatedOutput[uri] = { master: {}, sources: [] };

lib/data/transform/normalise-sources.js

@@ -1,8 +1,7 @@
 const YAML = require('js-yaml');
-const crypto = require('crypto');
 
 const {
-  getUri, logger, normalisePropertyNames, normaliseSplitters,
+  getUri, getHash, logger, normalisePropertyNames, normaliseSplitters,
 } = require('../util');
 
 /**
@@ -16,7 +15,8 @@ module.exports = async (rawSources) => {
 
   const dedupRepositoryValues = (data) => {
     if (Array.isArray(data)) {
-      return [...new Set(data.map((value) => value.toLowerCase()))];
+      return [...new Set(data.map((value) => value
+        .toLowerCase()))];
     }
     return data ? data.toLowerCase() : data;
   };
@@ -35,8 +35,8 @@ module.exports = async (rawSources) => {
 
   const normalisedSources = mergedSources
     .map((tool) => {
-    // Loop across source data and normalise based on occurances of a given property
-    // and naming conventions implied by preponderance in source data
+      // Loop across source data and normalise based on occurances of a given property
+      // and naming conventions implied by preponderance in source data
       const mergedSourceProperties = tool.sources
         .reduce((output, source) => Object.assign(
           output,
@@ -74,13 +74,15 @@ module.exports = async (rawSources) => {
           );
         }, {});
 
-      // Check whether there is any chance of setting a repository value based on any recognise
+      // Check whether there is any chance of setting a repository value based on any recognised
       // properties from sources
       if (!source.repository) {
         const candidate = getUri(source);
 
+        // This will be expanded as more sources are implemented
         if (candidate && candidate.match(/^https.*github\.com(?:\/[a-zA-Z0-9-_.~]+){2}$/)) {
-          source.repository = candidate;
+          source.repository = candidate
+            .toLowerCase();
         }
       }
 
@@ -91,13 +93,13 @@ module.exports = async (rawSources) => {
 
       if (!seed) {
         if (process.env.NODE_ENV !== 'test') {
-          console.error(tool);
+          console.error(`Cannot map URL to uniquely identity tools: ${JSON.stringify(tool)}`);
         }
 
         throw new Error('Could not discover URL and therefore generate ID for tooling source. Check error log for tool properties');
       }
 
-      return Object.assign(tool, { id: crypto.createHash('md5').update(seed).digest('hex') });
+      return Object.assign(tool, { id: getHash(seed.toLowerCase()) });
     });
 
   return YAML.dump(normalisedSources);

lib/data/transform/purge-sources.js

@@ -0,0 +1,82 @@
+const YAML = require('js-yaml');
+
+const { getHash, logger } = require('../util');
+
+module.exports = (rawSources) => {
+  logger(__filename, 'purgeSources');
+
+  const sources = YAML.load(rawSources);
+
+  // List all repository URLs
+  const allRepositories = sources
+    .filter((source) => source.repository)
+    .map((source) => source.repository);
+
+  // Collect up any repositories that have been moved as a pointer ready for merging them with the
+  // the new GitHub URL
+  const newRepositoryLocations = sources
+    .filter((source) => source.repositoryMetadata && source.repositoryMetadata.moved)
+    .reduce((output, source) => {
+      if (output[source.repositoryMetadata.newUrl]) {
+        // eslint-disable-next-line no-param-reassign
+        output[source.repositoryMetadata.newUrl] = output[source.repositoryMetadata.newUrl]
+          .concat(source.repository);
+
+        return output;
+      }
+
+      return Object.assign(
+        output,
+        { [source.repositoryMetadata.newUrl]: [source.repository] },
+      );
+    }, {});
+
+  // Remove anything that is not found or has been moved. If a repository has been moved but the
+  // new location is not in source then change the value of the source.repository property
+  return YAML.dump(
+    sources
+      .map((source) => {
+        const updatedSource = source;
+        const newRepositoryLocation = newRepositoryLocations[source.repository];
+
+        if (newRepositoryLocation) {
+          // Source repository is updated to include old locations for tracking purposes
+
+          // eslint-disable-next-line no-param-reassign
+          updatedSource.oldLocations = [...new Set((source.oldLocations || [])
+            .concat(newRepositoryLocations[source.repository]))];
+        } else if ((source.repositoryMetadata || {}).moved
+          && allRepositories.indexOf(source.repositoryMetadata.newUrl) === -1) {
+          // Source repository has been moved but there is no data in the repository to move it to
+          // Update the repository property with the new URL. This will then be ignored on the
+          // next run and the updated repository value retained in the tools.yaml
+
+          logger(`Moving repository metadata: ${source.repository} to new URL reference: ${updatedSource.repositoryMetadata.newUrl}`);
+
+          updatedSource.oldLocations = [source.repository];
+          updatedSource.repository = updatedSource.repositoryMetadata.newUrl;
+          updatedSource.id = getHash(source.repository);
+
+          delete updatedSource.repositoryMetadata.newUrl;
+          delete updatedSource.repositoryMetadata.moved;
+        }
+
+        return updatedSource;
+      })
+      .filter((source) => {
+        const { repositoryMetadata } = source;
+
+        if (!repositoryMetadata || (!repositoryMetadata.notFound && !repositoryMetadata.moved)) {
+          return true;
+        }
+
+        if (repositoryMetadata.notFound) {
+          logger(`Removing repository as not found at target location: ${source.repository}`);
+          return false;
+        }
+
+        logger(`Removing repository as it has been moved: ${source.repository} and is already catalogued at new URL reference: ${source.repositoryMetadata.newUrl}`);
+        return false;
+      }),
+  );
+};

lib/data/transform/read-source-data.js

@@ -6,7 +6,7 @@ const { logger } = require('../util');
 
 const readSource = async (output, source) => {
   const update = await output;
-  logger(source.title, 'Reading source data...');
+  logger(`${__dirname}/${source.processor}`, source.title, 'Reading source data...');
 
   // Yes, this is an anti-pattern and opinionated approach...
   // but it provides a nice level of flexibility in the build mechanism

lib/data/util.js

@@ -1,8 +1,10 @@
 const dice = require('talisman/metrics/dice');
 const dl = require('talisman/metrics/damerau-levenshtein');
 const log = require('fancy-log');
+const crypto = require('crypto');
 
 module.exports = {
+  getHash: (seed) => crypto.createHash('md5').update(seed).digest('hex'),
   getUri: (source) => (source.repository || source.github || source.link || source.homepage),
   logger: (...args) => {
     if (process.env.NODE_ENV !== 'test') {

package.json

@@ -10,13 +10,14 @@
     "build:site:webpack": "webpack --mode ${NODE_ENV:-development}",
     "build:site:eleventy": "eleventy",
     "build:site": "NODE_ENV=production npm-run-all -l clean build:site:*",
+    "build:all": "npm run build:data:full && npm run build:site",
     "coverage": "export NODE_ENV=test && nyc --reporter=html --reporter=text ./node_modules/.bin/_mocha test/lib/* --exit --recursive --timeout 10000",
     "coverage:desktop": "yarn run coverage && open coverage/index.html",
     "clean": "rimraf docs",
     "serve:webpack": "webpack --mode development --watch",
     "serve:eleventy": "ELEVENTY_ENV=development eleventy --serve",
     "serve": "npm-run-all clean --parallel serve:*",
-    "test": "export NODE_ENV=test && mocha test --exit --recursive --timeout 10000"
+    "test": "export NODE_ENV=test && mocha --exit --recursive --timeout 10000"
   },
   "dependencies": {
     "ajv": "^8.10.0",