diff --git a/bin/license-checker-rseidelsohn b/bin/license-checker-rseidelsohn old mode 100644 new mode 100755 index d0b1e6c..826e831 --- a/bin/license-checker-rseidelsohn +++ b/bin/license-checker-rseidelsohn @@ -47,6 +47,7 @@ const usageMessage = [ ' --start [filepath] path of the initial json to look for', ' --summary output a summary of the license usage', ' --unknown report guessed licenses as unknown licenses.', + ' --clarificationsFile A file that describe the license clarifications for each package, see clarificationExample.json, any field available to the customFormat option can be clarified. All clarifications must include a checksum to detect license changes.', '', ' --version The current version', ' --help The text you are reading right now :)', diff --git a/clarificationExample.json b/clarificationExample.json new file mode 100644 index 0000000..6e531cc --- /dev/null +++ b/clarificationExample.json @@ -0,0 +1,9 @@ +{ + "license-checker-rseidelsohn@0.0.0": { + "licenses": "MIT", + "licenseFile": "MY_IP", + "checksum": "71eb00f862028a6d940c742dfa96e8f5116823cc5be0b7fb4a640072d2080186", + "repository": "git://somecustomsite.com", + "copyright": "Someone else for some reason" + } +} \ No newline at end of file diff --git a/lib/args.js b/lib/args.js index b528b4e..20e5a4a 100644 --- a/lib/args.js +++ b/lib/args.js @@ -39,6 +39,7 @@ const knownOpts = { summary: Boolean, unknownOpts: Boolean, version: Boolean, + clarificationsFile: require('path'), }; const shortHands = { h: ['--help'], diff --git a/lib/index.d.ts b/lib/index.d.ts index 2836bfc..4113039 100644 --- a/lib/index.d.ts +++ b/lib/index.d.ts @@ -95,6 +95,10 @@ export interface InitOpts { * Ignore peerDependencies */ nopeer?: boolean; + /** + * A file that contains license clarifications for malformed or non-standard packages + */ + clarificationsFile?: string; } /** diff --git a/lib/index.js b/lib/index.js index 453fc6d..89573e7 100644 --- a/lib/index.js +++ b/lib/index.js @@ -16,6 +16,7 @@ const read = require('read-installed-packages'); const spdxCorrect = require('spdx-correct'); const spdxSatisfies = require('spdx-satisfies'); const treeify = require('treeify'); +const createHash = require('crypto').createHash; const getLicenseTitle = require('./getLicenseTitle'); const licenseFiles = require('./license-files'); @@ -27,6 +28,17 @@ const debugLog = debug('license-checker-rseidelsohn:log'); debugLog.log = console.log.bind(console); +function first_or_second(obj1, obj2) { + /*istanbul ignore next*/ + if (obj1 !== undefined) { + return obj1; + } else if (obj2 !== undefined) { + return obj2; + } else { + return undefined; + } +} + // This function calls itself recursively. On the first iteration, it collects the data of the main program, during the // second iteration, it collects the data from all direct dependencies, then it collects their dependencies and so on. const flatten = function flatten(options) { @@ -40,6 +52,8 @@ const flatten = function flatten(options) { let noticeFiles = []; let readmeFile; let { data } = options; + let clarification = options.clarifications?.[key]; + let passed_clarification_check = clarification === undefined ? true : false; if (json.private) { moduleInfo.private = true; @@ -64,35 +78,45 @@ const flatten = function flatten(options) { return options?.customFormat?.[property] !== false; } - if (mustInclude('repository') && json.repository) { - /*istanbul ignore else*/ - if (typeof json?.repository?.url === 'string') { - moduleInfo.repository = json.repository.url - .replace('git+ssh://git@', 'git://') - .replace('git+https://github.com', 'https://github.com') - .replace('git://github.com', 'https://github.com') - .replace('git@github.com:', 'https://github.com/') - .replace(/\.git$/, ''); + if (mustInclude('repository')) { + if (clarification?.repository) { + moduleInfo.repository = clarification.repository; + } else if (json.repository) { + /*istanbul ignore else*/ + if (typeof json?.repository?.url === 'string') { + moduleInfo.repository = json.repository.url + .replace('git+ssh://git@', 'git://') + .replace('git+https://github.com', 'https://github.com') + .replace('git://github.com', 'https://github.com') + .replace('git@github.com:', 'https://github.com/') + .replace(/\.git$/, ''); + } } } - if (mustInclude('url') && json?.url?.we) { + if (mustInclude('url')) { + let url = first_or_second(clarification?.url, json?.url?.we); /*istanbul ignore next*/ - moduleInfo.url = json.url.web; + if (url) { + moduleInfo.url = url; + } } if (json.author && typeof json.author === 'object') { /*istanbul ignore else - This should always be there*/ - if (mustInclude('publisher') && json.author.name) { - moduleInfo.publisher = json.author.name; + let publisher = first_or_second(clarification?.publisher, json?.author?.name); + if (mustInclude('publisher') && publisher) { + moduleInfo.publisher = publisher; } - if (mustInclude('email') && json.author.email) { - moduleInfo.email = json.author.email; + let email = first_or_second(clarification?.email, json?.author?.email); + if (mustInclude('email') && email) { + moduleInfo.email = email; } - if (mustInclude('url') && json.author.url) { - moduleInfo.url = json.author.url; + let url = first_or_second(clarification?.url, json?.author?.url); + if (mustInclude('url') && url) { + moduleInfo.url = url; } } @@ -101,14 +125,15 @@ const flatten = function flatten(options) { moduleInfo.dependencyPath = json.path; } - if (mustInclude('path') && typeof json?.path === 'string') { - moduleInfo.path = json.path; + let module_path = first_or_second(clarification?.path, json?.path); + if (mustInclude('path') && typeof module_path === 'string') { + moduleInfo.path = module_path; } - licenseData = json.license || json.licenses || undefined; + licenseData = clarification?.licenses || json.license || json.licenses || undefined; - if (json.path && (!json.readme || json.readme.toLowerCase().indexOf('no readme data found') > -1)) { - readmeFile = path.join(json.path, 'README.md'); + if (module_path && (!json.readme || json.readme.toLowerCase().indexOf('no readme data found') > -1)) { + readmeFile = path.join(module_path, 'README.md'); /*istanbul ignore if*/ if (fs.existsSync(readmeFile)) { json.readme = fs.readFileSync(readmeFile, 'utf8').toString(); @@ -146,8 +171,10 @@ const flatten = function flatten(options) { } /*istanbul ignore else*/ - if (fs.existsSync(json?.path)) { - dirFiles = fs.readdirSync(json.path); + if (clarification?.licenseFile) { + files = [clarification.licenseFile]; + } else if (fs.existsSync(module_path)) { + dirFiles = fs.readdirSync(module_path); files = licenseFiles(dirFiles); noticeFiles = dirFiles.filter((filename) => { @@ -159,7 +186,7 @@ const flatten = function flatten(options) { } files.forEach(function (filename, index) { - licenseFile = path.join(json.path, filename); + licenseFile = path.join(module_path, filename); // Checking that the file is in fact a normal file and not a directory for example. /*istanbul ignore else*/ if (fs.lstatSync(licenseFile).isFile()) { @@ -172,67 +199,104 @@ const flatten = function flatten(options) { ) { //Only re-check the license if we didn't get it from elsewhere content = fs.readFileSync(licenseFile, { encoding: 'utf8' }); + moduleInfo.licenses = getLicenseTitle(content); } if (index === 0) { // Treat the file with the highest precedence as licenseFile - /*istanbul ignore else*/ - if (mustInclude('licenseFile')) { - moduleInfo.licenseFile = options.basePath - ? path.relative(options.basePath, licenseFile) - : licenseFile; - } - if (mustInclude('licenseText') && options.customFormat) { + if (clarification !== undefined && !passed_clarification_check) { + if (clarification?.checksum === undefined) { + console.error('All clarifications must have a checksum'); + process.exit(1); + } + if (!content) { content = fs.readFileSync(licenseFile, { encoding: 'utf8' }); } - /*istanbul ignore else*/ - if (options._args && !options._args.csv) { - moduleInfo.licenseText = content.trim(); + let sha256 = createHash('sha256').update(content).digest('hex'); + + if (clarification.checksum !== sha256) { + console.error(`Clarification checksum mismatch for ${key} :(`); + process.exit(1); } else { - moduleInfo.licenseText = content - .replace(/"/g, "'") - .replace(/\r?\n|\r/g, ' ') - .trim(); + passed_clarification_check = true; } } - if (mustInclude('copyright') && options.customFormat) { - if (!content) { - content = fs.readFileSync(licenseFile, { encoding: 'utf8' }); - } + /*istanbul ignore else*/ + if (mustInclude('licenseFile')) { + moduleInfo.licenseFile = first_or_second( + clarification?.licenseFile, + options.basePath ? path.relative(options.basePath, licenseFile) : licenseFile, + ); + } - const linesWithCopyright = content - .replace(/\r\n/g, '\n') - .split('\n\n') - .filter(function selectCopyRightStatements(value) { - return ( - value.startsWith('opyright', 1) && // include copyright statements - !value.startsWith('opyright notice', 1) && // exclude lines from from license text - !value.startsWith('opyright and related rights', 1) - ); - }) - .filter(function removeDuplicates(value, index, list) { - return index === 0 || value !== list[0]; - }); - - if (linesWithCopyright.length > 0) { - moduleInfo.copyright = linesWithCopyright[0].replace(/\n/g, '. ').trim(); + if (mustInclude('licenseText') && options.customFormat) { + if (clarification?.licenseText) { + moduleInfo.licenseText = clarification.licenseText; + } else { + if (!content) { + content = fs.readFileSync(licenseFile, { encoding: 'utf8' }); + } + + /*istanbul ignore else*/ + if (options._args && !options._args.csv) { + moduleInfo.licenseText = content.trim(); + } else { + moduleInfo.licenseText = content + .replace(/"/g, "'") + .replace(/\r?\n|\r/g, ' ') + .trim(); + } } + } - // Mark files with multiple copyright statements. This might be - // an indicator to take a closer look at the LICENSE file. - if (linesWithCopyright.length > 1) { - moduleInfo.copyright = `${moduleInfo.copyright}*`; + if (mustInclude('copyright') && options.customFormat) { + if (clarification?.copyright) { + moduleInfo.copyright = clarification.copyright; + } else { + if (!content) { + content = fs.readFileSync(licenseFile, { encoding: 'utf8' }); + } + + const linesWithCopyright = content + .replace(/\r\n/g, '\n') + .split('\n\n') + .filter(function selectCopyRightStatements(value) { + return ( + value.startsWith('opyright', 1) && // include copyright statements + !value.startsWith('opyright notice', 1) && // exclude lines from from license text + !value.startsWith('opyright and related rights', 1) + ); + }) + .filter(function removeDuplicates(value, index, list) { + return index === 0 || value !== list[0]; + }); + + if (linesWithCopyright.length > 0) { + moduleInfo.copyright = linesWithCopyright[0].replace(/\n/g, '. ').trim(); + } + + // Mark files with multiple copyright statements. This might be + // an indicator to take a closer look at the LICENSE file. + if (linesWithCopyright.length > 1) { + moduleInfo.copyright = `${moduleInfo.copyright}*`; + } } } } } }); + if (!passed_clarification_check) { + console.error('All clarifications must come with a checksum'); + process.exit(1); + } + + // TODO: How do clarifications interact with notice files? noticeFiles.forEach((filename) => { const file = path.join(json.path, filename); /*istanbul ignore else*/ @@ -275,7 +339,10 @@ const flatten = function flatten(options) { if (options.customFormat) { Object.keys(options.customFormat).forEach((item) => { if (mustInclude(item) && moduleInfo[item] == null) { - moduleInfo[item] = typeof json[item] === 'string' ? json[item] : options.customFormat[item]; + moduleInfo[item] = first_or_second( + clarification?.[item], + typeof json[item] === 'string' ? json[item] : options.customFormat[item], + ); } }); } @@ -286,7 +353,7 @@ const flatten = function flatten(options) { /** * ! This function has a wanted sideeffect, as it modifies the json object that is passed by reference. * - * The detph attribute set in the opts parameter here - which is defined by setting the `--direct` flag - is of + * The depth attribute set in the opts parameter here - which is defined by setting the `--direct` flag - is of * no use with npm > 2, as the newer npm versions flatten all dependencies into one single directory. So in * order to making `--direct` work with newer versions of npm, we need to filter out all non-dependencies from * the json result. @@ -351,6 +418,12 @@ exports.init = function init(args, callback) { pusher = toCheckforFailOn; } + // An object mapping from Package name -> What contents it should have + let clarifications = {}; + if (args.clarificationsFile) { + clarifications = this.parseJson(args.clarificationsFile); + } + if (checker && pusher) { checker.split(';').forEach((license) => { license = license.trim(); @@ -375,6 +448,7 @@ exports.init = function init(args, callback) { production: args.production, unknown: args.unknown, depth: 0, + clarifications: clarifications, }); const colorize = args.color; diff --git a/tests/clarificationFile-test.js b/tests/clarificationFile-test.js new file mode 100644 index 0000000..e968a7e --- /dev/null +++ b/tests/clarificationFile-test.js @@ -0,0 +1,80 @@ +const assert = require('assert'); +const path = require('path'); +const checker = require('../lib/index'); +const { describe } = require('node:test'); +const spawn = require('child_process').spawn; + + +describe('clarifications', function() { + function parseAndClarify(parsePath, clarificationPath, result) { + return function(done) { + checker.init( + { + start: path.join(__dirname, parsePath), + clarificationsFile: path.join(__dirname, clarificationPath), + customFormat: { + "licenses": "", + "publisher": "", + "email": "", + "path": "", + "licenseFile": "", + "licenseText": "" + } + }, + function(err, filtered) { + result.output = filtered; + done(); + }, + ); + }; + } + + let result = {}; + + const clarifications_path = './fixtures/clarifications'; + + before(parseAndClarify(clarifications_path, '../clarificationExample.json', result)); + + it('should replace existing license', function() { + const output = result.output['license-checker-rseidelsohn@0.0.0']; + + assert.equal(output.licenseText, "Some mild rephrasing of an MIT license"); + assert.equal(output.licenses, "MIT"); + }); + + + it('should exit 1 if the checksum does not match', function(done) { + let data = ""; + let license_checker = spawn('node', [path.join(__dirname, '../bin/license-checker-rseidelsohn'), '--start', path.join(__dirname, clarifications_path), '--clarificationsFile', path.join(__dirname, clarifications_path, 'mismatchFile.json')], { + cwd: path.join(__dirname, '../'), + }); + + license_checker.stderr.on('data', function(stdout) { + data += stdout.toString(); + }); + + license_checker.on('exit', function(code) { + assert.equal(code, 1); + assert.equal(data.includes("checksum mismatch"), true) + done(); + }); + }); + + + it('should exit 1 if no checksum', function(done) { + let data = ""; + let license_checker = spawn('node', [path.join(__dirname, '../bin/license-checker-rseidelsohn'), '--start', path.join(__dirname, clarifications_path), '--clarificationsFile', path.join(__dirname, clarifications_path, 'badClarification.json')], { + cwd: path.join(__dirname, '../'), + }); + + license_checker.stderr.on('data', function(stdout) { + data += stdout.toString(); + }); + + license_checker.on('exit', function(code) { + assert.equal(code, 1); + assert.equal(data.includes("must have a checksum"), true) + done(); + }); + }) +}); diff --git a/tests/fixtures/clarifications/MISMATCH_LICENSE b/tests/fixtures/clarifications/MISMATCH_LICENSE new file mode 100644 index 0000000..eca84a2 --- /dev/null +++ b/tests/fixtures/clarifications/MISMATCH_LICENSE @@ -0,0 +1 @@ +This should have a different check sum than the other one \ No newline at end of file diff --git a/tests/fixtures/clarifications/MY_IP b/tests/fixtures/clarifications/MY_IP new file mode 100644 index 0000000..057fa67 --- /dev/null +++ b/tests/fixtures/clarifications/MY_IP @@ -0,0 +1 @@ +Some mild rephrasing of an MIT license \ No newline at end of file diff --git a/tests/fixtures/clarifications/badClarification.json b/tests/fixtures/clarifications/badClarification.json new file mode 100644 index 0000000..13502cc --- /dev/null +++ b/tests/fixtures/clarifications/badClarification.json @@ -0,0 +1,6 @@ +{ + "license-checker-rseidelsohn@0.0.0": { + "licenses": "MIT", + "licenseFile": "MY_IP" + } +} \ No newline at end of file diff --git a/tests/fixtures/clarifications/mismatchFile.json b/tests/fixtures/clarifications/mismatchFile.json new file mode 100644 index 0000000..bedeb8b --- /dev/null +++ b/tests/fixtures/clarifications/mismatchFile.json @@ -0,0 +1,7 @@ +{ + "license-checker-rseidelsohn@0.0.0": { + "licenses": "MIT", + "licenseFile": "MISMATCH_LICENSE", + "checksum": "71eb00f862028a6d940c742dfa96e8f5116823cc5be0b7fb4a640072d2080186" + } +} \ No newline at end of file diff --git a/tests/fixtures/clarifications/package.json b/tests/fixtures/clarifications/package.json new file mode 100644 index 0000000..cf307fe --- /dev/null +++ b/tests/fixtures/clarifications/package.json @@ -0,0 +1,6 @@ +{ + "name": "license-checker-rseidelsohn", + "version": "0.0.0", + "author": "Roman Seidelsohn ", + "license": "???????" +}