Skip to content

Commit

Permalink
Add a --clarificationFile option
Browse files Browse the repository at this point in the history 
This commit adds a --clarificationFile option, modeled after the behavior of `cargo-about`. It allows users to specify clarifications for specific packages and versions with missing or otherwise malformed fields.
  • Loading branch information
@mikayla-maki
mikayla-maki committed Feb 1, 2023
1 parent d17a761 commit 97508d6
Show file tree
Hide file tree
Showing 11 changed files with 255 additions and 65 deletions.
1 change: 1 addition & 0 deletions bin/license-checker-rseidelsohn
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ const usageMessage = [
' --start [filepath] path of the initial json to look for',
' --summary output a summary of the license usage',
' --unknown report guessed licenses as unknown licenses.',
' --clarificationsFile A file that describe the license clarifications for each package, see clarificationExample.json, any field available to the customFormat option can be clarified. All clarifications must include a checksum to detect license changes.',
'',
' --version The current version',
' --help The text you are reading right now :)',
Expand Down
9 changes: 9 additions & 0 deletions clarificationExample.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"license-checker-rseidelsohn@0.0.0": {
"licenses": "MIT",
"licenseFile": "MY_IP",
"checksum": "71eb00f862028a6d940c742dfa96e8f5116823cc5be0b7fb4a640072d2080186",
"repository": "git://somecustomsite.com",
"copyright": "Someone else for some reason"
}
}
1 change: 1 addition & 0 deletions lib/args.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ const knownOpts = {
summary: Boolean,
unknownOpts: Boolean,
version: Boolean,
clarificationsFile: require('path'),
};
const shortHands = {
h: ['--help'],
Expand Down
4 changes: 4 additions & 0 deletions lib/index.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ export interface InitOpts {
* Ignore peerDependencies
*/
nopeer?: boolean;
/**
* A file that contains license clarifications for malformed or non-standard packages
*/
clarificationsFile?: string;
}

/**
Expand Down
204 changes: 139 additions & 65 deletions lib/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ const read = require('read-installed-packages');
const spdxCorrect = require('spdx-correct');
const spdxSatisfies = require('spdx-satisfies');
const treeify = require('treeify');
const createHash = require('crypto').createHash;

const getLicenseTitle = require('./getLicenseTitle');
const licenseFiles = require('./license-files');
Expand All @@ -27,6 +28,17 @@ const debugLog = debug('license-checker-rseidelsohn:log');

debugLog.log = console.log.bind(console);

function first_or_second(obj1, obj2) {
/*istanbul ignore next*/
if (obj1 !== undefined) {
return obj1;
} else if (obj2 !== undefined) {
return obj2;
} else {
return undefined;
}
}

// This function calls itself recursively. On the first iteration, it collects the data of the main program, during the
// second iteration, it collects the data from all direct dependencies, then it collects their dependencies and so on.
const flatten = function flatten(options) {
Expand All @@ -40,6 +52,8 @@ const flatten = function flatten(options) {
let noticeFiles = [];
let readmeFile;
let { data } = options;
let clarification = options.clarifications?.[key];
let passed_clarification_check = clarification === undefined ? true : false;

if (json.private) {
moduleInfo.private = true;
Expand All @@ -64,35 +78,45 @@ const flatten = function flatten(options) {
return options?.customFormat?.[property] !== false;
}

if (mustInclude('repository') && json.repository) {
/*istanbul ignore else*/
if (typeof json?.repository?.url === 'string') {
moduleInfo.repository = json.repository.url
.replace('git+ssh://git@', 'git://')
.replace('git+https://github.com', 'https://github.com')
.replace('git://github.com', 'https://github.com')
.replace('git@github.com:', 'https://github.com/')
.replace(/\.git$/, '');
if (mustInclude('repository')) {
if (clarification?.repository) {
moduleInfo.repository = clarification.repository;
} else if (json.repository) {
/*istanbul ignore else*/
if (typeof json?.repository?.url === 'string') {
moduleInfo.repository = json.repository.url
.replace('git+ssh://git@', 'git://')
.replace('git+https://github.com', 'https://github.com')
.replace('git://github.com', 'https://github.com')
.replace('git@github.com:', 'https://github.com/')
.replace(/\.git$/, '');
}
}
}

if (mustInclude('url') && json?.url?.we) {
if (mustInclude('url')) {
let url = first_or_second(clarification?.url, json?.url?.we);
/*istanbul ignore next*/
moduleInfo.url = json.url.web;
if (url) {
moduleInfo.url = url;
}
}

if (json.author && typeof json.author === 'object') {
/*istanbul ignore else - This should always be there*/
if (mustInclude('publisher') && json.author.name) {
moduleInfo.publisher = json.author.name;
let publisher = first_or_second(clarification?.publisher, json?.author?.name);
if (mustInclude('publisher') && publisher) {
moduleInfo.publisher = publisher;
}

if (mustInclude('email') && json.author.email) {
moduleInfo.email = json.author.email;
let email = first_or_second(clarification?.email, json?.author?.email);
if (mustInclude('email') && email) {
moduleInfo.email = email;
}

if (mustInclude('url') && json.author.url) {
moduleInfo.url = json.author.url;
let url = first_or_second(clarification?.url, json?.author?.url);
if (mustInclude('url') && url) {
moduleInfo.url = url;
}
}

Expand All @@ -101,14 +125,15 @@ const flatten = function flatten(options) {
moduleInfo.dependencyPath = json.path;
}

if (mustInclude('path') && typeof json?.path === 'string') {
moduleInfo.path = json.path;
let module_path = first_or_second(clarification?.path, json?.path);
if (mustInclude('path') && typeof module_path === 'string') {
moduleInfo.path = module_path;
}

licenseData = json.license || json.licenses || undefined;
licenseData = clarification?.licenses || json.license || json.licenses || undefined;

if (json.path && (!json.readme || json.readme.toLowerCase().indexOf('no readme data found') > -1)) {
readmeFile = path.join(json.path, 'README.md');
if (module_path && (!json.readme || json.readme.toLowerCase().indexOf('no readme data found') > -1)) {
readmeFile = path.join(module_path, 'README.md');
/*istanbul ignore if*/
if (fs.existsSync(readmeFile)) {
json.readme = fs.readFileSync(readmeFile, 'utf8').toString();
Expand Down Expand Up @@ -146,8 +171,10 @@ const flatten = function flatten(options) {
}

/*istanbul ignore else*/
if (fs.existsSync(json?.path)) {
dirFiles = fs.readdirSync(json.path);
if (clarification?.licenseFile) {
files = [clarification.licenseFile];
} else if (fs.existsSync(module_path)) {
dirFiles = fs.readdirSync(module_path);
files = licenseFiles(dirFiles);

noticeFiles = dirFiles.filter((filename) => {
Expand All @@ -159,7 +186,7 @@ const flatten = function flatten(options) {
}

files.forEach(function (filename, index) {
licenseFile = path.join(json.path, filename);
licenseFile = path.join(module_path, filename);
// Checking that the file is in fact a normal file and not a directory for example.
/*istanbul ignore else*/
if (fs.lstatSync(licenseFile).isFile()) {
Expand All @@ -172,67 +199,104 @@ const flatten = function flatten(options) {
) {
//Only re-check the license if we didn't get it from elsewhere
content = fs.readFileSync(licenseFile, { encoding: 'utf8' });

moduleInfo.licenses = getLicenseTitle(content);
}

if (index === 0) {
// Treat the file with the highest precedence as licenseFile
/*istanbul ignore else*/
if (mustInclude('licenseFile')) {
moduleInfo.licenseFile = options.basePath
? path.relative(options.basePath, licenseFile)
: licenseFile;
}

if (mustInclude('licenseText') && options.customFormat) {
if (clarification !== undefined && !passed_clarification_check) {
if (clarification?.checksum === undefined) {
console.error('All clarifications must have a checksum');
process.exit(1);
}

if (!content) {
content = fs.readFileSync(licenseFile, { encoding: 'utf8' });
}

/*istanbul ignore else*/
if (options._args && !options._args.csv) {
moduleInfo.licenseText = content.trim();
let sha256 = createHash('sha256').update(content).digest('hex');

if (clarification.checksum !== sha256) {
console.error(`Clarification checksum mismatch for ${key} :(`);
process.exit(1);
} else {
moduleInfo.licenseText = content
.replace(/"/g, "'")
.replace(/\r?\n|\r/g, ' ')
.trim();
passed_clarification_check = true;
}
}

if (mustInclude('copyright') && options.customFormat) {
if (!content) {
content = fs.readFileSync(licenseFile, { encoding: 'utf8' });
}
/*istanbul ignore else*/
if (mustInclude('licenseFile')) {
moduleInfo.licenseFile = first_or_second(
clarification?.licenseFile,
options.basePath ? path.relative(options.basePath, licenseFile) : licenseFile,
);
}

const linesWithCopyright = content
.replace(/\r\n/g, '\n')
.split('\n\n')
.filter(function selectCopyRightStatements(value) {
return (
value.startsWith('opyright', 1) && // include copyright statements
!value.startsWith('opyright notice', 1) && // exclude lines from from license text
!value.startsWith('opyright and related rights', 1)
);
})
.filter(function removeDuplicates(value, index, list) {
return index === 0 || value !== list[0];
});

if (linesWithCopyright.length > 0) {
moduleInfo.copyright = linesWithCopyright[0].replace(/\n/g, '. ').trim();
if (mustInclude('licenseText') && options.customFormat) {
if (clarification?.licenseText) {
moduleInfo.licenseText = clarification.licenseText;
} else {
if (!content) {
content = fs.readFileSync(licenseFile, { encoding: 'utf8' });
}

/*istanbul ignore else*/
if (options._args && !options._args.csv) {
moduleInfo.licenseText = content.trim();
} else {
moduleInfo.licenseText = content
.replace(/"/g, "'")
.replace(/\r?\n|\r/g, ' ')
.trim();
}
}
}

// Mark files with multiple copyright statements. This might be
// an indicator to take a closer look at the LICENSE file.
if (linesWithCopyright.length > 1) {
moduleInfo.copyright = `${moduleInfo.copyright}*`;
if (mustInclude('copyright') && options.customFormat) {
if (clarification?.copyright) {
moduleInfo.copyright = clarification.copyright;
} else {
if (!content) {
content = fs.readFileSync(licenseFile, { encoding: 'utf8' });
}

const linesWithCopyright = content
.replace(/\r\n/g, '\n')
.split('\n\n')
.filter(function selectCopyRightStatements(value) {
return (
value.startsWith('opyright', 1) && // include copyright statements
!value.startsWith('opyright notice', 1) && // exclude lines from from license text
!value.startsWith('opyright and related rights', 1)
);
})
.filter(function removeDuplicates(value, index, list) {
return index === 0 || value !== list[0];
});

if (linesWithCopyright.length > 0) {
moduleInfo.copyright = linesWithCopyright[0].replace(/\n/g, '. ').trim();
}

// Mark files with multiple copyright statements. This might be
// an indicator to take a closer look at the LICENSE file.
if (linesWithCopyright.length > 1) {
moduleInfo.copyright = `${moduleInfo.copyright}*`;
}
}
}
}
}
});

if (!passed_clarification_check) {
console.error('All clarifications must come with a checksum');
process.exit(1);
}

// TODO: How do clarifications interact with notice files?
noticeFiles.forEach((filename) => {
const file = path.join(json.path, filename);
/*istanbul ignore else*/
Expand Down Expand Up @@ -275,7 +339,10 @@ const flatten = function flatten(options) {
if (options.customFormat) {
Object.keys(options.customFormat).forEach((item) => {
if (mustInclude(item) && moduleInfo[item] == null) {
moduleInfo[item] = typeof json[item] === 'string' ? json[item] : options.customFormat[item];
moduleInfo[item] = first_or_second(
clarification?.[item],
typeof json[item] === 'string' ? json[item] : options.customFormat[item],
);
}
});
}
Expand All @@ -286,7 +353,7 @@ const flatten = function flatten(options) {
/**
* ! This function has a wanted sideeffect, as it modifies the json object that is passed by reference.
*
* The detph attribute set in the opts parameter here - which is defined by setting the `--direct` flag - is of
* The depth attribute set in the opts parameter here - which is defined by setting the `--direct` flag - is of
* no use with npm > 2, as the newer npm versions flatten all dependencies into one single directory. So in
* order to making `--direct` work with newer versions of npm, we need to filter out all non-dependencies from
* the json result.
Expand Down Expand Up @@ -351,6 +418,12 @@ exports.init = function init(args, callback) {
pusher = toCheckforFailOn;
}

// An object mapping from Package name -> What contents it should have
let clarifications = {};
if (args.clarificationsFile) {
clarifications = this.parseJson(args.clarificationsFile);
}

if (checker && pusher) {
checker.split(';').forEach((license) => {
license = license.trim();
Expand All @@ -375,6 +448,7 @@ exports.init = function init(args, callback) {
production: args.production,
unknown: args.unknown,
depth: 0,
clarifications: clarifications,
});

const colorize = args.color;
Expand Down
Loading

0 comments on commit 97508d6

Please sign in to comment.