diff --git a/README.md b/README.md index 02f0446..4738346 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ That is because the bulk of the logic of what OCD3 does lives here. Almost all J This is how one would build the underlying Node JS scripts: ```bash -gradle node-scripts:build +./gradlew node-scripts:build ``` And this must be done before submitting code commits. However note that the code base is not really built into anything since the container links directly to **/node-scripts**, but this formats the code and runs the test suite. @@ -231,7 +231,7 @@ See [here](readme/docker/README.md). OCD3 not only needs a Docker image for its binaries but also requires a 'Jenkins home' folder that provides a pre-configured Jenkins setup: ```bash -gradle jenkins:build +./gradlew jenkins:build ``` This will package a zip archive of the jenkins folder. diff --git a/docker/Dockerfile b/docker/Dockerfile index 66d8c3a..54a1c7a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,13 +1,9 @@ -ARG VERSION=2.346.3 +ARG VERSION=2.387.3 FROM jenkins/jenkins:$VERSION MAINTAINER Mekom Solutions USER root -# Add the APP_DATA_DIR volume to keep OpenMRS CD data out of the Jenkins Home -ARG APP_DATA_DIR="/var/lib/openmrs_cd/app_data" -RUN mkdir -p $APP_DATA_DIR && chown -R jenkins:jenkins $APP_DATA_DIR - RUN apt update --fix-missing RUN apt -y install nano jq openssh-server rsync libxml2-utils @@ -22,6 +18,10 @@ RUN sudo apt install -y nodejs # Jenkins needs to be a sudo to execute some build tasks (particularly Bahmni Apps) RUN echo "jenkins ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +# Add the APP_DATA_DIR volume to keep OpenMRS CD data out of the Jenkins Home +ARG APP_DATA_DIR="/var/lib/openmrs_cd/app_data" +RUN mkdir -p $APP_DATA_DIR && chown -R jenkins:jenkins $APP_DATA_DIR + # Provide access to the /usr/share/jenkins directory RUN chown -R jenkins /usr/share/jenkins diff --git a/docker/build.gradle b/docker/build.gradle index b60f1f1..8571741 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -20,7 +20,13 @@ repositories { def dockerHubRepo = "mekomsolutions/openmrscd" def getShortCommit() { - return ['sh', '-c', 'git log -1 --format=%h'].execute().text.trim() + def stdout = new ByteArrayOutputStream() + exec { + commandLine 'git', 'rev-parse', '--short', 'HEAD' + standardOutput = stdout + } + return stdout.toString().trim() + } task buildDockerImage(type: DockerBuildImage) { diff --git a/docker/config/plugins.txt b/docker/config/plugins.txt index 0287881..3963c29 100644 --- a/docker/config/plugins.txt +++ b/docker/config/plugins.txt @@ -1,4 +1,4 @@ -git:4.12.1 +git:5.2.0 ansicolor:1.0.2 workflow-aggregator:590.v6a_d052e5a_a_b_5 nodejs:1.5.1 @@ -6,7 +6,7 @@ envinject:2.866.v5c0403e3d4df conditional-buildstep:1.4.2 parameterized-trigger:2.45 build-name-setter:2.2.0 -pipeline-build-step:2.18 +pipeline-build-step:496.v2449a_9a_221f2 generic-webhook-trigger:1.84 matrix-auth:3.1.5 pipeline-utility-steps:2.13.0 diff --git a/jenkins/jenkins_home/config.xml b/jenkins/jenkins_home/config.xml index cbd739c..f1bfd28 100644 --- a/jenkins/jenkins_home/config.xml +++ b/jenkins/jenkins_home/config.xml @@ -1,7 +1,7 @@ - 2.346.2 + 2.387.3 2 NORMAL true diff --git a/node-scripts/spec/pipeline3/impl/dockerComposeFromArtifact.spec.js b/node-scripts/spec/pipeline3/impl/dockerComposeFromArtifact.spec.js new file mode 100644 index 0000000..95b67b8 --- /dev/null +++ b/node-scripts/spec/pipeline3/impl/dockerComposeFromArtifact.spec.js @@ -0,0 +1,138 @@ +"use strict"; + +describe("Docker Compose Generic Deployment implementation", function() { + // deps + const path = require("path"); + const config = require(path.resolve("src/utils/config")); + const cst = require(path.resolve("src/const")); + const dockerCompose = require(path.resolve( + "src/pipeline3/impl/dockerComposeFromArtifact" + )); + + const scripts = require(path.resolve( + "src/" + config.getJobNameForPipeline3() + "/scripts" + )); + + var instanceDef = { + uuid: "336af1ee-90a1-4d1b-baaf-db12c84deec0", + name: "cambodia1", + type: "dev", + group: "tlc", + deployment: { + hostDir: "/var/docker-volumes/", + type: "dockerComposeFromArtifact", + composePlugin: true, + dockerComposeFiles: ["docker-compose.yml", "docker-compose-2.yml"], + envFiles: ["env-file-1", "env-file-2"], + value: { + projectPath: "/var/docker-volumes/artifacts/run/docker", + services: ["proxy", "openmrs", "mysql"] + }, + timezone: "Europe/Amsterdam", + host: { + type: "ssh", + value: { + ip: "hsc-dev.mekomsolutions.net", + user: "mekom", + port: "22" + } + } + }, + data: [ + { + type: "sqlDocker", + value: { + service: "mysql", + sourceFile: "/var/instance-data/sql-script.sql" + } + } + ] + }; + it("should use the correct compose command if composePlugin is set to true", () => { + var expected = "docker compose"; + expect(dockerCompose.composeExec(true)).toEqual(expected); + }); + + it("should use the correct compose command if composePlugin is set to false", () => { + var expected = "docker-compose"; + expect(dockerCompose.composeExec(false)).toEqual(expected); + }); + + it("should generate the correct compose command given multiple docker compose files", () => { + var expected = " -f docker-compose.yml -f docker-compose-2.yml "; + expect( + dockerCompose.combineComposeFiles( + instanceDef.deployment.dockerComposeFiles + ) + ).toEqual(expected); + }); + it("should generate Pre-Host Preparation deployment script", () => { + var expected = ""; + expected += scripts.initFolder( + config.getCDDockerDirPath(instanceDef.uuid), + "jenkins", + "jenkins", + true + ); + + expect( + dockerCompose.preHostPreparation.getDeploymentScript(instanceDef) + ).toEqual(expected); + }); + + it("should generate Host Preparation deployment script", () => { + var expected = ""; + + expected += + scripts.remote( + instanceDef.deployment.host.value, + scripts.writeProperty( + "TIMEZONE", + instanceDef.deployment.timezone, + path + .resolve(instanceDef.deployment.value.projectPath, ".env") + .toString() + ) + ) + "\n"; + + expected += + scripts.remote( + instanceDef.deployment.host.value, + scripts.createEnvVarFileDockerGeneric(instanceDef) + ) + + "\n" + + scripts.remote( + instanceDef.deployment.host.value, + "cd " + + path.resolve(instanceDef.deployment.value.projectPath).toString() + + " && docker compose -p " + + instanceDef.name + + " -f docker-compose.yml -f docker-compose-2.yml " + + " --env-file=env-file-1 --env-file=env-file-2 --env-file=/var/docker-volumes/artifacts/run/docker/cambodia1.env" + + " build --pull proxy openmrs mysql" + + "\n" + ) + + scripts.remote( + instanceDef.deployment.host.value, + "cd " + + path.resolve(instanceDef.deployment.value.projectPath).toString() + + " && docker compose -p " + + instanceDef.name + + " -f docker-compose.yml -f docker-compose-2.yml " + + " --env-file=env-file-1 --env-file=env-file-2 --env-file=/var/docker-volumes/artifacts/run/docker/cambodia1.env" + + " pull proxy openmrs mysql" + + "\n" + ) + + scripts.remote( + instanceDef.deployment.host.value, + "sudo chown -R root:root " + + path + .resolve(instanceDef.deployment.hostDir, instanceDef.name) + .toString() + ); + let generated = dockerCompose.hostPreparation.getDeploymentScript( + instanceDef + ); + expect(generated).toEqual(expected); + }); +}); diff --git a/node-scripts/src/instance-event/validator.js b/node-scripts/src/instance-event/validator.js index f19d8a2..fd83d3b 100644 --- a/node-scripts/src/instance-event/validator.js +++ b/node-scripts/src/instance-event/validator.js @@ -112,7 +112,6 @@ module.exports = { } else if (deployment.hostDir === "") { throw new Error("The 'host dir' is not specified."); } - // validating the actual config based on its type module.exports .getConfigValidatorsMap() @@ -248,6 +247,18 @@ module.exports = { ); } }, + validateDockerComposeFromArtifactDeploymentConfigValue: function(value) { + if ( + JSON.stringify(Object.keys(value).sort()) >= + JSON.stringify( + Object.keys(new model.DockerComposeFromArtifactDeployment()).sort() + ) + ) { + throw new Error( + "The Docker compose deployment value should be provided as an instance of 'DockerComposeFromArtifactDeployment'." + ); + } + }, validateFileTLSDeploymentConfigValue: function(value) { if ( JSON.stringify(Object.keys(value).sort()) !== @@ -306,8 +317,9 @@ module.exports = { ); } } else if (element.type === "sqlDocker") { + // Since we just check the number of keys we can loosen the check to just if the keys are more than the model if ( - JSON.stringify(Object.keys(element.value).sort()) !== + JSON.stringify(Object.keys(element.value).sort()) >= JSON.stringify(Object.keys(new model.SqlDocker()).sort()) ) { throw new Error( @@ -369,7 +381,9 @@ module.exports = { dockerComposeMaven: module.exports.validateDockerComposeMavenDeploymentConfigValue, dockerComposeGenericMaven: - module.exports.validateDockerComposeGenericMavenDeploymentConfigValue + module.exports.validateDockerComposeGenericMavenDeploymentConfigValue, + dockerComposeFromArtifact: + module.exports.validateDockerComposeFromArtifactDeploymentConfigValue }; } }; diff --git a/node-scripts/src/pipeline3/impl/dockerComposeFromArtifact.js b/node-scripts/src/pipeline3/impl/dockerComposeFromArtifact.js new file mode 100644 index 0000000..a62e598 --- /dev/null +++ b/node-scripts/src/pipeline3/impl/dockerComposeFromArtifact.js @@ -0,0 +1,368 @@ +"use strict"; + +const path = require("path"); +const _ = require("lodash"); + +const cst = require("../../const"); +const scripts = require("../scripts"); +const config = require(cst.CONFIGPATH); +const heredoc_2 = cst.HEREDOC_2; + +const cdScript = function(instanceDef, sudo) { + let path = require("path"); + let projectPath = instanceDef.deployment.value.projectPath; + let script = ""; + script += "set -e\n"; + script += "cd " + projectPath + " && "; + if (sudo) { + script += "sudo "; + } + return script; +}; + +const composeExec = function(composePlugin) { + let exec = "docker-compose"; + if (composePlugin) { + exec = "docker compose"; + } + return exec; +}; + +const combineComposeFiles = function(composeFiles = []) { + let dockerComposeFiles = " "; + for (const composeFile of composeFiles) { + dockerComposeFiles = dockerComposeFiles + `-f ${composeFile} `; + } + return dockerComposeFiles; +}; + +const combineEnvFiles = function(instanceDef = {}) { + let defaultEnvDir = path + .resolve( + instanceDef.deployment.value.projectPath, + instanceDef.name + ".env" + ) + .toString(); + let files = " "; + if ( + instanceDef && + instanceDef.deployment && + instanceDef.deployment.envFiles + ) { + for (const envFile of instanceDef.deployment.envFiles) { + files = files + `--env-file=${envFile} `; + } + } + files = files + "--env-file=" + defaultEnvDir; + return files; +}; + +/** + * Implementation of script utils to specifically manipulate Docker Compose containers. + * + */ +module.exports = { + preHostPreparation: { + getDeploymentScript: function(instanceDef) { + // Retrieve the Docker Compose project + const mavenProject = instanceDef.deployment.value.mavenProject; + const dockerDirPath = path.resolve( + config.getCDDockerDirPath(instanceDef.uuid) + ); + var script = ""; + script += scripts.initFolder(dockerDirPath, "jenkins", "jenkins", true); + if (mavenProject) { + script += scripts.fetchArtifact( + mavenProject, + "maven", + dockerDirPath, + null + ); + } + return script; + }, + getDataScript: function(instanceDef) { + let script = ""; + + return script; + }, + getArtifactsScript: function(instanceDef) { + return ""; + } + }, + hostPreparation: { + getDeploymentScript: function(instanceDef) { + const scripts = require("../scripts"); + let script = ""; + + // Rsync the Docker Compose project files to the target machine + const hostDir = path.resolve( + instanceDef.deployment.hostDir, + instanceDef.name + ); + const ssh = instanceDef.deployment.host.value; + let projectPath = instanceDef.deployment.value.projectPath; + // Set the Timezone via a env var "TIMEZONE" + if (instanceDef.deployment.timezone) { + script += scripts.remote( + instanceDef.deployment.host.value, + scripts.writeProperty( + "TIMEZONE", + instanceDef.deployment.timezone, + path.resolve(projectPath, ".env").toString() + ) + ); + } + + script += "\n"; + script += scripts.remote( + instanceDef.deployment.host.value, + scripts.createEnvVarFileDockerGeneric(instanceDef) + ); + script += "\n"; + + // docker-compose build + script += scripts.remote( + instanceDef.deployment.host.value, + "cd " + + projectPath + + " && " + + composeExec(instanceDef.deployment.composePlugin) + + " -p " + + instanceDef.name + + combineComposeFiles(instanceDef.deployment.dockerComposeFiles) + + combineEnvFiles(instanceDef) + + " build --pull" + + require("./dockerCompose").getInstanceServicesAsStringList( + instanceDef + ) + + "\n" + ); + + // docker-compose pull + script += scripts.remote( + instanceDef.deployment.host.value, + "cd " + + projectPath + + " && " + + composeExec(instanceDef.deployment.composePlugin) + + " -p " + + instanceDef.name + + combineComposeFiles(instanceDef.deployment.dockerComposeFiles) + + combineEnvFiles(instanceDef) + + " pull" + + require("./dockerCompose").getInstanceServicesAsStringList( + instanceDef + ) + + "\n" + ); + + script += scripts.remote( + instanceDef.deployment.host.value, + "sudo chown -R root:root " + + path + .resolve(instanceDef.deployment.hostDir, instanceDef.name) + .toString() + ); + + return script; + }, + getDataScript: function(instanceDef) { + var script = ""; + var ssh = instanceDef.deployment.host.value; + let projectPath = instanceDef.deployment.value.projectPath; + instanceDef.data.forEach(function(data) { + var applyData = { + instance: function() { + // 'instance' type must be handled differently as it requires access to the 'db'. + // therefore, the script is provided in the 'stage' script (host-prepation.js, start-instance.js...) + }, + sqlDocker: function() { + let sql = data.value; + let destFolder = path.resolve( + instanceDef.deployment.hostDir, + instanceDef.name, + "docker_compose/sqls", + sql.service + ); + if (sql.destinationDir) { + destFolder = path.resolve(sql.destinationDir); + } + script += scripts.remote( + ssh, + "sudo cp " + sql.sourceFile + " " + destFolder + "\n" + ); + script += scripts.remote( + ssh, + "cd " + + projectPath + + " && " + + composeExec(instanceDef.deployment.composePlugin) + + " -p " + + instanceDef.name + + combineComposeFiles(instanceDef.deployment.dockerComposeFiles) + + combineEnvFiles(instanceDef) + + " rm -vsf " + + sql.service + + " && " + + composeExec(instanceDef.deployment.composePlugin) + + " -p " + + instanceDef.name + + combineComposeFiles(instanceDef.deployment.dockerComposeFiles) + + combineEnvFiles(instanceDef) + + " up -d " + + sql.service + ); + } + }; + applyData[data.type](); + }); + return script; + }, + getArtifactsScript: function(instanceDef) { + return ""; + } + }, + startInstance: { + getDeploymentScript: function(instanceDef) { + let script = ""; + let projectPath = instanceDef.deployment.value.projectPath; + script += scripts.remote( + instanceDef.deployment.host.value, + "cd " + + projectPath + + " && " + + composeExec(instanceDef.deployment.composePlugin) + + " -p " + + instanceDef.name + + combineComposeFiles(instanceDef.deployment.dockerComposeFiles) + + combineEnvFiles(instanceDef) + + " up -d" + + require("./dockerCompose").getInstanceServicesAsStringList( + instanceDef + ) + + "\n" + ); + return script; + }, + getDataScript: function(instanceDef) { + return ""; + }, + getArtifactsScript: function(instanceDef) { + return ""; + } + }, + ifExists: function() {}, + restart: function(instanceDef, sudo) { + let script = ""; + script += cdScript(instanceDef); + script += + composeExec(instanceDef.deployment.composePlugin) + + " -p " + + instanceDef.name + + combineComposeFiles(instanceDef.deployment.dockerComposeFiles) + + combineEnvFiles(instanceDef) + + " restart "; + + return script + "\n"; + }, + remove: function(instanceDef, sudo) { + let script = ""; + script += cdScript(instanceDef); + script += + composeExec(instanceDef.deployment.composePlugin) + + " -p " + + instanceDef.name; + script += combineEnvFiles(instanceDef); + var rmVolumes = + instanceDef.type.toString() != cst.INSTANCETYPE_PROD ? " -v" : ""; + script += " down" + rmVolumes; + return script + "\n"; + }, + down: function(instanceDef, sudo) { + let script = ""; + script += cdScript(instanceDef, sudo); + script += + composeExec(instanceDef.deployment.composePlugin) + + " -p " + + instanceDef.name; + script += combineEnvFiles(instanceDef); + script += " down"; + return script + "\n"; + }, + pull: function() {}, + exec: (instanceDef, command, service) => { + let script = ""; + script += cdScript(instanceDef); + script += + composeExec(instanceDef.deployment.composePlugin) + + " -p " + + instanceDef.name + + combineComposeFiles(instanceDef.deployment.dockerComposeFiles) + + " exec -T " + + service + + " /bin/bash -s <<" + + heredoc_2 + + "\n"; + script += "set -e\n"; + script += command + "\n"; + script += heredoc_2; + + return script + "\n"; + }, + setProperties: function(instanceDef, property, output) { + let script = ""; + let path = require("path"); + let propPath = path + .resolve( + instanceDef.deployment.hostDir, + instanceDef.name, + "docker_compose/properties/" + property.service + ) + .toString(); + + let propFilePath = path.resolve(propPath, property.filename).toString(); + script += require("../scripts").remote( + instanceDef.deployment.host.value, + "\n" + + "if [[ ! -e " + + propFilePath + + " ]]; then\n" + + "sudo mkdir -p " + + propPath + + "\n" + + "sudo touch " + + propFilePath + + "\n" + + "fi\n" + + "sudo bash -c 'cat > " + + propFilePath + + " < { + script += " " + service.toString(); + }); + return script; + }, + stop: function(instanceDef, sudo) { + let script = ""; + script += cdScript(instanceDef); + script += + composeExec(instanceDef.deployment.composePlugin) + + " -p " + + instanceDef.name + + combineComposeFiles(instanceDef.deployment.dockerComposeFiles); + script += combineEnvFiles(instanceDef); + script += " stop"; + return script + "\n"; + }, + combineComposeFiles: combineComposeFiles, + composeExec: composeExec +}; diff --git a/node-scripts/src/pipeline3/scripts.js b/node-scripts/src/pipeline3/scripts.js index d910dd4..8a7063e 100644 --- a/node-scripts/src/pipeline3/scripts.js +++ b/node-scripts/src/pipeline3/scripts.js @@ -10,26 +10,47 @@ const heredoc_2 = cst.HEREDOC_2; const heredoc_3 = cst.HEREDOC_3; const utils = require("../utils/utils"); const model = require("../utils/model"); -const createEnvVarFile = function(instanceDef, dockerComposePath) { - let script = ""; - - var distEnvFile = path +const createEnvVarFile = function(instanceDef, deploymentBasePath) { + let projectPath = path .resolve( instanceDef.deployment.hostDir, instanceDef.name, - instanceDef.name + ".env" + deploymentBasePath ) .toString(); + if ( + instanceDef.deployment && + instanceDef.deployment.value && + instanceDef.deployment.value.projectPath + ) { + projectPath = instanceDef.deployment.value.projectPath; + } + + let script = ""; - var envFile = path + let distEnvFile = path .resolve( instanceDef.deployment.hostDir, instanceDef.name, - dockerComposePath, - ".env" + instanceDef.name + ".env" ) .toString(); + if ( + instanceDef.deployment && + instanceDef.deployment.value && + instanceDef.deployment.value.projectPath + ) { + distEnvFile = path + .resolve( + instanceDef.deployment.value.projectPath, + instanceDef.name + ".env" + ) + .toString(); + } + + var envFile = path.resolve(projectPath, ".env").toString(); + script += "if [[ ! -e " + distEnvFile + diff --git a/node-scripts/src/utils/config.js b/node-scripts/src/utils/config.js index 87a56a2..cf3760b 100644 --- a/node-scripts/src/utils/config.js +++ b/node-scripts/src/utils/config.js @@ -382,7 +382,8 @@ module.exports = { "dockerCompose", "dockerComposeGit", "dockerComposeMaven", - "dockerComposeGenericMaven" + "dockerComposeGenericMaven", + "dockerComposeFromArtifact" ]; }, getInstanceArtifactsTypes: function() { diff --git a/node-scripts/src/utils/model.js b/node-scripts/src/utils/model.js index 4a7a458..26f6c62 100644 --- a/node-scripts/src/utils/model.js +++ b/node-scripts/src/utils/model.js @@ -68,12 +68,22 @@ class DockerComposeMavenDeployment { } } class DockerComposeGenericMavenDeployment { - constructor(mavenProject, services) { + constructor(mavenProject, services, mavenUrl) { this.mavenProject = mavenProject; this.services = services; this.mavenUrl = this.mavenUrl; } } + +/** + * A type which assumes that the Docker Compose project will be installed through the processing of the 'artifacts' section. + */ +class DockerComposeFromArtifactDeployment { + constructor(services) { + this.services = services; + } +} + /** * Describes a File TLS deployment section */ @@ -273,6 +283,7 @@ module.exports = { DockerComposeGitDeployment: DockerComposeGitDeployment, DockerComposeMavenDeployment: DockerComposeMavenDeployment, DockerComposeGenericMavenDeployment: DockerComposeGenericMavenDeployment, + DockerComposeFromArtifactDeployment: DockerComposeFromArtifactDeployment, FileTLSDeployment: FileTLSDeployment, VaultTLSDeployment: VaultTLSDeployment, Artifact: Artifact,