[DownloadBuildArtifactsV0] Add download post-check

Tasks/DownloadBuildArtifactsV0/Strings/resources.resjson/en-US/resources.resjson

@@ -31,6 +31,8 @@
   "loc.input.help.downloadPath": "Path on the agent machine where the artifacts will be downloaded",
   "loc.input.label.parallelizationLimit": "Parallelization limit",
   "loc.input.help.parallelizationLimit": "Number of files to download simultaneously",
+  "loc.input.label.checkDownloadedFiles": "Check downloaded files",
+  "loc.input.help.checkDownloadedFiles": "If checked, this build task will check that all files are fully downloaded.",
   "loc.messages.DownloadArtifacts": "Downloading artifact %s from: %s",
   "loc.messages.DownloadingArtifactsForBuild": "Downloading artifacts for build: %s",
   "loc.messages.LinkedArtifactCount": "Linked artifacts count:  %s",
@@ -51,5 +53,12 @@
   "loc.messages.DownloadingContainerResource": "Downloading items from container resource %s",
   "loc.messages.DefinitionNameMatchFound": "Definition Name %s resolved to id %d",
   "loc.messages.InvalidBuildDefinitionName": "Definition name %s didn't correspond to a valid definition",
-  "loc.messages.UnresolvedDefinitionId": "Could not resolve build definition id"
+  "loc.messages.UnresolvedDefinitionId": "Could not resolve build definition id",
+  "loc.messages.BeginCheckArtifactConsistency": "Starting artifact consistency check",
+  "loc.messages.CorruptedItemsList": "The following files seem to be corrupted during artifact download:",
+  "loc.messages.BuildArtifactNotHealthy": "Downloaded build artifact is not healthy",
+  "loc.messages.BuildArtifactHealthy": "All artifacts items are healthy",
+  "loc.messages.BuildArtifactCheckRetry": "Check of downloaded files not passed. Now trying to download the build artifact again",
+  "loc.messages.BuildArtifactCheckRetryAttempt": "Attempt %s to download build artifact",
+  "loc.messages.BuildArtifactCheckRetryError": "Downloaded files check failed %s times"
 }

Tasks/DownloadBuildArtifactsV0/download_handlers/handlers_config.ts

@@ -0,0 +1,23 @@
+import { ArtifactEngineOptions } from 'artifact-engine/Engine';
+import { BuildArtifact } from 'azure-devops-node-api/interfaces/BuildInterfaces'
+import { PersonalAccessTokenCredentialHandler } from 'artifact-engine/Providers/typed-rest-client/Handlers';
+
+export interface IBaseHandlerConfig {
+    artifactInfo: BuildArtifact,
+    downloadPath: string,
+    downloaderOptions: ArtifactEngineOptions,
+    checkDownloadedFiles: boolean
+}
+
+export interface IContainerHandlerConfig extends IBaseHandlerConfig {
+    endpointUrl: string,
+    templatePath: string,
+    handler: PersonalAccessTokenCredentialHandler,
+}
+
+export interface IContainerHandlerZipConfig extends IBaseHandlerConfig {
+    endpointUrl: string,
+    projectId: string,
+    buildId: number,
+    handler: PersonalAccessTokenCredentialHandler
+}

Tasks/DownloadBuildArtifactsV0/download_handlers/handlers_implementation.ts

@@ -0,0 +1,190 @@
+import { IBaseHandlerConfig, IContainerHandlerConfig, IContainerHandlerZipConfig } from './handlers_config'
+import { handlerCheckDownloadedFiles } from "../download_helper";
+import { WebProvider, FilesystemProvider, ZipProvider, } from 'artifact-engine/Providers';
+import { ArtifactEngine } from 'artifact-engine/Engine';
+import { IArtifactProvider, ArtifactDownloadTicket } from 'artifact-engine/Models';
+import * as tl from 'azure-pipelines-task-lib/task';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as DecompressZip from "decompress-zip";
+
+abstract class DownloadHandler {
+    config: IBaseHandlerConfig;
+
+    constructor(handlerConfig: IBaseHandlerConfig) {
+        this.config = handlerConfig;
+    }
+
+    public async downloadResources(): Promise<Array<ArtifactDownloadTicket>> {
+        const downloader: ArtifactEngine = new ArtifactEngine();
+        const sourceProvider: IArtifactProvider = this.getSourceProvider();
+        const destinationProvider: IArtifactProvider = this.getDestinationProvider();
+
+        const downloadPromise: Promise<Array<ArtifactDownloadTicket>> = new Promise<Array<ArtifactDownloadTicket>>(async (downloadComplete, downloadFailed) => {
+            try {
+                // First attempt to download artifact
+                const downloadTickets: Array<ArtifactDownloadTicket> = await downloader.processItems(sourceProvider, destinationProvider, this.config.downloaderOptions);
+
+                // We will proceed with the files check only if the "Check download files" option enabled
+                if (this.config.checkDownloadedFiles && Array.isArray(downloadTickets)) {
+                    try {
+                        // Launch the files check, if all files are fully downloaded no exceptions will be thrown.
+                        handlerCheckDownloadedFiles(downloadTickets);
+                        downloadComplete(downloadTickets);
+                    } catch (error) {
+                        tl.warning('Check of downloaded files not passed. Now trying to download the build artifact again.');
+                        downloadFailed(error);
+                    }
+                } else {
+                    downloadComplete(downloadTickets);
+                }
+            } catch (error) {
+                downloadFailed(error);
+            }
+        });
+
+        return downloadPromise;
+    };
+
+    public abstract getSourceProvider(): IArtifactProvider;
+    public abstract getDestinationProvider(): IArtifactProvider;
+}
+
+export class DownloadHandlerContainer extends DownloadHandler {
+    config: IContainerHandlerConfig;
+
+    public getSourceProvider(): WebProvider {
+        console.log(tl.loc("DownloadingContainerResource", this.config.artifactInfo.resource.data));
+        const containerParts: Array<string> = this.config.artifactInfo.resource.data.split('/');
+
+        if (containerParts.length < 3) {
+            throw new Error(tl.loc("FileContainerInvalidArtifactData"));
+        }
+
+        const containerId: number = parseInt(containerParts[1]);
+        let containerPath: string = containerParts.slice(2, containerParts.length).join('/');
+
+        if (containerPath == "/") {
+            //container REST api oddity. Passing '/' as itemPath downloads the first file instead of returning the meta data about the all the files in the root level. 
+            //This happens only if the first item is a file.
+            containerPath = ""
+        }
+
+        const variables = {};
+        const itemsUrl: string = `${this.config.endpointUrl}/_apis/resources/Containers/${containerId}?itemPath=${encodeURIComponent(containerPath)}&isShallow=true&api-version=4.1-preview.4`;
+
+        console.log(tl.loc("DownloadArtifacts", this.config.artifactInfo.name, itemsUrl));
+
+        const provider: WebProvider = new WebProvider(itemsUrl, this.config.templatePath, variables, this.config.handler);
+        return provider;
+    }
+
+    public getDestinationProvider(): FilesystemProvider {
+        const provider: FilesystemProvider = new FilesystemProvider(this.config.downloadPath);
+        return provider;
+    }
+}
+
+export class DownloadHandlerFilePath extends DownloadHandler {
+    public getSourceProvider(): FilesystemProvider {
+        let downloadUrl = this.config.artifactInfo.resource.data;
+        let artifactName = this.config.artifactInfo.name.replace('/', '\\');
+        let artifactLocation = path.join(downloadUrl, artifactName);
+
+        if (!fs.existsSync(artifactLocation)) {
+            console.log(tl.loc("ArtifactNameDirectoryNotFound", artifactLocation, downloadUrl));
+            artifactLocation = downloadUrl;
+        }
+
+        const provider: FilesystemProvider = new FilesystemProvider(artifactLocation, artifactName);
+        return provider;
+    }
+
+    public getDestinationProvider(): FilesystemProvider {
+        const provider: FilesystemProvider = new FilesystemProvider(this.config.downloadPath);
+        return provider;
+    }
+}
+
+export class DownloadHandlerContainerZip extends DownloadHandler {
+    config: IContainerHandlerZipConfig;
+    readonly archiveUrl: string;
+    readonly zipLocation: string;
+
+    constructor(handlerConfig: IContainerHandlerZipConfig) {
+        super(handlerConfig);
+        this.archiveUrl = `${this.config.endpointUrl}/${this.config.projectId}/_apis/build/builds/${this.config.buildId}/artifacts?artifactName=${this.config.artifactInfo.name}&$format=zip`;
+        this.zipLocation = path.join(this.config.downloadPath, `${this.config.artifactInfo.name}.zip`);
+    };
+
+    public downloadResources(): Promise<any> {
+        const downloadProcess: Promise<any> = new Promise((resolve, reject) => {
+            tl.debug("Starting downloadZip action");
+
+            if (tl.exist(this.zipLocation)) {
+                tl.rmRF(this.zipLocation);
+            }
+
+            super.downloadResources().then(() => {
+                tl.debug(`Successfully downloaded from ${this.archiveUrl}`);
+
+                this.unzipContainer(this.config.downloadPath).then(() => {
+                    tl.debug(`Successfully extracted ${this.zipLocation}`);
+
+                    if (tl.exist(this.zipLocation)) {
+                        tl.rmRF(this.zipLocation);
+                    }
+
+                    resolve();
+
+                }).catch((error) => {
+                    reject(error);
+                });
+
+            }).catch((error) => {
+                reject(error);
+            });
+        });
+
+        return downloadProcess;
+    }
+
+    private unzipContainer(unzipLocation: string): Promise<void> {
+        const unZipPromise: Promise<void> = new Promise<void>((resolve, reject) => {
+            if (!tl.exist(this.zipLocation)) {
+                return resolve();
+            }
+
+            tl.debug(`Extracting ${this.zipLocation} to ${unzipLocation}`);
+
+            const unzipper = new DecompressZip(this.zipLocation);
+
+            unzipper.on('error', err => {
+                return reject(tl.loc("ExtractionFailed", err))
+            });
+
+            unzipper.on('extract', log => {
+                tl.debug(`Extracted ${this.zipLocation} to ${unzipLocation} successfully`);
+                return resolve();
+            });
+
+            unzipper.extract({
+                path: unzipLocation
+            });
+
+        });
+
+        return unZipPromise;
+    }
+
+    public getSourceProvider(): ZipProvider {
+        console.log(tl.loc("DownloadArtifacts", this.config.artifactInfo.name, this.archiveUrl));
+        const provider: ZipProvider = new ZipProvider(this.archiveUrl, this.config.handler);
+        return provider;
+    }
+
+    public getDestinationProvider(): FilesystemProvider {
+        const provider: FilesystemProvider = new FilesystemProvider(this.zipLocation);
+        return provider;
+    }
+}

Tasks/DownloadBuildArtifactsV0/download_helper.ts

@@ -0,0 +1,98 @@
+import { debug, loc } from 'azure-pipelines-task-lib/task';
+import { ArtifactDownloadTicket, ItemType, TicketState } from 'artifact-engine/Models';
+import { getFileSizeInBytes } from './file_helper';
+
+/**
+ * Just a Promise wrapper for setTimeout function
+ * @param {number} interval - timeout interval in milliseconds
+ */
+export function timeoutPromise(interval: number): Promise<{}> {
+    debug(`Wait for ${interval} milliseconds`);
+    return new Promise(resolve => setTimeout(resolve, interval));
+}
+
+/**
+ * This function checks a result of artifact download
+ * @param  {Array<ArtifactDownloadTicket>} downloadTickets
+ * @throws Exception if downloaded build artifact is not healthy
+ * @returns void
+ */
+export function handlerCheckDownloadedFiles(downloadTickets: Array<ArtifactDownloadTicket>): void {
+    debug(`Items count: ${downloadTickets.length}`);
+
+    const corruptedItems: Array<ArtifactDownloadTicket> = downloadTickets.filter(ticket => isItemCorrupted(ticket));
+
+    if (corruptedItems.length > 0) {
+        console.log(loc("CorruptedItemsList"));
+        corruptedItems.map(item => console.log(item.artifactItem.metadata.destinationUrl));
+
+        throw new Error(loc('BuildArtifactNotHealthy'));
+    }
+
+    console.log(loc('BuildArtifactHealthy'));
+}
+
+/**
+ * This function investigates the download ticket of the artifact item.
+ * 
+ * Since artifact's items stored as compressed files the only appropriate way (at the moment) 
+ * to make sure that the item fully downloaded is to compare bytes length before compress 
+ * that provided by Azure DevOps and actual bytes length from local storage.
+ * 
+ * @param  {ArtifactDownloadTicket} ticket - download ticket of artifact item
+ * @returns {boolean} `true` if item corrupted, `false` if item healthy
+ */
+function isItemCorrupted(ticket: ArtifactDownloadTicket): boolean {
+    let isCorrupted: boolean = false;
+
+    // We check the tickets only with processed status and File item type 
+    if ((ticket.state === TicketState.Processed) && (ticket.artifactItem.itemType === ItemType.File)) {
+        debug(`Start check for item: ${ticket.artifactItem.path}`);
+        debug(`Getting info from download ticket`);
+
+        const localPathToFile: string = ticket.artifactItem.metadata.destinationUrl;
+        debug(`Local path to the item: ${localPathToFile}`);
+
+        if (ticket.artifactItem.fileLength) {
+            const expectedBytesLength: number = Number(ticket.artifactItem.fileLength);
+
+            if (expectedBytesLength === NaN) {
+                debug('Incorrect data in related download ticket, skip item validation.');
+                isCorrupted = true;
+            } else {
+                debug(`Expected length in bytes ${expectedBytesLength}`);
+
+                let actualBytesLength: number = -1;
+
+                try {
+                    actualBytesLength = getFileSizeInBytes(localPathToFile);
+                    debug(`Actual length in bytes ${actualBytesLength}`);
+                    isCorrupted = (expectedBytesLength !== actualBytesLength);
+                } catch (error) {
+                    debug("Unable to get file stats from local storage due to the following error:");
+                    debug(error);
+                    debug('Skip item validation');
+                    isCorrupted = true;
+                }
+            }
+        } else if (ticket.artifactItem.metadata.downloadUrl.endsWith('format=zip')) {
+            // When we use a Zip Provider the Artifact Engine returns only "fileSizeInBytes"
+            try {
+                const expectedBytesLength: number = Number(ticket.fileSizeInBytes);
+                const actualBytesLength: number = getFileSizeInBytes(localPathToFile);
+
+                debug(`Expected length in bytes ${expectedBytesLength}`);
+                debug(`Actual length in bytes ${actualBytesLength}`);
+
+                isCorrupted = (expectedBytesLength !== actualBytesLength);
+            } catch (error) {
+                debug("Unable to get file stats from local storage due to the following error:");
+                debug(error);
+                debug('Skip item validation');
+                isCorrupted = true;
+            }
+        }
+    }
+
+    return isCorrupted;
+}

Tasks/DownloadBuildArtifactsV0/file_helper.ts

@@ -0,0 +1,21 @@
+import { Stats, statSync as getFile } from 'fs';
+
+/**
+ * Get size of file on local storage
+ * @param  {string} path - path to the file in local storage
+ * @throws Exception if path to the file is empty
+ * @returns Size of the file in bytes
+ */
+export function getFileSizeInBytes(path: string): number {
+    let fileSize: number = 0;
+
+    if (path) {
+        // TODO: Add support of BigInt after migration on Node10
+        const file: Stats = getFile(path);
+        fileSize = file.size;
+    } else {
+        throw 'Path to the file is empty';
+    }
+
+    return fileSize;
+}