Skip to content

Commit

Permalink
Replace node downloader with vcpkg x-download. (#826)
Browse files Browse the repository at this point in the history 
* Replace bespoke node-based HTTP backend with calling back to vcpkg x-download.

* Deduplicate hash check and don't pass file:// to vcpkg.

* Hook up progress reporting to x-download, except WinHTTP.

* Localize WinHTTP download status messages.

* More WinHTTP localization.

* Add progress reporting for WinHTTP.

* I hate deleters.

* Format/messages.

* Missed an official builds edge case :)

* Add configurations for non-artifacts and official builds, and fix build breaks in there.

* Add note about curl's progress being contractual.
  • Loading branch information
@BillyONeal
BillyONeal authored Dec 10, 2022
1 parent e3f7262 commit 8350f07
Show file tree
Hide file tree
Showing 50 changed files with 2,331 additions and 1,145 deletions.
81 changes: 81 additions & 0 deletions CMakeSettings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,87 @@
}
]
},
{
"name": "x64-Debug-NoArtifacts",
"generator": "Ninja",
"configurationType": "Debug",
"inheritEnvironments": [ "msvc_x64_x64" ],
"buildRoot": "${projectDir}\\out\\build\\${name}",
"installRoot": "${projectDir}\\out\\install\\${name}",
"cmakeCommandArgs": "",
"buildCommandArgs": "",
"ctestCommandArgs": "",
"variables": [
{
"name": "VCPKG_ARTIFACTS_DEVELOPMENT",
"value": "False",
"type": "BOOL"
},
{
"name": "VCPKG_BUILD_TLS12_DOWNLOADER",
"value": "True",
"type": "BOOL"
},
{
"name": "VCPKG_BUILD_BENCHMARKING",
"value": "True",
"type": "BOOL"
},
{
"name": "VCPKG_BUILD_FUZZING",
"value": "True",
"type": "BOOL"
}
]
},
{
"name": "x64-Debug-Official-2022-11-10",
"generator": "Ninja",
"configurationType": "Debug",
"inheritEnvironments": [ "msvc_x64_x64" ],
"buildRoot": "${projectDir}\\out\\build\\${name}",
"installRoot": "${projectDir}\\out\\install\\${name}",
"cmakeCommandArgs": "",
"buildCommandArgs": "",
"ctestCommandArgs": "",
"variables": [
{
"name": "VCPKG_OFFICIAL_BUILD",
"value": "True",
"type": "BOOL"
},
{
"name": "VCPKG_BASE_VERSION",
"value": "2022-11-10",
"type": "STRING"
},
{
"name": "VCPKG_STANDALONE_BUNDLE_SHA",
"value": "edc1bad5689508953842d13b062a750791af57e0c6bb9d441d4545e81d99841dead2d33a5cb382b6cf50d2d32414ee617ada6e761c868fcbb28fa9bcb7bca6ba",
"type": "STRING"
},
{
"name": "VCPKG_CE_SHA",
"value": "b677e4d66e711e623a2765499cc5b662544c1df07a95b495f31a195f6e525c00cef0111dbb008b544d987fdcd1140fd69877908b3c3e7771231eaaa2cb1939ac",
"type": "STRING"
},
{
"name": "VCPKG_BUILD_TLS12_DOWNLOADER",
"value": "True",
"type": "BOOL"
},
{
"name": "VCPKG_BUILD_BENCHMARKING",
"value": "True",
"type": "BOOL"
},
{
"name": "VCPKG_BUILD_FUZZING",
"value": "True",
"type": "BOOL"
}
]
},
{
"name": "x64-Release",
"generator": "Ninja",
Expand Down
2 changes: 0 additions & 2 deletions ce/ce/archivers/git.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,11 @@
// Licensed under the MIT License.

import { UnpackEvents } from '../interfaces/events';
import { Credentials } from '../util/credentials';
import { execute } from '../util/exec-cmd';
import { isFilePath, Uri } from '../util/uri';

export interface CloneOptions {
force?: boolean;
credentials?: Credentials;
}

/** @internal */
Expand Down
177 changes: 27 additions & 150 deletions ce/ce/fs/acquire.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,48 +2,23 @@
// Licensed under the MIT License.

import { strict } from 'assert';
import { pipeline as origPipeline } from 'stream';
import { promisify } from 'util';
import { i } from '../i18n';
import { DownloadEvents } from '../interfaces/events';
import { Session } from '../session';
import { Credentials } from '../util/credentials';
import { RemoteFileUnavailable } from '../util/exceptions';
import { Algorithm, Hash } from '../util/hash';
import { Hash } from '../util/hash';
import { Uri } from '../util/uri';
import { get, getStream, RemoteFile, resolveRedirect } from './https';
import { ProgressTrackingStream } from './streams';

const pipeline = promisify(origPipeline);

const size32K = 1 << 15;
const size64K = 1 << 16;
import { vcpkgDownload } from '../vcpkg';

export interface AcquireOptions extends Hash {
/** force a redownload even if it's in cache */
force?: boolean;
credentials?: Credentials;
}

export async function acquireArtifactFile(session: Session, uris: Array<Uri>, outputFilename: string, events: Partial<DownloadEvents>, options?: AcquireOptions) {
await session.downloads.createDirectory();
const outputFile = session.downloads.join(outputFilename);
session.channels.debug(`Acquire file '${outputFilename}' from [${uris.map(each => each.toString()).join(',')}]`);

if (options?.algorithm && options?.value) {
session.channels.debug(`We have a hash: ${options.algorithm}/${options.value}`);

// if we have hash data, check to see if the output file is good.
if (await outputFile.isFile()) {
session.channels.debug(`There is an output file already, verifying: ${outputFile.fsPath}`);

if (await outputFile.hashValid(events, options)) {
session.channels.debug(`Cached file matched hash: ${outputFile.fsPath}`);
return outputFile;
}
}
}

// is the file present on a local filesystem?
for (const uri of uris) {
if (uri.isLocal) {
Expand Down Expand Up @@ -80,146 +55,51 @@ export async function acquireArtifactFile(session: Session, uris: Array<Uri>, ou
/** */
async function https(session: Session, uris: Array<Uri>, outputFilename: string, events: Partial<DownloadEvents>, options?: AcquireOptions) {
session.channels.debug(`Attempting to download file '${outputFilename}' from [${uris.map(each => each.toString()).join(',')}]`);

let resumeAtOffset = 0;
await session.downloads.createDirectory();
const hashAlgorithm = options?.algorithm;
const outputFile = session.downloads.join(outputFilename);

if (options?.force) {
session.channels.debug(`Acquire '${outputFilename}': force specified, forcing download`);
// is force specified; delete the current file
await outputFile.delete();
}

// start this peeking at the target uris.
session.channels.debug(`Acquire '${outputFilename}': checking remote connections`);
events.downloadStart?.(uris, outputFile.fsPath);
const locations = new RemoteFile(uris, { credentials: options?.credentials });
let url: Uri | undefined;

// is there a file in the cache
if (await outputFile.exists()) {
session.channels.debug(`Acquire '${outputFilename}': local file exists`);
if (options?.algorithm) {
// does it match a hash that we have?
if (await outputFile.hashValid(events, options)) {
session.channels.debug(`Acquire '${outputFilename}': local file hash matches metdata`);
// yes it does. let's just return done.
return outputFile;
}
} else if (hashAlgorithm) {
// does it match a hash that we have?
if (await outputFile.hashValid(events, options)) {
session.channels.debug(`Acquire '${outputFilename}': local file hash matches metdata`);
// yes it does. let's just return done.
return outputFile;
}

// it doesn't match a known hash.
const contentLength = await locations.contentLength;
session.channels.debug(`Acquire '${outputFilename}': remote connection info is back`);
const onDiskSize = await outputFile.size();
if (!await locations.availableLocation) {
if (locations.failures.all(each => each.code === 404)) {
let msg = i`Unable to download file`;
if (options?.credentials) {
msg += (i` - It could be that your authentication credentials are not correct`);
}

session.channels.error(msg);
throw new RemoteFileUnavailable(uris);
}
}
// first, make sure that there is a remote that is accessible.
strict.ok(!!await locations.availableLocation, `Requested file ${outputFilename} has no accessible locations ${uris.map(each => each.toString()).join(',')}`);

url = await locations.resumableLocation;
// ok, does it support resume?
if (url) {
// yes, let's check what the size is expected to be.

if (!options?.algorithm) {

if (contentLength === onDiskSize) {
session.channels.debug(`Acquire '${outputFilename}': on disk file matches length of remote file`);
const algorithm = <Algorithm>(await locations.algorithm);
const value = await locations.hash;
session.channels.debug(`Acquire '${outputFilename}': remote alg/hash: '${algorithm}'/'${value}`);
if (algorithm && value && await outputFile.hashValid(events, { algorithm, value, ...options })) {
session.channels.debug(`Acquire '${outputFilename}': on disk file hash matches the server hash`);
// so *we* don't have the hash, but ... if the server has a hash, we could see if what we have is what they have?
// it does match what the server has.
// I call this an win.
return outputFile;
}

// we don't have a hash, or what we have doesn't match.
// maybe we will get a match below (or resume)
}
}

if (onDiskSize > size64K) {
// it's bigger than 64k. Good. otherwise, we're just wasting time.

// so, how big is the remote
if (contentLength >= onDiskSize) {
session.channels.debug(`Acquire '${outputFilename}': local file length is less than or equal to remote file length`);
// looks like there could be more remotely than we have.
// lets compare the first 32k and the last 32k of what we have
// against what they have and see if they match.
const top = (await get(url, { start: 0, end: size32K - 1, credentials: options?.credentials })).rawBody;
const bottom = (await get(url, { start: onDiskSize - size32K, end: onDiskSize - 1, credentials: options?.credentials })).rawBody;

const onDiskTop = await outputFile.readBlock(0, size32K - 1);
const onDiskBottom = await outputFile.readBlock(onDiskSize - size32K, onDiskSize - 1);

if (top.compare(onDiskTop) === 0 && bottom.compare(onDiskBottom) === 0) {
session.channels.debug(`Acquire '${outputFilename}': first/last blocks are equal`);
// the start and end of what we have does match what they have.
// is this file the same size?
if (contentLength === onDiskSize) {
// same file size, front and back match, let's accept this. begrudgingly
session.channels.debug(`Acquire '${outputFilename}': file size is identical. keeping this one`);
return outputFile;
}
// looks like we can continue from here.
session.channels.debug(`Acquire '${outputFilename}': ok to resume`);
resumeAtOffset = onDiskSize;
}
}
}
}
}

if (resumeAtOffset === 0) {
// clearly we mean to not resume. clean any existing file.
session.channels.debug(`Acquire '${outputFilename}': not resuming file, full download`);
// invalid hash, deleting file
session.channels.debug(`Acquire '${outputFilename}': local file hash mismatch, redownloading`);
await outputFile.delete();
} else if (await outputFile.exists()) {
session.channels.debug(`Acquire '${outputFilename}': skipped due to existing file, no hash known`);
session.channels.warning(i`Assuming '${outputFilename}' is correct; supply a hash in the artifact metadata to suppress this message.`);
return outputFile;
}

url = url || await locations.availableLocation;
strict.ok(!!url, `Requested file ${outputFilename} has no accessible locations ${uris.map(each => each.toString()).join(',')}`);
session.channels.debug(`Acquire '${outputFilename}': initiating download`);
const length = await locations.contentLength;

const inputStream = getStream(url, { start: resumeAtOffset, end: length > 0 ? length : undefined, credentials: options?.credentials });
let progressStream;
if (length > 0) {
progressStream = new ProgressTrackingStream(resumeAtOffset, length);
progressStream.on('progress', (filePercentage) => events.downloadProgress?.(url!, outputFile.fsPath, filePercentage));
session.channels.debug(`Acquire '${outputFilename}': checking remote connections`);
events.downloadStart?.(uris, outputFile.fsPath);
let sha512 = undefined;
if (hashAlgorithm == 'sha512') {
sha512 = options?.value;
}

const outputStream = await outputFile.writeStream({ append: true });
// whoooosh. write out the file
if (progressStream) {
await pipeline(inputStream, progressStream, outputStream);
} else {
await pipeline(inputStream, outputStream);
}
await vcpkgDownload(session, outputFile.fsPath, sha512, uris, events);

events.downloadComplete?.();
// we've downloaded the file, let's see if it matches the hash we have.
if (options?.algorithm) {
if (hashAlgorithm == 'sha512') {
// vcpkg took care of it already
session.channels.debug(`Acquire '${outputFilename}': vcpkg checked SHA512`);
} else if (hashAlgorithm) {
session.channels.debug(`Acquire '${outputFilename}': checking downloaded file hash`);
// does it match the hash that we have?
if (!await outputFile.hashValid(events, options)) {
await outputFile.delete();
throw new Error(i`Downloaded file '${outputFile.fsPath}' did not have the correct hash (${options.algorithm}: ${options.value}) `);
}

session.channels.debug(`Acquire '${outputFilename}': downloaded file hash matches specified hash`);
}

Expand All @@ -233,10 +113,7 @@ export async function resolveNugetUrl(session: Session, pkg: string) {

// let's resolve the redirect first, since nuget servers don't like us getting HEAD data on the targets via a redirect.
// even if this wasn't the case, this is lower cost now rather than later.
const url = await resolveRedirect(session.fileSystem.parseUri(`https://www.nuget.org/api/v2/package/${name}/${version}`));

session.channels.debug(`Resolving nuget package for '${pkg}' to '${url}'`);
return url;
return session.fileSystem.parseUri(`https://www.nuget.org/api/v2/package/${name}/${version}`);
}

export async function acquireNugetFile(session: Session, pkg: string, outputFilename: string, events: Partial<DownloadEvents>, options?: AcquireOptions): Promise<Uri> {
Expand Down
Loading

0 comments on commit 8350f07

Please sign in to comment.