Merge branch 'v2-prototype' of https://github.com/0xProject/0x-monorepo into feature/web3-wrapper/unexport-marshaller

10 files changed, 116 insertions, 111 deletions

diff --git a/packages/monorepo-scripts/src/constants.ts b/packages/monorepo-scripts/src/constants.ts
index f08313bd2..e5d3348bd 100644
--- a/packages/monorepo-scripts/src/constants.ts
+++ b/packages/monorepo-scripts/src/constants.ts
@@ -3,6 +3,6 @@ import * as path from 'path';
 export const constants = {
     monorepoRootPath: path.join(__dirname, '../../..'),
     stagingWebsite: 'http://staging-0xproject.s3-website-us-east-1.amazonaws.com',
-    lernaExecutable: path.join('node_modules', 'lerna', 'cli.js'),
+    lernaExecutable: path.join('node_modules', '@0x-lerna-fork', 'lerna', 'cli.js'),
     githubPersonalAccessToken: process.env.GITHUB_PERSONAL_ACCESS_TOKEN_0X_JS,
 };
diff --git a/packages/monorepo-scripts/src/globals.d.ts b/packages/monorepo-scripts/src/globals.d.ts
index cb6a61b55..2b25802ee 100644
--- a/packages/monorepo-scripts/src/globals.d.ts
+++ b/packages/monorepo-scripts/src/globals.d.ts
@@ -14,3 +14,4 @@ declare module 'semver-sort' {
 }
 
 declare module 'promisify-child-process';
+declare module '@lerna/batch-packages';
diff --git a/packages/monorepo-scripts/src/postpublish_utils.ts b/packages/monorepo-scripts/src/postpublish_utils.ts
index 3ecb7b7c5..37861f0dd 100644
--- a/packages/monorepo-scripts/src/postpublish_utils.ts
+++ b/packages/monorepo-scripts/src/postpublish_utils.ts
@@ -6,6 +6,7 @@ import * as path from 'path';
 import * as publishRelease from 'publish-release';
 
 import { constants } from './constants';
+import { configs } from './utils/configs';
 import { utils } from './utils/utils';
 
 const publishReleaseAsync = promisify(publishRelease);
@@ -34,7 +35,7 @@ export const postpublishUtils = {
             throw new Error('version field required in package.json. Cannot publish release notes to Github.');
         }
         const postpublishConfig = _.get(packageJSON, 'config.postpublish', {});
-        const configs: PostpublishConfigs = {
+        const postpublishConfigs: PostpublishConfigs = {
             cwd,
             packageName: packageJSON.name,
             version: packageJSON.version,
@@ -48,44 +49,47 @@ export const postpublishUtils = {
                 s3StagingBucketPath: _.get(postpublishConfig, 'docPublishConfigs.s3StagingBucketPath'),
             },
         };
-        return configs;
+        return postpublishConfigs;
     },
     async runAsync(packageJSON: any, tsConfigJSON: any, cwd: string): Promise<void> {
-        const configs = postpublishUtils.generateConfig(packageJSON, tsConfigJSON, cwd);
+        if (configs.IS_LOCAL_PUBLISH) {
+            return;
+        }
+        const postpublishConfigs = postpublishUtils.generateConfig(packageJSON, tsConfigJSON, cwd);
         await postpublishUtils.publishReleaseNotesAsync(
-            configs.cwd,
-            configs.packageName,
-            configs.version,
-            configs.assets,
+            postpublishConfigs.cwd,
+            postpublishConfigs.packageName,
+            postpublishConfigs.version,
+            postpublishConfigs.assets,
         );
         if (
-            !_.isUndefined(configs.docPublishConfigs.s3BucketPath) ||
-            !_.isUndefined(configs.docPublishConfigs.s3StagingBucketPath)
+            !_.isUndefined(postpublishConfigs.docPublishConfigs.s3BucketPath) ||
+            !_.isUndefined(postpublishConfigs.docPublishConfigs.s3StagingBucketPath)
         ) {
             utils.log('POSTPUBLISH: Release successful, generating docs...');
             await postpublishUtils.generateAndUploadDocsAsync(
-                configs.cwd,
-                configs.docPublishConfigs.fileIncludes,
-                configs.version,
-                configs.docPublishConfigs.s3BucketPath,
+                postpublishConfigs.cwd,
+                postpublishConfigs.docPublishConfigs.fileIncludes,
+                postpublishConfigs.version,
+                postpublishConfigs.docPublishConfigs.s3BucketPath,
             );
         } else {
             utils.log(`POSTPUBLISH: No S3Bucket config found for ${packageJSON.name}. Skipping doc JSON generation.`);
         }
     },
     async publishDocsToStagingAsync(packageJSON: any, tsConfigJSON: any, cwd: string): Promise<void> {
-        const configs = postpublishUtils.generateConfig(packageJSON, tsConfigJSON, cwd);
-        if (_.isUndefined(configs.docPublishConfigs.s3StagingBucketPath)) {
+        const postpublishConfigs = postpublishUtils.generateConfig(packageJSON, tsConfigJSON, cwd);
+        if (_.isUndefined(postpublishConfigs.docPublishConfigs.s3StagingBucketPath)) {
             utils.log('config.postpublish.docPublishConfigs.s3StagingBucketPath entry in package.json not found!');
             return;
         }
 
         utils.log('POSTPUBLISH: Generating docs...');
         await postpublishUtils.generateAndUploadDocsAsync(
-            configs.cwd,
-            configs.docPublishConfigs.fileIncludes,
-            configs.version,
-            configs.docPublishConfigs.s3StagingBucketPath,
+            postpublishConfigs.cwd,
+            postpublishConfigs.docPublishConfigs.fileIncludes,
+            postpublishConfigs.version,
+            postpublishConfigs.docPublishConfigs.s3StagingBucketPath,
         );
     },
     async publishReleaseNotesAsync(cwd: string, packageName: string, version: string, assets: string[]): Promise<void> {
diff --git a/packages/monorepo-scripts/src/prepublish_checks.ts b/packages/monorepo-scripts/src/prepublish_checks.ts
index 431e848ca..683c26094 100644
--- a/packages/monorepo-scripts/src/prepublish_checks.ts
+++ b/packages/monorepo-scripts/src/prepublish_checks.ts
@@ -50,7 +50,7 @@ async function checkGitTagsForNextVersionAndDeleteIfExistAsync(updatedPublicPack
 async function checkCurrentVersionMatchesLatestPublishedNPMPackageAsync(
     updatedPublicPackages: Package[],
 ): Promise<void> {
-    utils.log('Check package versions against npmjs.org...');
+    utils.log('Check package versions against npm registry...');
     const versionMismatches = [];
     for (const pkg of updatedPublicPackages) {
         const packageName = pkg.packageJson.name;
diff --git a/packages/monorepo-scripts/src/publish.ts b/packages/monorepo-scripts/src/publish.ts
index c44e1f85e..d1df4c56e 100644
--- a/packages/monorepo-scripts/src/publish.ts
+++ b/packages/monorepo-scripts/src/publish.ts
@@ -4,26 +4,20 @@ import * as promisify from 'es6-promisify';
 import * as _ from 'lodash';
 import * as moment from 'moment';
 import opn = require('opn');
-import { exec as execAsync, spawn } from 'promisify-child-process';
+import { exec as execAsync } from 'promisify-child-process';
 import * as prompt from 'prompt';
 import semver = require('semver');
-import semverDiff = require('semver-diff');
 import semverSort = require('semver-sort');
 
 import { constants } from './constants';
-import { Package, PackageToVersionChange, SemVerIndex, VersionChangelog } from './types';
+import { Package, PackageToNextVersion, VersionChangelog } from './types';
 import { changelogUtils } from './utils/changelog_utils';
+import { configs } from './utils/configs';
 import { utils } from './utils/utils';
 
 const DOC_GEN_COMMAND = 'docs:json';
 const NPM_NAMESPACE = '@0xproject/';
-const IS_DRY_RUN = process.env.IS_DRY_RUN === 'true';
 const TODAYS_TIMESTAMP = moment().unix();
-const semverNameToIndex: { [semver: string]: number } = {
-    patch: SemVerIndex.Patch,
-    minor: SemVerIndex.Minor,
-    major: SemVerIndex.Major,
-};
 const packageNameToWebsitePath: { [name: string]: string } = {
     '0x.js': '0xjs',
     'web3-wrapper': 'web3_wrapper',
@@ -39,28 +33,43 @@ const packageNameToWebsitePath: { [name: string]: string } = {
 
 (async () => {
     // Fetch public, updated Lerna packages
-    const shouldIncludePrivate = false;
-    const updatedPublicPackages = await utils.getUpdatedPackagesAsync(shouldIncludePrivate);
+    const shouldIncludePrivate = true;
+    const allUpdatedPackages = await utils.getUpdatedPackagesAsync(shouldIncludePrivate);
 
-    await confirmDocPagesRenderAsync(updatedPublicPackages);
+    if (!configs.IS_LOCAL_PUBLISH) {
+        await confirmDocPagesRenderAsync(allUpdatedPackages);
+    }
 
     // Update CHANGELOGs
+    const updatedPublicPackages = _.filter(allUpdatedPackages, pkg => !pkg.packageJson.private);
     const updatedPublicPackageNames = _.map(updatedPublicPackages, pkg => pkg.packageJson.name);
     utils.log(`Will update CHANGELOGs and publish: \n${updatedPublicPackageNames.join('\n')}\n`);
-    const packageToVersionChange = await updateChangeLogsAsync(updatedPublicPackages);
+    const packageToNextVersion = await updateChangeLogsAsync(updatedPublicPackages);
+
+    const updatedPrivatePackages = _.filter(allUpdatedPackages, pkg => pkg.packageJson.private);
+    _.each(updatedPrivatePackages, pkg => {
+        const currentVersion = pkg.packageJson.version;
+        const packageName = pkg.packageJson.name;
+        const nextPatchVersionIfValid = semver.inc(currentVersion, 'patch');
+        if (!_.isNull(nextPatchVersionIfValid)) {
+            packageToNextVersion[packageName] = nextPatchVersionIfValid;
+        } else {
+            throw new Error(`Encountered invalid semver version: ${currentVersion} for package: ${packageName}`);
+        }
+    });
 
     // Push changelog changes to Github
-    if (!IS_DRY_RUN) {
+    if (!configs.IS_LOCAL_PUBLISH) {
         await pushChangelogsToGithubAsync();
     }
 
     // Call LernaPublish
     utils.log('Version updates to apply:');
-    _.each(packageToVersionChange, (versionChange: string, packageName: string) => {
+    _.each(packageToNextVersion, (versionChange: string, packageName: string) => {
         utils.log(`${packageName} -> ${versionChange}`);
     });
     utils.log(`Calling 'lerna publish'...`);
-    await lernaPublishAsync(packageToVersionChange);
+    await lernaPublishAsync(packageToNextVersion);
 })().catch(err => {
     utils.log(err);
     process.exit(1);
@@ -115,8 +124,8 @@ async function pushChangelogsToGithubAsync(): Promise<void> {
     utils.log(`Pushed CHANGELOG updates to Github`);
 }
 
-async function updateChangeLogsAsync(updatedPublicPackages: Package[]): Promise<PackageToVersionChange> {
-    const packageToVersionChange: PackageToVersionChange = {};
+async function updateChangeLogsAsync(updatedPublicPackages: Package[]): Promise<PackageToNextVersion> {
+    const packageToNextVersion: PackageToNextVersion = {};
     for (const pkg of updatedPublicPackages) {
         const packageName = pkg.packageJson.name;
         let changelog = changelogUtils.getChangelogOrCreateIfMissing(packageName, pkg.location);
@@ -143,7 +152,7 @@ async function updateChangeLogsAsync(updatedPublicPackages: Package[]): Promise<
                 ],
             };
             changelog = [newChangelogEntry, ...changelog];
-            packageToVersionChange[packageName] = semverDiff(currentVersion, nextPatchVersionIfValid);
+            packageToNextVersion[packageName] = nextPatchVersionIfValid;
         } else {
             // Update existing entry with timestamp
             const lastEntry = changelog[0];
@@ -154,7 +163,7 @@ async function updateChangeLogsAsync(updatedPublicPackages: Package[]): Promise<
             const proposedNextVersion = lastEntry.version;
             lastEntry.version = updateVersionNumberIfNeeded(currentVersion, proposedNextVersion);
             changelog[0] = lastEntry;
-            packageToVersionChange[packageName] = semverDiff(currentVersion, lastEntry.version);
+            packageToNextVersion[packageName] = lastEntry.version;
         }
 
         // Save updated CHANGELOG.json
@@ -166,49 +175,21 @@ async function updateChangeLogsAsync(updatedPublicPackages: Package[]): Promise<
         utils.log(`${packageName}: Updated CHANGELOG.md`);
     }
 
-    return packageToVersionChange;
+    return packageToNextVersion;
 }
 
-async function lernaPublishAsync(packageToVersionChange: { [name: string]: string }): Promise<void> {
-    // HACK: Lerna publish does not provide a way to specify multiple package versions via
-    // flags so instead we need to interact with their interactive prompt interface.
-    const PACKAGE_REGISTRY = 'https://registry.npmjs.org/';
-    const child = spawn('lerna', ['publish', `--registry=${PACKAGE_REGISTRY}`], {
-        cwd: constants.monorepoRootPath,
-    });
-    let shouldPrintOutput = false;
-    child.stdout.on('data', (data: Buffer) => {
-        const output = data.toString('utf8');
-        if (shouldPrintOutput) {
-            utils.log(output);
-        }
-        const isVersionPrompt = _.includes(output, 'Select a new version');
-        if (isVersionPrompt) {
-            const outputStripLeft = output.split('new version for ')[1];
-            const packageName = outputStripLeft.split(' ')[0];
-            let versionChange = packageToVersionChange[packageName];
-            const isPrivatePackage = _.isUndefined(versionChange);
-            if (isPrivatePackage) {
-                versionChange = 'patch'; // Always patch updates to private packages.
-            }
-            const semVerIndex = semverNameToIndex[versionChange];
-            child.stdin.write(`${semVerIndex}\n`);
-        }
-        const isFinalPrompt = _.includes(output, 'Are you sure you want to publish the above changes?');
-        if (isFinalPrompt && !IS_DRY_RUN) {
-            child.stdin.write(`y\n`);
-            // After confirmations, we want to print the output to watch the `lerna publish` command
-            shouldPrintOutput = true;
-        } else if (isFinalPrompt && IS_DRY_RUN) {
-            utils.log(
-                `Submitted all versions to Lerna but since this is a dry run, did not confirm. You need to CTRL-C to exit.`,
-            );
-        }
-    });
-    child.stderr.on('data', (data: Buffer) => {
-        const output = data.toString('utf8');
-        utils.log('Stderr:', output);
-    });
+async function lernaPublishAsync(packageToNextVersion: { [name: string]: string }): Promise<void> {
+    const packageVersionString = _.map(packageToNextVersion, (nextVersion: string, packageName: string) => {
+        return `${packageName}@${nextVersion}`;
+    }).join(',');
+    let lernaPublishCmd = `node ${constants.lernaExecutable} publish --cdVersions=${packageVersionString} --registry=${
+        configs.NPM_REGISTRY_URL
+    }`;
+    if (configs.IS_LOCAL_PUBLISH) {
+        lernaPublishCmd += ` --skip-git --yes`;
+    }
+    utils.log('Lerna is publishing...');
+    await execAsync(lernaPublishCmd, { cwd: constants.monorepoRootPath });
 }
 
 function updateVersionNumberIfNeeded(currentVersion: string, proposedNextVersion: string): string {
diff --git a/packages/monorepo-scripts/src/test_installation.ts b/packages/monorepo-scripts/src/test_installation.ts
index 668d4ef1d..a9610e5ee 100644
--- a/packages/monorepo-scripts/src/test_installation.ts
+++ b/packages/monorepo-scripts/src/test_installation.ts
@@ -10,22 +10,27 @@ import { utils } from './utils/utils';
 
 (async () => {
     const monorepoRootPath = path.join(__dirname, '../../..');
-    const packages = utils.getPackages(monorepoRootPath);
+    const packages = utils.getTopologicallySortedPackages(monorepoRootPath);
     const installablePackages = _.filter(
         packages,
         pkg => !pkg.packageJson.private && !_.isUndefined(pkg.packageJson.main) && pkg.packageJson.main.endsWith('.js'),
     );
+    utils.log('Testing packages:');
+    _.map(installablePackages, pkg => utils.log(`* ${pkg.packageJson.name}`));
     for (const installablePackage of installablePackages) {
-        const packagePath = installablePackage.location;
+        const changelogPath = path.join(installablePackage.location, 'CHANGELOG.json');
+        const lastChangelogVersion = JSON.parse(fs.readFileSync(changelogPath).toString())[0].version;
         const packageName = installablePackage.packageJson.name;
         utils.log(`Testing ${packageName}`);
-        let result = await execAsync('npm pack', { cwd: packagePath });
-        const packedPackageFileName = result.stdout.trim();
         const testDirectory = path.join(monorepoRootPath, '../test-env');
         fs.mkdirSync(testDirectory);
-        result = await execAsync('yarn init --yes', { cwd: testDirectory });
-        utils.log(`Installing ${packedPackageFileName}`);
-        result = await execAsync(`yarn add ${packagePath}/${packedPackageFileName}`, { cwd: testDirectory });
+        await execAsync('yarn init --yes', { cwd: testDirectory });
+        const npmrcFilePath = path.join(testDirectory, '.npmrc');
+        fs.writeFileSync(npmrcFilePath, `registry=http://localhost:4873`);
+        utils.log(`Installing ${packageName}@${lastChangelogVersion}`);
+        await execAsync(`npm install --save ${packageName}@${lastChangelogVersion} --registry=http://localhost:4873`, {
+            cwd: testDirectory,
+        });
         const indexFilePath = path.join(testDirectory, 'index.ts');
         fs.writeFileSync(indexFilePath, `import * as Package from '${packageName}';\n`);
         const tsConfig = {
diff --git a/packages/monorepo-scripts/src/types.ts b/packages/monorepo-scripts/src/types.ts
index 9f991c86c..d9e1dfabb 100644
--- a/packages/monorepo-scripts/src/types.ts
+++ b/packages/monorepo-scripts/src/types.ts
@@ -17,14 +17,7 @@ export interface VersionChangelog {
     changes: Change[];
 }
 
-export enum SemVerIndex {
-    Invalid,
-    Patch,
-    Minor,
-    Major,
-}
-
-export interface PackageToVersionChange {
+export interface PackageToNextVersion {
     [name: string]: string;
 }
 
@@ -41,16 +34,18 @@ export interface GitTagsByPackageName {
     [packageName: string]: string[];
 }
 
+export interface PackageJSON {
+    private?: boolean;
+    version: string;
+    name: string;
+    main?: string;
+    scripts?: { [command: string]: string };
+    config?: {
+        additionalTsTypings?: string[];
+    };
+}
+
 export interface Package {
     location: string;
-    packageJson: {
-        private?: boolean;
-        version: string;
-        name: string;
-        main?: string;
-        scripts?: { [command: string]: string };
-        config?: {
-            additionalTsTypings?: string[];
-        };
-    };
+    packageJson: PackageJSON;
 }
diff --git a/packages/monorepo-scripts/src/utils/configs.ts b/packages/monorepo-scripts/src/utils/configs.ts
new file mode 100644
index 000000000..e579bdb7c
--- /dev/null
+++ b/packages/monorepo-scripts/src/utils/configs.ts
@@ -0,0 +1,8 @@
+const IS_LOCAL_PUBLISH = process.env.IS_LOCAL_PUBLISH === 'true';
+const LOCAL_NPM_REGISTRY_URL = 'http://localhost:4873';
+const REMOTE_NPM_REGISTRY_URL = 'https://registry.npmjs.org';
+
+export const configs = {
+    IS_LOCAL_PUBLISH,
+    NPM_REGISTRY_URL: IS_LOCAL_PUBLISH ? LOCAL_NPM_REGISTRY_URL : REMOTE_NPM_REGISTRY_URL,
+};
diff --git a/packages/monorepo-scripts/src/utils/npm_utils.ts b/packages/monorepo-scripts/src/utils/npm_utils.ts
index cc1e046e7..363e31fbb 100644
--- a/packages/monorepo-scripts/src/utils/npm_utils.ts
+++ b/packages/monorepo-scripts/src/utils/npm_utils.ts
@@ -3,19 +3,20 @@ import * as _ from 'lodash';
 
 import { PackageRegistryJson } from '../types';
 
-const NPM_REGISTRY_BASE_URL = 'https://registry.npmjs.org';
+import { configs } from './configs';
+
 const SUCCESS_STATUS = 200;
 const NOT_FOUND_STATUS = 404;
 
 export const npmUtils = {
     async getPackageRegistryJsonIfExistsAsync(packageName: string): Promise<PackageRegistryJson | undefined> {
-        const url = `${NPM_REGISTRY_BASE_URL}/${packageName}`;
+        const url = `${configs.NPM_REGISTRY_URL}/${packageName}`;
         const response = await fetch(url);
 
         if (response.status === NOT_FOUND_STATUS) {
             return undefined;
         } else if (response.status !== SUCCESS_STATUS) {
-            throw new Error(`Request to ${url} failed. Check your internet connection and that npmjs.org is up.`);
+            throw new Error(`Request to ${url} failed. Check your internet connection and that npm registry is up.`);
         }
         const packageRegistryJson = await response.json();
         return packageRegistryJson;
diff --git a/packages/monorepo-scripts/src/utils/utils.ts b/packages/monorepo-scripts/src/utils/utils.ts
index 8ba59c81d..d9bae3ea9 100644
--- a/packages/monorepo-scripts/src/utils/utils.ts
+++ b/packages/monorepo-scripts/src/utils/utils.ts
@@ -1,10 +1,11 @@
+import batchPackages = require('@lerna/batch-packages');
 import * as fs from 'fs';
 import * as _ from 'lodash';
 import { exec as execAsync } from 'promisify-child-process';
 import semver = require('semver');
 
 import { constants } from '../constants';
-import { GitTagsByPackageName, Package, UpdatedPackage } from '../types';
+import { GitTagsByPackageName, Package, PackageJSON, UpdatedPackage } from '../types';
 
 import { changelogUtils } from './changelog_utils';
 
@@ -12,6 +13,15 @@ export const utils = {
     log(...args: any[]): void {
         console.log(...args); // tslint:disable-line:no-console
     },
+    getTopologicallySortedPackages(rootDir: string): Package[] {
+        const packages = utils.getPackages(rootDir);
+        const batchedPackages: PackageJSON[] = _.flatten(batchPackages(_.map(packages, pkg => pkg.packageJson), false));
+        const topsortedPackages: Package[] = _.map(
+            batchedPackages,
+            (pkg: PackageJSON) => _.find(packages, pkg1 => pkg1.packageJson.name === pkg.name) as Package,
+        );
+        return topsortedPackages;
+    },
     getPackages(rootDir: string): Package[] {
         const rootPackageJsonString = fs.readFileSync(`${rootDir}/package.json`, 'utf8');
         const rootPackageJson = JSON.parse(rootPackageJsonString);