Merge branch 'development'

* development: (175 commits)
  small README fixes
  Update docs list in README
  Add manual postpublish command to all public packages and update CHANGELOG.json
  Fix postpublish util to ignore namespace
  Fix release notes bug
  Should print out `lerna publish` stdout so we can see if anything went wrong
  Publish
  Updated CHANGELOGS
  Generate CHANGELOG.json files
  Fix hasty find/replace
  Default to 4sp
  Update moment, no longer need separate moment types
  Move prettify command to utils and also call it on CHANGELOG.md
  Add prettier run on generated CHANGELOG.json and fix scripts
  Remove semi-colons from monorepo-scripts package.json
  Get rid of ; in top-level package.json
  Fix TSLint error
  Make dry-run configurable from top-level package.json
  Improve naming
  Run prettier, update deployer CHANGELOG
  ...

8 files changed, 491 insertions, 6 deletions

diff --git a/packages/monorepo-scripts/src/constants.ts b/packages/monorepo-scripts/src/constants.ts
new file mode 100644
index 000000000..74387a159
--- /dev/null
+++ b/packages/monorepo-scripts/src/constants.ts
@@ -0,0 +1,5 @@
+import * as path from 'path';
+
+export const constants = {
+    monorepoRootPath: path.join(__dirname, '../../..'),
+};
diff --git a/packages/monorepo-scripts/src/convert_changelogs.ts b/packages/monorepo-scripts/src/convert_changelogs.ts
new file mode 100644
index 000000000..b5be14ed8
--- /dev/null
+++ b/packages/monorepo-scripts/src/convert_changelogs.ts
@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+/**
+ * TEMPORARY SCRIPT
+ * This script exists to migrate the legacy CHANGELOG.md to the canonical CHANGELOG.md
+ * TODO: Remove after migration is successful and committed.
+ */
+
+import * as fs from 'fs';
+import lernaGetPackages = require('lerna-get-packages');
+import * as _ from 'lodash';
+import * as moment from 'moment';
+import * as path from 'path';
+import { exec as execAsync } from 'promisify-child-process';
+
+import { constants } from './constants';
+import { Changelog, Changes, UpdatedPackage } from './types';
+import { utils } from './utils';
+
+const HEADER_PRAGMA = '##';
+
+(async () => {
+    const allLernaPackages = lernaGetPackages(constants.monorepoRootPath);
+    const publicLernaPackages = _.filter(allLernaPackages, pkg => !pkg.package.private);
+    for (const lernaPackage of publicLernaPackages) {
+        const changelogMdIfExists = getChangelogMdIfExists(lernaPackage.package.name, lernaPackage.location);
+        if (_.isUndefined(changelogMdIfExists)) {
+            throw new Error(`${lernaPackage.package.name} should have CHANGELOG.md b/c it's public. Add one.`);
+        }
+
+        const lines = changelogMdIfExists.split('\n');
+        const changelogs: Changelog[] = [];
+        let changelog: Changelog = {
+            version: '',
+            changes: [],
+        };
+        /**
+         * Example MD entry:
+         * ## v0.3.1 - _March 18, 2018_
+         *
+         *    * Add TS types for `yargs` (#400)
+         */
+        for (const line of lines) {
+            if (_.startsWith(line, `${HEADER_PRAGMA} `)) {
+                let version = line.substr(4).split(' - ')[0];
+                if (version === '0.x.x') {
+                    version = utils.getNextPatchVersion(lernaPackage.package.version);
+                }
+                const dateStr = line.split('_')[1];
+                let date;
+                if (!_.includes(dateStr, 'TBD')) {
+                    date = moment(dateStr, 'MMMM D, YYYY');
+                }
+                changelog = {
+                    version,
+                    changes: [],
+                };
+                if (!_.isUndefined(date)) {
+                    changelog.timestamp = date.unix();
+                }
+                if (!_.includes(dateStr, 'TBD')) {
+                    changelog.isPublished = true;
+                }
+                changelogs.push(changelog);
+            } else if (_.includes(line, '* ')) {
+                const note = line.split('* ')[1].split(' (#')[0];
+                const prChunk = line.split(' (#')[1];
+                let pr;
+                if (!_.isUndefined(prChunk)) {
+                    pr = prChunk.split(')')[0];
+                }
+                const changes: Changes = {
+                    note,
+                };
+                if (!_.isUndefined(pr)) {
+                    changes.pr = _.parseInt(pr);
+                }
+                changelog.changes.push(changes);
+            }
+        }
+        const changelogJSON = JSON.stringify(changelogs, null, 4);
+        const changelogJSONPath = `${lernaPackage.location}/CHANGELOG.json`;
+        fs.writeFileSync(changelogJSONPath, changelogJSON);
+        await utils.prettifyAsync(changelogJSONPath, constants.monorepoRootPath);
+    }
+})().catch(err => {
+    utils.log(err.stdout);
+    process.exit(1);
+});
+
+function getChangelogMdIfExists(packageName: string, location: string): string | undefined {
+    const changelogPath = path.join(location, 'CHANGELOG.md');
+    let changelogMd: string;
+    try {
+        changelogMd = fs.readFileSync(changelogPath, 'utf-8');
+        return changelogMd;
+    } catch (err) {
+        return undefined;
+    }
+}
diff --git a/packages/monorepo-scripts/src/globals.d.ts b/packages/monorepo-scripts/src/globals.d.ts
index 757ae4097..c5898d0f5 100644
--- a/packages/monorepo-scripts/src/globals.d.ts
+++ b/packages/monorepo-scripts/src/globals.d.ts
@@ -1,8 +1,29 @@
 declare module 'async-child-process';
 declare module 'publish-release';
 declare module 'es6-promisify';
+declare module 'semver-diff';
 
 // semver-sort declarations
 declare module 'semver-sort' {
     const desc: (versions: string[]) => string[];
 }
+
+declare interface LernaPackage {
+    location: string;
+    package: {
+        private?: boolean;
+        version: string;
+        name: string;
+        main?: string;
+        config?: {
+            additionalTsTypings?: string[];
+        };
+    };
+}
+declare function lernaGetPackages(path: string): LernaPackage[];
+// lerna-get-packages declarations
+declare module 'lerna-get-packages' {
+    export = lernaGetPackages;
+}
+
+declare module 'promisify-child-process';
diff --git a/packages/monorepo-scripts/src/postpublish_utils.ts b/packages/monorepo-scripts/src/postpublish_utils.ts
index 898b00c47..236b54379 100644
--- a/packages/monorepo-scripts/src/postpublish_utils.ts
+++ b/packages/monorepo-scripts/src/postpublish_utils.ts
@@ -1,9 +1,12 @@
 import { execAsync } from 'async-child-process';
 import * as promisify from 'es6-promisify';
+import * as fs from 'fs';
 import * as _ from 'lodash';
+import * as path from 'path';
 import * as publishRelease from 'publish-release';
 import semverSort = require('semver-sort');
 
+import { constants } from './constants';
 import { utils } from './utils';
 
 const publishReleaseAsync = promisify(publishRelease);
@@ -88,23 +91,64 @@ export const postpublishUtils = {
         );
     },
     async publishReleaseNotesAsync(cwd: string, packageName: string, version: string, assets: string[]): Promise<void> {
+        const notes = this.getReleaseNotes(packageName);
         const releaseName = this.getReleaseName(packageName, version);
         const tag = this.getTag(packageName, version);
-        utils.log('POSTPUBLISH: Releasing ', releaseName, '...');
         const finalAssets = this.adjustAssetPaths(cwd, assets);
+        utils.log('POSTPUBLISH: Releasing ', releaseName, '...');
         const result = await publishReleaseAsync({
             token: githubPersonalAccessToken,
             owner: '0xProject',
             repo: '0x-monorepo',
             tag,
             name: releaseName,
-            notes: 'N/A',
+            notes,
             draft: false,
             prerelease: false,
             reuseRelease: true,
             reuseDraftOnly: false,
             assets,
         });
+        this.updateChangelogIsPublished(packageName);
+    },
+    getReleaseNotes(packageName: string) {
+        const packageNameWithNamespace = packageName.replace('@0xproject/', '');
+        const changelogJSONPath = path.join(
+            constants.monorepoRootPath,
+            'packages',
+            packageNameWithNamespace,
+            'CHANGELOG.json',
+        );
+        const changelogJSON = fs.readFileSync(changelogJSONPath, 'utf-8');
+        const changelogs = JSON.parse(changelogJSON);
+        const latestLog = changelogs[0];
+        if (_.isUndefined(latestLog.isPublished)) {
+            let notes = '';
+            _.each(latestLog.changes, change => {
+                notes += `* ${change.note}`;
+                if (change.pr) {
+                    notes += ` (${change.pr})`;
+                }
+                notes += `\n`;
+            });
+            return notes;
+        }
+        return 'N/A';
+    },
+    updateChangelogIsPublished(packageName: string) {
+        const packageNameWithNamespace = packageName.replace('@0xproject/', '');
+        const changelogJSONPath = path.join(
+            constants.monorepoRootPath,
+            'packages',
+            packageNameWithNamespace,
+            'CHANGELOG.json',
+        );
+        const changelogJSON = fs.readFileSync(changelogJSONPath, 'utf-8');
+        const changelogs = JSON.parse(changelogJSON);
+        const latestLog = changelogs[0];
+        latestLog.isPublished = true;
+        changelogs[0] = latestLog;
+        fs.writeFileSync(changelogJSONPath, JSON.stringify(changelogs, null, '\t'));
     },
     getTag(packageName: string, version: string) {
         return `${packageName}@${version}`;
@@ -122,14 +166,16 @@ export const postpublishUtils = {
     },
     adjustFileIncludePaths(fileIncludes: string[], cwd: string): string[] {
         const fileIncludesAdjusted = _.map(fileIncludes, fileInclude => {
-            let path = _.startsWith(fileInclude, './') ? `${cwd}/${fileInclude.substr(2)}` : `${cwd}/${fileInclude}`;
+            let includePath = _.startsWith(fileInclude, './')
+                ? `${cwd}/${fileInclude.substr(2)}`
+                : `${cwd}/${fileInclude}`;
 
             // HACK: tsconfig.json needs wildcard directory endings as `/**/*`
             // but TypeDoc needs it as `/**` in order to pick up files at the root
-            if (_.endsWith(path, '/**/*')) {
-                path = path.slice(0, -2);
+            if (_.endsWith(includePath, '/**/*')) {
+                includePath = includePath.slice(0, -2);
             }
-            return path;
+            return includePath;
         });
         return fileIncludesAdjusted;
     },
diff --git a/packages/monorepo-scripts/src/publish.ts b/packages/monorepo-scripts/src/publish.ts
new file mode 100644
index 000000000..d749ec630
--- /dev/null
+++ b/packages/monorepo-scripts/src/publish.ts
@@ -0,0 +1,217 @@
+#!/usr/bin/env node
+
+import * as fs from 'fs';
+import lernaGetPackages = require('lerna-get-packages');
+import * as _ from 'lodash';
+import * as moment from 'moment';
+import * as path from 'path';
+import { exec as execAsync, spawn } from 'promisify-child-process';
+import semverDiff = require('semver-diff');
+import semverSort = require('semver-sort');
+
+import { constants } from './constants';
+import { Changelog, Changes, SemVerIndex, UpdatedPackage } from './types';
+import { utils } from './utils';
+
+const IS_DRY_RUN = process.env.IS_DRY_RUN === 'true';
+const TODAYS_TIMESTAMP = moment().unix();
+const LERNA_EXECUTABLE = './node_modules/lerna/bin/lerna.js';
+const semverNameToIndex: { [semver: string]: number } = {
+    patch: SemVerIndex.Patch,
+    minor: SemVerIndex.Minor,
+    major: SemVerIndex.Major,
+};
+
+(async () => {
+    const updatedPublicPackages = await getPublicLernaUpdatedPackagesAsync();
+    const updatedPackageNames = _.map(updatedPublicPackages, pkg => pkg.name);
+
+    const allLernaPackages = lernaGetPackages(constants.monorepoRootPath);
+    const updatedPublicLernaPackages = _.filter(allLernaPackages, pkg => {
+        return _.includes(updatedPackageNames, pkg.package.name);
+    });
+    const updatedPublicLernaPackageNames = _.map(updatedPublicLernaPackages, pkg => pkg.package.name);
+    utils.log(`Will update CHANGELOGs and publish: \n${updatedPublicLernaPackageNames.join('\n')}\n`);
+
+    const packageToVersionChange: { [name: string]: string } = {};
+    for (const lernaPackage of updatedPublicLernaPackages) {
+        const packageName = lernaPackage.package.name;
+        const changelogJSONPath = path.join(lernaPackage.location, 'CHANGELOG.json');
+        const changelogJSON = getChangelogJSONOrCreateIfMissing(lernaPackage.package.name, changelogJSONPath);
+        let changelogs: Changelog[];
+        try {
+            changelogs = JSON.parse(changelogJSON);
+        } catch (err) {
+            throw new Error(
+                `${lernaPackage.package.name}'s CHANGELOG.json contains invalid JSON. Please fix and try again.`,
+            );
+        }
+
+        const currentVersion = lernaPackage.package.version;
+        const shouldAddNewEntry = shouldAddNewChangelogEntry(changelogs);
+        if (shouldAddNewEntry) {
+            // Create a new entry for a patch version with generic changelog entry.
+            const nextPatchVersion = utils.getNextPatchVersion(currentVersion);
+            const newChangelogEntry: Changelog = {
+                timestamp: TODAYS_TIMESTAMP,
+                version: nextPatchVersion,
+                changes: [
+                    {
+                        note: 'Dependencies updated',
+                    },
+                ],
+            };
+            changelogs = [newChangelogEntry, ...changelogs];
+            packageToVersionChange[packageName] = semverDiff(currentVersion, nextPatchVersion);
+        } else {
+            // Update existing entry with timestamp
+            const lastEntry = changelogs[0];
+            if (_.isUndefined(lastEntry.timestamp)) {
+                lastEntry.timestamp = TODAYS_TIMESTAMP;
+            }
+            // Check version number is correct.
+            const proposedNextVersion = lastEntry.version;
+            lastEntry.version = updateVersionNumberIfNeeded(currentVersion, proposedNextVersion);
+            changelogs[0] = lastEntry;
+            packageToVersionChange[packageName] = semverDiff(currentVersion, lastEntry.version);
+        }
+
+        // Save updated CHANGELOG.json
+        fs.writeFileSync(changelogJSONPath, JSON.stringify(changelogs, null, 4));
+        await utils.prettifyAsync(changelogJSONPath, constants.monorepoRootPath);
+        utils.log(`${packageName}: Updated CHANGELOG.json`);
+        // Generate updated CHANGELOG.md
+        const changelogMd = generateChangelogMd(changelogs);
+        const changelogMdPath = path.join(lernaPackage.location, 'CHANGELOG.md');
+        fs.writeFileSync(changelogMdPath, changelogMd);
+        await utils.prettifyAsync(changelogMdPath, constants.monorepoRootPath);
+        utils.log(`${packageName}: Updated CHANGELOG.md`);
+    }
+
+    if (!IS_DRY_RUN) {
+        await execAsync(`git add . --all`, { cwd: constants.monorepoRootPath });
+        await execAsync(`git commit -m "Updated CHANGELOGS"`, { cwd: constants.monorepoRootPath });
+        await execAsync(`git push`, { cwd: constants.monorepoRootPath });
+        utils.log(`Pushed CHANGELOG updates to Github`);
+    }
+
+    utils.log('Version updates to apply:');
+    _.each(packageToVersionChange, (versionChange: string, packageName: string) => {
+        utils.log(`${packageName} -> ${versionChange}`);
+    });
+    utils.log(`Calling 'lerna publish'...`);
+    await lernaPublishAsync(packageToVersionChange);
+})().catch(err => {
+    utils.log(err);
+    process.exit(1);
+});
+
+async function lernaPublishAsync(packageToVersionChange: { [name: string]: string }) {
+    // HACK: Lerna publish does not provide a way to specify multiple package versions via
+    // flags so instead we need to interact with their interactive prompt interface.
+    const child = spawn('lerna', ['publish', '--registry=https://registry.npmjs.org/'], {
+        cwd: constants.monorepoRootPath,
+    });
+    let shouldPrintOutput = false;
+    child.stdout.on('data', (data: Buffer) => {
+        const output = data.toString('utf8');
+        if (shouldPrintOutput) {
+            utils.log(output);
+        }
+        const isVersionPrompt = _.includes(output, 'Select a new version');
+        if (isVersionPrompt) {
+            const outputStripLeft = output.split('new version for ')[1];
+            const packageName = outputStripLeft.split(' ')[0];
+            let versionChange = packageToVersionChange[packageName];
+            const isPrivatePackage = _.isUndefined(versionChange);
+            if (isPrivatePackage) {
+                versionChange = 'patch'; // Always patch updates to private packages.
+            }
+            const semVerIndex = semverNameToIndex[versionChange];
+            child.stdin.write(`${semVerIndex}\n`);
+        }
+        const isFinalPrompt = _.includes(output, 'Are you sure you want to publish the above changes?');
+        if (isFinalPrompt && !IS_DRY_RUN) {
+            child.stdin.write(`y\n`);
+            // After confirmations, we want to print the output to watch the `lerna publish` command
+            shouldPrintOutput = true;
+        } else if (isFinalPrompt && IS_DRY_RUN) {
+            utils.log(
+                `Submitted all versions to Lerna but since this is a dry run, did not confirm. You need to CTRL-C to exit.`,
+            );
+        }
+    });
+}
+
+async function getPublicLernaUpdatedPackagesAsync(): Promise<UpdatedPackage[]> {
+    const result = await execAsync(`${LERNA_EXECUTABLE} updated --json`, { cwd: constants.monorepoRootPath });
+    const updatedPackages = JSON.parse(result.stdout);
+    const updatedPublicPackages = _.filter(updatedPackages, updatedPackage => !updatedPackage.private);
+    return updatedPublicPackages;
+}
+
+function updateVersionNumberIfNeeded(currentVersion: string, proposedNextVersion: string) {
+    if (proposedNextVersion === currentVersion) {
+        return utils.getNextPatchVersion(currentVersion);
+    }
+    const sortedVersions = semverSort.desc([proposedNextVersion, currentVersion]);
+    if (sortedVersions[0] !== proposedNextVersion) {
+        return utils.getNextPatchVersion(currentVersion);
+    }
+    return proposedNextVersion;
+}
+
+function getChangelogJSONOrCreateIfMissing(packageName: string, changelogPath: string): string {
+    let changelogJSON: string;
+    try {
+        changelogJSON = fs.readFileSync(changelogPath, 'utf-8');
+        return changelogJSON;
+    } catch (err) {
+        // If none exists, create new, empty one.
+        const emptyChangelogJSON = JSON.stringify([], null, 4);
+        fs.writeFileSync(changelogPath, emptyChangelogJSON);
+        return emptyChangelogJSON;
+    }
+}
+
+function shouldAddNewChangelogEntry(changelogs: Changelog[]): boolean {
+    if (_.isEmpty(changelogs)) {
+        return true;
+    }
+    const lastEntry = changelogs[0];
+    return !!lastEntry.isPublished;
+}
+
+function generateChangelogMd(changelogs: Changelog[]): string {
+    let changelogMd = `<!--
+This file is auto-generated using the monorepo-scripts package. Don't edit directly.
+Edit the package's CHANGELOG.json file only.
+-->
+
+CHANGELOG
+    `;
+
+    _.each(changelogs, changelog => {
+        if (_.isUndefined(changelog.timestamp)) {
+            throw new Error(
+                'All CHANGELOG.json entries must be updated to include a timestamp before generating their MD version',
+            );
+        }
+        const date = moment(`${changelog.timestamp}`, 'X').format('MMMM D, YYYY');
+        const title = `\n## v${changelog.version} - _${date}_\n\n`;
+        changelogMd += title;
+
+        let changes = '';
+        _.each(changelog.changes, change => {
+            let line = `    * ${change.note}`;
+            if (!_.isUndefined(change.pr)) {
+                line += ` (#${change.pr})`;
+            }
+            line += '\n';
+            changes += line;
+        });
+        changelogMd += `${changes}`;
+    });
+
+    return changelogMd;
+}
diff --git a/packages/monorepo-scripts/src/test_installation.ts b/packages/monorepo-scripts/src/test_installation.ts
new file mode 100644
index 000000000..195b64b2a
--- /dev/null
+++ b/packages/monorepo-scripts/src/test_installation.ts
@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+
+import * as fs from 'fs';
+import lernaGetPackages = require('lerna-get-packages');
+import * as _ from 'lodash';
+import * as path from 'path';
+import { exec as execAsync } from 'promisify-child-process';
+import * as rimraf from 'rimraf';
+
+import { utils } from './utils';
+
+(async () => {
+    const monorepoRootPath = path.join(__dirname, '../../..');
+    const lernaPackages = lernaGetPackages(monorepoRootPath);
+    const installablePackages = _.filter(
+        lernaPackages,
+        lernaPackage =>
+            !lernaPackage.package.private &&
+            !_.isUndefined(lernaPackage.package.main) &&
+            lernaPackage.package.main.endsWith('.js'),
+    );
+    for (const installableLernaPackage of installablePackages) {
+        const packagePath = installableLernaPackage.location;
+        const packageName = installableLernaPackage.package.name;
+        utils.log(`Testing ${packageName}`);
+        let result = await execAsync('npm pack', { cwd: packagePath });
+        const packedPackageFileName = result.stdout.trim();
+        const testDirectory = path.join(monorepoRootPath, '../test-env');
+        fs.mkdirSync(testDirectory);
+        result = await execAsync('yarn init --yes', { cwd: testDirectory });
+        utils.log(`Installing ${packedPackageFileName}`);
+        result = await execAsync(`yarn add ${packagePath}/${packedPackageFileName}`, { cwd: testDirectory });
+        const indexFilePath = path.join(testDirectory, 'index.ts');
+        fs.writeFileSync(indexFilePath, `import * as Package from '${packageName}'`);
+        const tsConfig = {
+            compilerOptions: {
+                typeRoots: ['node_modules/@0xproject/typescript-typings/types', 'node_modules/@types'],
+                module: 'commonjs',
+                target: 'es5',
+                lib: ['es2017', 'dom'],
+                declaration: true,
+                noImplicitReturns: true,
+                pretty: true,
+                strict: true,
+            },
+            include: ['index.ts'],
+        };
+        const tsconfigFilePath = path.join(testDirectory, 'tsconfig.json');
+        fs.writeFileSync(tsconfigFilePath, JSON.stringify(tsConfig, null, 4));
+        utils.log(`Compiling ${packageName}`);
+        await execAsync('../node_modules/typescript/bin/tsc', { cwd: testDirectory });
+        utils.log(`Successfully compiled with ${packageName} as a dependency`);
+        rimraf.sync(testDirectory);
+    }
+})().catch(err => {
+    utils.log(err.stdout);
+    process.exit(1);
+});
diff --git a/packages/monorepo-scripts/src/types.ts b/packages/monorepo-scripts/src/types.ts
new file mode 100644
index 000000000..7adec202f
--- /dev/null
+++ b/packages/monorepo-scripts/src/types.ts
@@ -0,0 +1,24 @@
+export interface UpdatedPackage {
+    name: string;
+    version: string;
+    private: boolean;
+}
+
+export interface Changes {
+    note: string;
+    pr?: number;
+}
+
+export interface Changelog {
+    timestamp?: number;
+    version: string;
+    changes: Changes[];
+    isPublished?: boolean;
+}
+
+export enum SemVerIndex {
+    Invalid,
+    Patch,
+    Minor,
+    Major,
+}
diff --git a/packages/monorepo-scripts/src/utils.ts b/packages/monorepo-scripts/src/utils.ts
index 5423cabd9..9aa37e272 100644
--- a/packages/monorepo-scripts/src/utils.ts
+++ b/packages/monorepo-scripts/src/utils.ts
@@ -1,5 +1,20 @@
+import * as _ from 'lodash';
+import { exec as execAsync, spawn } from 'promisify-child-process';
+
 export const utils = {
     log(...args: any[]): void {
         console.log(...args); // tslint:disable-line:no-console
     },
+    getNextPatchVersion(currentVersion: string): string {
+        const versionSegments = currentVersion.split('.');
+        const patch = _.parseInt(_.last(versionSegments) as string);
+        const newPatch = patch + 1;
+        const newPatchVersion = `${versionSegments[0]}.${versionSegments[1]}.${newPatch}`;
+        return newPatchVersion;
+    },
+    async prettifyAsync(filePath: string, cwd: string) {
+        await execAsync(`prettier --write ${filePath} --config .prettierrc`, {
+            cwd,
+        });
+    },
 };