Merge pull request #541 from 0xProject/improveMonorepoScripts

Improve Monorepo-scripts

7 files changed, 174 insertions, 36 deletions

diff --git a/packages/monorepo-scripts/README.md b/packages/monorepo-scripts/README.md
index a740ba28a..cec77a10d 100644
--- a/packages/monorepo-scripts/README.md
+++ b/packages/monorepo-scripts/README.md
@@ -2,6 +2,16 @@
 
 This repository contains a few helpful scripts for working with this mono repo.
 
+#### Scripts
+
+**`yarn deps_versions`**: Since we use Lerna + Yarn workspaces, shared dependencies between packages in the monorepo get hoisted to a top-level `node_modules` directory. If two packages use different versions of the same dependency however, both get installed. To avoid having many versions of a dependency installed, we try to keep dependency versions the same across packages in the monorepo. This script will list any dependencies for which we have multiple versions installed. We can then go through them and try to consolidate to a single version where possible.
+
+**`yarn find_unused_deps`**: Sometimes we accidentally leave dependencies listed in `package.json` that are no longer being used. This script finds potential dependencies that might no longer be in use. Please verify that it is no longer in use before removing, the `depcheck` package we use under-the-hood doesn't handle some TS quirks perfectly.
+
+**`yarn remove_tags`**: Our publishing script calls `lerna publish` under-the-hood. If this command fails, it might have created new versioned git tags for each package. Removing these manually is tedious, so you can also run this command instead. Before doing so, check to see if `lerna` already created the publish commit. If so, first revert that with `git reset --hard HEAD~1`, then run this command.
+
+**`yarn test:publish`**: Execute a test-run of the publish script. This dry run won't actually publish, nor will it commit/push anything to Github.
+
 ## Usage
 
 #### Dependency versions
diff --git a/packages/monorepo-scripts/package.json b/packages/monorepo-scripts/package.json
index 8bf1d83e5..733a099d1 100644
--- a/packages/monorepo-scripts/package.json
+++ b/packages/monorepo-scripts/package.json
@@ -11,9 +11,11 @@
         "build": "tsc",
         "test:publish": "run-s build script:publish",
         "find_unused_deps": "run-s build script:find_unused_deps",
+        "remove_tags": "run-s build script:remove_tags",
         "script:deps_versions": "node ./lib/deps_versions.js",
         "script:publish": "IS_DRY_RUN=true node ./lib/publish.js",
-        "script:find_unused_deps": "node ./lib/find_unused_dependencies.js"
+        "script:find_unused_deps": "node ./lib/find_unused_dependencies.js",
+        "script:remove_tags": "node ./lib/remove_tags.js"
     },
     "repository": {
         "type": "git",
diff --git a/packages/monorepo-scripts/src/constants.ts b/packages/monorepo-scripts/src/constants.ts
index 081a49332..3aaf881cb 100644
--- a/packages/monorepo-scripts/src/constants.ts
+++ b/packages/monorepo-scripts/src/constants.ts
@@ -3,4 +3,6 @@ import * as path from 'path';
 export const constants = {
     monorepoRootPath: path.join(__dirname, '../../..'),
     stagingWebsite: 'http://staging-0xproject.s3-website-us-east-1.amazonaws.com',
+    lernaExecutable: path.join('node_modules', 'lerna', 'bin', 'lerna.js'),
+    githubPersonalAccessToken: process.env.GITHUB_PERSONAL_ACCESS_TOKEN_0X_JS,
 };
diff --git a/packages/monorepo-scripts/src/postpublish_utils.ts b/packages/monorepo-scripts/src/postpublish_utils.ts
index ca4c92f5d..df2bcb128 100644
--- a/packages/monorepo-scripts/src/postpublish_utils.ts
+++ b/packages/monorepo-scripts/src/postpublish_utils.ts
@@ -10,7 +10,6 @@ import { constants } from './constants';
 import { utils } from './utils';
 
 const publishReleaseAsync = promisify(publishRelease);
-const githubPersonalAccessToken = process.env.GITHUB_PERSONAL_ACCESS_TOKEN_0X_JS;
 const generatedDocsDirectoryName = 'generated_docs';
 
 export interface PostpublishConfigs {
@@ -97,7 +96,7 @@ export const postpublishUtils = {
         const finalAssets = this.adjustAssetPaths(cwd, assets);
         utils.log('POSTPUBLISH: Releasing ', releaseName, '...');
         const result = await publishReleaseAsync({
-            token: githubPersonalAccessToken,
+            token: constants.githubPersonalAccessToken,
             owner: '0xProject',
             repo: '0x-monorepo',
             tag,
diff --git a/packages/monorepo-scripts/src/publish.ts b/packages/monorepo-scripts/src/publish.ts
index a2d641ff9..0ca3ea7a1 100644
--- a/packages/monorepo-scripts/src/publish.ts
+++ b/packages/monorepo-scripts/src/publish.ts
@@ -38,8 +38,14 @@ const packageNameToWebsitePath: { [name: string]: string } = {
 };
 
 (async () => {
+    const hasRequiredSetup = await checkPublishRequiredSetupAsync();
+    if (!hasRequiredSetup) {
+        return; // abort
+    }
+
     // Fetch public, updated Lerna packages
-    const updatedPublicLernaPackages = await getUpdatedPublicLernaPackagesAsync();
+    const shouldIncludePrivate = false;
+    const updatedPublicLernaPackages = await utils.getUpdatedLernaPackagesAsync(shouldIncludePrivate);
 
     await confirmDocPagesRenderAsync(updatedPublicLernaPackages);
 
@@ -107,6 +113,54 @@ package.ts. Please add an entry for it and try again.`,
     }
 }
 
+async function checkPublishRequiredSetupAsync(): Promise<boolean> {
+    // check to see if logged into npm before publishing
+    try {
+        // HACK: for some reason on some setups, the `npm whoami` will not recognize a logged-in user
+        // unless run with `sudo` (i.e Fabio's NVM setup) but is fine for others (Jacob's N setup).
+        await execAsync(`sudo npm whoami`);
+    } catch (err) {
+        utils.log('You must be logged into npm in the commandline to publish. Run `npm login` and try again.');
+        return false;
+    }
+
+    // Check to see if Git personal token setup
+    if (_.isUndefined(constants.githubPersonalAccessToken)) {
+        utils.log(
+            'You must have a Github personal access token set to an envVar named `GITHUB_PERSONAL_ACCESS_TOKEN_0X_JS`. Add it then try again.',
+        );
+        return false;
+    }
+
+    // Check NPM version is 5.X
+    const result = await execAsync(`npm --version`);
+    const version = result.stdout;
+    const versionSegments = version.split('.');
+    const majorVersion = _.parseInt(versionSegments[0]);
+    if (majorVersion < 5) {
+        utils.log('You npm version must be v5.x or higher. Upgrade your npm and try again.');
+        return false;
+    }
+
+    // Check that `aws` commandline tool is installed
+    try {
+        await execAsync(`aws help`);
+    } catch (err) {
+        utils.log('You must have `awscli` commandline tool installed. Install it and try again.');
+        return false;
+    }
+
+    // Check that `aws` credentials are setup
+    try {
+        await execAsync(`aws sts get-caller-identity`);
+    } catch (err) {
+        utils.log('You must setup your AWS credentials by running `aws configure`. Do this and try again.');
+        return false;
+    }
+
+    return true;
+}
+
 async function pushChangelogsToGithubAsync() {
     await execAsync(`git add . --all`, { cwd: constants.monorepoRootPath });
     await execAsync(`git commit -m "Updated CHANGELOGS"`, { cwd: constants.monorepoRootPath });
@@ -114,23 +168,12 @@ async function pushChangelogsToGithubAsync() {
     utils.log(`Pushed CHANGELOG updates to Github`);
 }
 
-async function getUpdatedPublicLernaPackagesAsync(): Promise<LernaPackage[]> {
-    const updatedPublicPackages = await getPublicLernaUpdatedPackagesAsync();
-    const updatedPackageNames = _.map(updatedPublicPackages, pkg => pkg.name);
-
-    const allLernaPackages = lernaGetPackages(constants.monorepoRootPath);
-    const updatedPublicLernaPackages = _.filter(allLernaPackages, pkg => {
-        return _.includes(updatedPackageNames, pkg.package.name);
-    });
-    return updatedPublicLernaPackages;
-}
-
 async function updateChangeLogsAsync(updatedPublicLernaPackages: LernaPackage[]): Promise<PackageToVersionChange> {
     const packageToVersionChange: PackageToVersionChange = {};
     for (const lernaPackage of updatedPublicLernaPackages) {
         const packageName = lernaPackage.package.name;
         const changelogJSONPath = path.join(lernaPackage.location, 'CHANGELOG.json');
-        const changelogJSON = getChangelogJSONOrCreateIfMissing(lernaPackage.package.name, changelogJSONPath);
+        const changelogJSON = utils.getChangelogJSONOrCreateIfMissing(changelogJSONPath);
         let changelogs: Changelog[];
         try {
             changelogs = JSON.parse(changelogJSON);
@@ -225,13 +268,6 @@ async function lernaPublishAsync(packageToVersionChange: { [name: string]: strin
     });
 }
 
-async function getPublicLernaUpdatedPackagesAsync(): Promise<UpdatedPackage[]> {
-    const result = await execAsync(`${LERNA_EXECUTABLE} updated --json`, { cwd: constants.monorepoRootPath });
-    const updatedPackages = JSON.parse(result.stdout);
-    const updatedPublicPackages = _.filter(updatedPackages, updatedPackage => !updatedPackage.private);
-    return updatedPublicPackages;
-}
-
 function updateVersionNumberIfNeeded(currentVersion: string, proposedNextVersion: string) {
     if (proposedNextVersion === currentVersion) {
         return utils.getNextPatchVersion(currentVersion);
@@ -243,19 +279,6 @@ function updateVersionNumberIfNeeded(currentVersion: string, proposedNextVersion
     return proposedNextVersion;
 }
 
-function getChangelogJSONOrCreateIfMissing(packageName: string, changelogPath: string): string {
-    let changelogJSON: string;
-    try {
-        changelogJSON = fs.readFileSync(changelogPath, 'utf-8');
-        return changelogJSON;
-    } catch (err) {
-        // If none exists, create new, empty one.
-        const emptyChangelogJSON = JSON.stringify([], null, 4);
-        fs.writeFileSync(changelogPath, emptyChangelogJSON);
-        return emptyChangelogJSON;
-    }
-}
-
 function shouldAddNewChangelogEntry(currentVersion: string, changelogs: Changelog[]): boolean {
     if (_.isEmpty(changelogs)) {
         return true;
diff --git a/packages/monorepo-scripts/src/remove_tags.ts b/packages/monorepo-scripts/src/remove_tags.ts
new file mode 100644
index 000000000..6d09729c7
--- /dev/null
+++ b/packages/monorepo-scripts/src/remove_tags.ts
@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+
+import lernaGetPackages = require('lerna-get-packages');
+import * as _ from 'lodash';
+import * as path from 'path';
+import { exec as execAsync } from 'promisify-child-process';
+import semverSort = require('semver-sort');
+
+import { constants } from './constants';
+import { Changelog } from './types';
+import { utils } from './utils';
+
+(async () => {
+    const shouldIncludePrivate = true;
+    const updatedPublicLernaPackages = await utils.getUpdatedLernaPackagesAsync(shouldIncludePrivate);
+
+    for (const lernaPackage of updatedPublicLernaPackages) {
+        const packageName = lernaPackage.package.name;
+        const currentVersion = lernaPackage.package.version;
+        const changelogJSONPath = path.join(lernaPackage.location, 'CHANGELOG.json');
+        // Private packages don't have changelogs, and their versions are always incremented
+        // by a patch version.
+        const changelogJSONIfExists = utils.getChangelogJSONIfExists(changelogJSONPath);
+
+        let latestChangelogVersion: string;
+        if (!_.isUndefined(changelogJSONIfExists)) {
+            let changelogs: Changelog[];
+            try {
+                changelogs = JSON.parse(changelogJSONIfExists);
+            } catch (err) {
+                throw new Error(
+                    `${lernaPackage.package.name}'s CHANGELOG.json contains invalid JSON. Please fix and try again.`,
+                );
+            }
+            latestChangelogVersion = changelogs[0].version;
+        } else {
+            latestChangelogVersion = utils.getNextPatchVersion(currentVersion);
+        }
+
+        const sortedVersions = semverSort.desc([latestChangelogVersion, currentVersion]);
+        if (sortedVersions[0] === latestChangelogVersion && latestChangelogVersion !== currentVersion) {
+            const tagName = `${packageName}@${latestChangelogVersion}`;
+            try {
+                await execAsync(`git tag -d ${tagName}`, { cwd: constants.monorepoRootPath });
+                utils.log(`removed tag: ${tagName}`);
+            } catch (err) {
+                if (_.includes(err.message, 'not found')) {
+                    utils.log(`Could not find tag: ${tagName}`);
+                } else {
+                    throw err;
+                }
+            }
+        }
+    }
+})().catch(err => {
+    utils.log(err);
+    process.exit(1);
+});
diff --git a/packages/monorepo-scripts/src/utils.ts b/packages/monorepo-scripts/src/utils.ts
index 9aa37e272..4412f753a 100644
--- a/packages/monorepo-scripts/src/utils.ts
+++ b/packages/monorepo-scripts/src/utils.ts
@@ -1,6 +1,11 @@
+import * as fs from 'fs';
+import lernaGetPackages = require('lerna-get-packages');
 import * as _ from 'lodash';
 import { exec as execAsync, spawn } from 'promisify-child-process';
 
+import { constants } from './constants';
+import { UpdatedPackage } from './types';
+
 export const utils = {
     log(...args: any[]): void {
         console.log(...args); // tslint:disable-line:no-console
@@ -17,4 +22,43 @@ export const utils = {
             cwd,
         });
     },
+    async getUpdatedLernaPackagesAsync(shouldIncludePrivate: boolean): Promise<LernaPackage[]> {
+        const updatedPublicPackages = await this.getLernaUpdatedPackagesAsync(shouldIncludePrivate);
+        const updatedPackageNames = _.map(updatedPublicPackages, pkg => pkg.name);
+
+        const allLernaPackages = lernaGetPackages(constants.monorepoRootPath);
+        const updatedPublicLernaPackages = _.filter(allLernaPackages, pkg => {
+            return _.includes(updatedPackageNames, pkg.package.name);
+        });
+        return updatedPublicLernaPackages;
+    },
+    async getLernaUpdatedPackagesAsync(shouldIncludePrivate: boolean): Promise<UpdatedPackage[]> {
+        const result = await execAsync(`${constants.lernaExecutable} updated --json`, {
+            cwd: constants.monorepoRootPath,
+        });
+        const updatedPackages = JSON.parse(result.stdout);
+        if (!shouldIncludePrivate) {
+            const updatedPublicPackages = _.filter(updatedPackages, updatedPackage => !updatedPackage.private);
+            return updatedPublicPackages;
+        }
+        return updatedPackages;
+    },
+    getChangelogJSONIfExists(changelogPath: string) {
+        try {
+            const changelogJSON = fs.readFileSync(changelogPath, 'utf-8');
+            return changelogJSON;
+        } catch (err) {
+            return undefined;
+        }
+    },
+    getChangelogJSONOrCreateIfMissing(changelogPath: string): string {
+        const changelogIfExists = this.getChangelogJSONIfExists(changelogPath);
+        if (_.isUndefined(changelogIfExists)) {
+            // If none exists, create new, empty one.
+            const emptyChangelogJSON = JSON.stringify([]);
+            fs.writeFileSync(changelogPath, emptyChangelogJSON);
+            return emptyChangelogJSON;
+        }
+        return changelogIfExists;
+    },
 };