Use bash for loop for generating docs for each package, revert changes to script

7 files changed, 67 insertions, 67 deletions

diff --git a/.circleci/config.yml b/.circleci/config.yml
index ae8519a56..844eb3b98 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -66,6 +66,15 @@ jobs:
           keys:
             - repo-{{ .Environment.CIRCLE_SHA1 }}
       - run: yarn test:publish:circleci
+  test-doc-generation:
+    docker:
+      - image: circleci/node:9
+    working_directory: ~/repo
+    steps:
+      - restore_cache:
+          keys:
+            - repo-{{ .Environment.CIRCLE_SHA1 }}
+      - run: yarn test:generate_docs:circleci
   test-rest:
     docker:
       - image: circleci/node:9
@@ -237,6 +246,9 @@ workflows:
       - test-publish:
           requires:
             - build
+      - test-doc-generation:
+          requires:
+            - build
       - submit-coverage:
           requires:
             - test-rest
\ No newline at end of file
diff --git a/package.json b/package.json
index a86bbd42a..eca1fbe60 100644
--- a/package.json
+++ b/package.json
@@ -32,12 +32,14 @@
         "rebuild": "run-s clean build",
         "rebuild:no_website": "run-s clean build:no_website",
         "test": "wsrun test $PKG --fast-exit --serial --exclude-missing",
-        "generate_docs": "node ./packages/monorepo-scripts/lib/doc_generate_and_upload.js",
+        "generate_doc": "node ./packages/monorepo-scripts/lib/doc_generate_and_upload.js",
+        "test:generate_docs:circleci": "for i in ${npm_package_config_packagesWithDocPages}; do yarn generate_doc --package $i --shouldUpload false --isStaging true || break; done;",
         "lint": "wsrun lint $PKG --fast-exit --parallel --exclude-missing",
         "comment:postinstall": "HACK: For some reason `yarn` is not setting up symlinks properly for order-utils. We temporarily set them manually. Remove this after V2 refactor is complete."
     },
     "config": {
-        "mnemonic": "concert load couple harbor equip island argue ramp clarify fence smart topic"
+        "mnemonic": "concert load couple harbor equip island argue ramp clarify fence smart topic",
+        "packagesWithDocPages": "0x.js connect json-schemas subproviders web3-wrapper contract-wrappers order-utils order-watcher sol-compiler sol-cov ethereum-types"
     },
     "devDependencies": {
         "@0x-lerna-fork/lerna": "3.0.0-beta.23",
diff --git a/packages/monorepo-scripts/src/doc_gen_configs.ts b/packages/monorepo-scripts/src/doc_gen_configs.ts
index a7314776e..52cbc4f59 100644
--- a/packages/monorepo-scripts/src/doc_gen_configs.ts
+++ b/packages/monorepo-scripts/src/doc_gen_configs.ts
@@ -2,20 +2,6 @@ import { DocGenConfigs } from './types';
 
 export const docGenConfigs: DocGenConfigs = {
     DOC_JSON_VERSION: '0.0.1',
-    // Packages for which doc pages exist
-    PACKAGES_WITH_DOC_PAGES: [
-        '0x.js',
-        'connect',
-        'json-schemas',
-        'subproviders',
-        'web3-wrapper',
-        'contract-wrappers',
-        'order-utils',
-        'order-watcher',
-        'sol-compiler',
-        'sol-cov',
-        'ethereum-types',
-    ];
     EXTERNAL_TYPE_TO_LINK: {
         Array: 'https://developer.mozilla.org/pt-PT/docs/Web/JavaScript/Reference/Global_Objects/Array',
         BigNumber: 'http://mikemcl.github.io/bignumber.js',
diff --git a/packages/monorepo-scripts/src/doc_generate_and_upload.ts b/packages/monorepo-scripts/src/doc_generate_and_upload.ts
index 7b6f2d9f0..2a53628b5 100644
--- a/packages/monorepo-scripts/src/doc_generate_and_upload.ts
+++ b/packages/monorepo-scripts/src/doc_generate_and_upload.ts
@@ -5,9 +5,9 @@ import { DocGenerateAndUploadUtils } from './utils/doc_generate_and_upload_utils
 const args = yargs
     .option('package', {
         describe:
-            'Monorepo sub-package for which to generate DocJSON. If not supplied, it will do all defined in docGenConfigs.',
+            'Monorepo sub-package for which to generate DocJSON',
         type: 'string',
-        demandOption: false,
+        demandOption: true,
     })
     .option('isStaging', {
         describe: 'Whether we wish to publish docs to staging or production',
@@ -23,10 +23,15 @@ const args = yargs
     .example("$0 --package '0x.js' --isStaging true", 'Full usage example').argv;
 
 (async () => {
-    const packageNameIfExists = args.package;
+    const packageName = args.package;
     const isStaging = args.isStaging;
     const shouldUploadDocs = args.shouldUpload;
 
-    const docGenerateAndUploadUtils = new DocGenerateAndUploadUtils(packageNameIfExists, isStaging, shouldUploadDocs);
+    const docGenerateAndUploadUtils = new DocGenerateAndUploadUtils(packageName, isStaging, shouldUploadDocs);
     await docGenerateAndUploadUtils.generateAndUploadDocsAsync();
-})();
+
+    process.exit(0);
+})().catch(err => {
+    console.log(err);
+    process.exit(1);
+});
diff --git a/packages/monorepo-scripts/src/publish.ts b/packages/monorepo-scripts/src/publish.ts
index 6691fd3c1..7e91b9281 100644
--- a/packages/monorepo-scripts/src/publish.ts
+++ b/packages/monorepo-scripts/src/publish.ts
@@ -85,7 +85,8 @@ async function generateAndUploadDocJsonsAsync(updatedPublicPackages: Package[],
     for (const pkg of updatedPublicPackages) {
         const packageName = pkg.packageJson.name;
         const nameWithoutPrefix = packageName.replace('@0xproject/', '');
-        const docGenerateAndUploadUtils = new DocGenerateAndUploadUtils(nameWithoutPrefix, isStaging);
+        const shouldUploadDocs = true;
+        const docGenerateAndUploadUtils = new DocGenerateAndUploadUtils(nameWithoutPrefix, isStaging, shouldUploadDocs);
         await docGenerateAndUploadUtils.generateAndUploadDocsAsync();
     }
 }
diff --git a/packages/monorepo-scripts/src/types.ts b/packages/monorepo-scripts/src/types.ts
index b87cc47eb..3c2ec5069 100644
--- a/packages/monorepo-scripts/src/types.ts
+++ b/packages/monorepo-scripts/src/types.ts
@@ -52,7 +52,6 @@ export interface Package {
 
 export interface DocGenConfigs {
     DOC_JSON_VERSION: string;
-    PACKAGES_WITH_DOC_PAGES: string[];
     EXTERNAL_TYPE_TO_LINK: { [externalType: string]: string };
     EXTERNAL_EXPORT_TO_LINK: { [externalExport: string]: string };
     CLASSES_WITH_HIDDEN_CONSTRUCTORS: string[];
diff --git a/packages/monorepo-scripts/src/utils/doc_generate_and_upload_utils.ts b/packages/monorepo-scripts/src/utils/doc_generate_and_upload_utils.ts
index c84413e64..9e5f437fa 100644
--- a/packages/monorepo-scripts/src/utils/doc_generate_and_upload_utils.ts
+++ b/packages/monorepo-scripts/src/utils/doc_generate_and_upload_utils.ts
@@ -13,7 +13,7 @@ import { utils } from './utils';
 export class DocGenerateAndUploadUtils {
     private _isStaging: boolean;
     private _shouldUploadDocs: boolean;
-    private _packageNameIfExists: string;
+    private _packageName: string;
     private _omitExports: string[];
     private _packagePath: string;
     private _exportPathToExportedItems: ExportPathToExportedItems;
@@ -156,21 +156,21 @@ export class DocGenerateAndUploadUtils {
         };
         return exportInfo;
     }
-    constructor(packageNameIfExists: string, isStaging: boolean, shouldUploadDocs: boolean) {
+    constructor(packageName: string, isStaging: boolean, shouldUploadDocs: boolean) {
         this._isStaging = isStaging;
-        this._packageNameIfExists = packageNameIfExists;
+        this._packageName = packageName;
         this._shouldUploadDocs = shouldUploadDocs;
-        this._packagePath = `${constants.monorepoRootPath}/packages/${packageNameIfExists}`;
+        this._packagePath = `${constants.monorepoRootPath}/packages/${packageName}`;
 
         this._monoRepoPkgNameToPath = {};
         const monorepoPackages = utils.getPackages(constants.monorepoRootPath);
         _.each(monorepoPackages, p => (this._monoRepoPkgNameToPath[p.packageJson.name] = p.location));
 
         const pkg = _.find(monorepoPackages, monorepoPackage => {
-            return _.includes(monorepoPackage.packageJson.name, packageNameIfExists);
+            return _.includes(monorepoPackage.packageJson.name, packageName);
         });
         if (_.isUndefined(pkg)) {
-            throw new Error(`Couldn't find a package.json for ${packageNameIfExists}`);
+            throw new Error(`Couldn't find a package.json for ${packageName}`);
         }
         this._packageJson = pkg.packageJson;
         this._omitExports = _.get(this._packageJson, 'config.postpublish.omitExports', []);
@@ -191,26 +191,18 @@ export class DocGenerateAndUploadUtils {
         }
     }
     public async generateAndUploadDocsAsync(): Promise<void> {
-        if (!_.isUndefined(this._packageNameIfExists)) {
-            await this.generateAndUploadDocsForPackageAsync(this._packageNameIfExists);
-        } else {
-            for (const packageName of docGenConfigs.PACKAGES_WITH_DOC_PAGES) {
-                await this.generateAndUploadDocsForPackageAsync(packageName);
-            }
-        }
-    }
-    private async generateAndUploadDocsForPackageAsync(packageName: string): Promise<void> {
         // For each dep that is another one of our monorepo packages, we fetch it's index.ts
         // and see which specific files we must pass to TypeDoc, in order to generate a Doc JSON
         // the includes everything exported by the public interface.
-        const typeDocExtraFileIncludes: string[] = this._getTypeDocFileIncludesForPackage(packageName);
+        const typeDocExtraFileIncludes: string[] = this._getTypeDocFileIncludesForPackage();
 
         // In order to avoid TS errors, we need to pass TypeDoc the package's global.d.ts file
         typeDocExtraFileIncludes.push(path.join(this._packagePath, 'src', 'globals.d.ts'));
 
+        utils.log(`GENERATE_UPLOAD_DOCS: Generating Typedoc JSON for ${this._packageName}...`);
         const jsonFilePath = path.join(this._packagePath, 'generated_docs', 'index.json');
         const projectFiles = typeDocExtraFileIncludes.join(' ');
-        const cwd = path.join(constants.monorepoRootPath, 'packages', packageName);
+        const cwd = path.join(constants.monorepoRootPath, 'packages', this._packageName);
         // HACK: For some reason calling `typedoc` command directly from here, even with `cwd` set to the
         // packages root dir, does not work. It only works when called via a `package.json` script located
         // in the package's root.
@@ -218,20 +210,21 @@ export class DocGenerateAndUploadUtils {
             cwd,
         });
 
+        utils.log('GENERATE_UPLOAD_DOCS: Modifying Typedoc JSON to our custom format...');
         const typedocOutputString = readFileSync(jsonFilePath).toString();
         const typedocOutput = JSON.parse(typedocOutputString);
-        let modifiedTypedocOutput = this._standardizeTypedocOutputTopLevelChildNames(typedocOutput, packageName);
-        modifiedTypedocOutput = this._pruneTypedocOutput(modifiedTypedocOutput, packageName);
+        let modifiedTypedocOutput = this._standardizeTypedocOutputTopLevelChildNames(typedocOutput);
+        modifiedTypedocOutput = this._pruneTypedocOutput(modifiedTypedocOutput);
 
-        if (!_.includes(docGenConfigs.TYPES_ONLY_LIBRARIES, packageName)) {
+        if (!_.includes(docGenConfigs.TYPES_ONLY_LIBRARIES, this._packageName)) {
             const propertyName = ''; // Root has no property name
             const referenceNames = DocGenerateAndUploadUtils._getAllReferenceNames(
                 propertyName,
                 modifiedTypedocOutput,
                 [],
             );
-            this._lookForUnusedExportedTypesThrowIfExists(referenceNames, modifiedTypedocOutput, packageName);
-            this._lookForMissingReferenceExportsThrowIfExists(referenceNames, packageName);
+            this._lookForUnusedExportedTypesThrowIfExists(referenceNames, modifiedTypedocOutput);
+            this._lookForMissingReferenceExportsThrowIfExists(referenceNames);
         }
 
         // Some of our packages re-export external package exports in their index.ts
@@ -249,7 +242,9 @@ export class DocGenerateAndUploadUtils {
         });
         if (!_.isEmpty(externalExportsWithoutLinks)) {
             throw new Error(
-                `Found the following external exports in ${packageName}'s index.ts:\n ${externalExportsWithoutLinks.join(
+                `Found the following external exports in ${
+                    this._packageName
+                }'s index.ts:\n ${externalExportsWithoutLinks.join(
                     '\n',
                 )}\nThey are missing from the EXTERNAL_EXPORT_TO_LINK mapping. Add them and try again.`,
             );
@@ -257,7 +252,7 @@ export class DocGenerateAndUploadUtils {
 
         const exportPathToTypedocNames: ExportNameToTypedocNames = {};
         _.each(modifiedTypedocOutput.children, file => {
-            const exportPath = this._findExportPathGivenTypedocName(file.name, packageName);
+            const exportPath = this._findExportPathGivenTypedocName(file.name);
             exportPathToTypedocNames[exportPath] = _.isUndefined(exportPathToTypedocNames[exportPath])
                 ? [file.name]
                 : [...exportPathToTypedocNames[exportPath], file.name];
@@ -275,19 +270,21 @@ export class DocGenerateAndUploadUtils {
             },
             typedocJson: modifiedTypedocOutput,
         };
+        utils.log(`GENERATE_UPLOAD_DOCS: Saving Doc JSON to: ${jsonFilePath}`);
         writeFileSync(jsonFilePath, JSON.stringify(docJson, null, 2));
 
         if (this._shouldUploadDocs) {
-            await this._uploadDocsAsync(jsonFilePath, cwd, packageName);
+            await this._uploadDocsAsync(jsonFilePath, cwd);
         }
+        utils.log(`GENERATE_UPLOAD_DOCS: Doc generation done for ${this._packageName}`);
     }
-    private async _uploadDocsAsync(jsonFilePath: string, cwd: string, packageName: string) {
+    private async _uploadDocsAsync(jsonFilePath: string, cwd: string) {
         const fileName = `v${this._packageJson.version}.json`;
         utils.log(`GENERATE_UPLOAD_DOCS: Doc generation successful, uploading docs... as ${fileName}`);
 
         const S3BucketPath = this._isStaging
-            ? `s3://staging-doc-jsons/${packageName}/`
-            : `s3://doc-jsons/${packageName}/`;
+            ? `s3://staging-doc-jsons/${this._packageName}/`
+            : `s3://doc-jsons/${this._packageName}/`;
         const s3Url = `${S3BucketPath}${fileName}`;
         await execAsync(
             `aws s3 cp ${jsonFilePath} ${s3Url} --profile 0xproject --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers --content-type application/json`,
@@ -304,7 +301,7 @@ export class DocGenerateAndUploadUtils {
     /**
      *  Look for types that are used by the public interface but are missing from a package's index.ts
      */
-    private _lookForMissingReferenceExportsThrowIfExists(referenceNames: string[], packageName: string): void {
+    private _lookForMissingReferenceExportsThrowIfExists(referenceNames: string[]): void {
         const allExportedItems = _.flatten(_.values(this._exportPathToExportedItems));
         const missingReferences: string[] = [];
         _.each(referenceNames, referenceName => {
@@ -317,7 +314,7 @@ export class DocGenerateAndUploadUtils {
         });
         if (!_.isEmpty(missingReferences)) {
             throw new Error(
-                `${packageName} package needs to export: \n${missingReferences.join(
+                `${this._packageName} package needs to export: \n${missingReferences.join(
                     '\n',
                 )} \nFrom it\'s index.ts. If any are from external dependencies, then add them to the EXTERNAL_TYPE_TO_LINK mapping.`,
             );
@@ -326,11 +323,7 @@ export class DocGenerateAndUploadUtils {
     /**
      * Look for exported types that are not used by the package's public interface
      */
-    private _lookForUnusedExportedTypesThrowIfExists(
-        referenceNames: string[],
-        typedocOutput: any,
-        packageName: string,
-    ): void {
+    private _lookForUnusedExportedTypesThrowIfExists(referenceNames: string[], typedocOutput: any): void {
         const exportedTypes = DocGenerateAndUploadUtils._getAllTypeNames(typedocOutput, []);
         const excessiveReferences = _.difference(exportedTypes, referenceNames);
         const excessiveReferencesExceptIgnored = _.difference(
@@ -339,7 +332,7 @@ export class DocGenerateAndUploadUtils {
         );
         if (!_.isEmpty(excessiveReferencesExceptIgnored)) {
             throw new Error(
-                `${packageName} package exports BUT does not need: \n${excessiveReferencesExceptIgnored.join(
+                `${this._packageName} package exports BUT does not need: \n${excessiveReferencesExceptIgnored.join(
                     '\n',
                 )} \nin it\'s index.ts. Remove them then try again.`,
             );
@@ -351,10 +344,10 @@ export class DocGenerateAndUploadUtils {
      * - the constructor is to be ignored
      * - it begins with an underscore (i.e is private)
      */
-    private _pruneTypedocOutput(typedocOutput: any, packageName: string): any {
+    private _pruneTypedocOutput(typedocOutput: any): any {
         const modifiedTypedocOutput = _.clone(typedocOutput);
         _.each(typedocOutput.children, (file, i) => {
-            const exportPath = this._findExportPathGivenTypedocName(file.name, packageName);
+            const exportPath = this._findExportPathGivenTypedocName(file.name);
             const exportItems = this._exportPathToExportedItems[exportPath];
             _.each(file.children, (child, j) => {
                 const isNotExported = !_.includes(exportItems, child.name);
@@ -388,12 +381,12 @@ export class DocGenerateAndUploadUtils {
      * monorepo). In order to enforce that the names are always prefixed with the package's name, we check and add
      * them here when necessary.
      */
-    private _standardizeTypedocOutputTopLevelChildNames(typedocOutput: any, packageName: string): any {
+    private _standardizeTypedocOutputTopLevelChildNames(typedocOutput: any): any {
         const modifiedTypedocOutput = _.clone(typedocOutput);
         _.each(typedocOutput.children, (child, i) => {
             if (!_.includes(child.name, '/src/')) {
                 const nameWithoutQuotes = child.name.replace(/"/g, '');
-                const standardizedName = `"${packageName}/src/${nameWithoutQuotes}"`;
+                const standardizedName = `"${this._packageName}/src/${nameWithoutQuotes}"`;
                 modifiedTypedocOutput.children[i].name = standardizedName;
             }
         });
@@ -402,13 +395,13 @@ export class DocGenerateAndUploadUtils {
     /**
      * Maps back each top-level TypeDoc JSON object name to the exportPath from which it was generated.
      */
-    private _findExportPathGivenTypedocName(typedocName: string, packageName: string): string {
+    private _findExportPathGivenTypedocName(typedocName: string): string {
         const typeDocNameWithoutQuotes = _.replace(typedocName, /"/g, '');
         const sanitizedExportPathToExportPath: { [sanitizedName: string]: string } = {};
         const exportPaths = _.keys(this._exportPathToExportedItems);
         const sanitizedExportPaths = _.map(exportPaths, exportPath => {
             if (_.startsWith(exportPath, './')) {
-                const sanitizedExportPath = path.join(packageName, 'src', exportPath);
+                const sanitizedExportPath = path.join(this._packageName, 'src', exportPath);
                 sanitizedExportPathToExportPath[sanitizedExportPath] = exportPath;
                 return sanitizedExportPath;
             }
@@ -449,7 +442,7 @@ export class DocGenerateAndUploadUtils {
         });
         return externalExports;
     }
-    private _getTypeDocFileIncludesForPackage(packageName: string): string[] {
+    private _getTypeDocFileIncludesForPackage(): string[] {
         let typeDocExtraFileIncludes: string[] = [];
         _.each(this._exportPathToExportedItems, (exportedItems, exportPath) => {
             const isInternalToPkg = _.startsWith(exportPath, '.');
@@ -475,7 +468,9 @@ export class DocGenerateAndUploadUtils {
                     }
                     if (!_.startsWith(innerExportPath, './')) {
                         throw new Error(
-                            `GENERATE_UPLOAD_DOCS: WARNING - ${packageName} is exporting one of ${innerExportItems} which is
+                            `GENERATE_UPLOAD_DOCS: WARNING - ${
+                                this._packageName
+                            } is exporting one of ${innerExportItems} which is
                             itself exported from an external package. To fix this, export the external dependency directly,
                             not indirectly through ${innerExportPath}.`,
                         );