path: root/packages/monorepo-scripts/src/utils/doc_generate_and_upload_utils.ts

import { PackageJSON } from '@0x/types';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import * as _ from 'lodash';
import * as path from 'path';
import { exec as execAsync } from 'promisify-child-process';
import * as ts from 'typescript';

import { constants } from '../constants';
import { docGenConfigs } from '../doc_gen_configs';
import { ExportInfo, ExportNameToTypedocNames, ExportPathToExportedItems } from '../types';

import { utils } from './utils';

export class DocGenerateAndUploadUtils {
    private readonly _isStaging: boolean;
    private readonly _shouldUploadDocs: boolean;
    private readonly _packageName: string;
    private readonly _omitExports: string[];
    private readonly _packagePath: string;
    private readonly _exportPathToExportedItems: ExportPathToExportedItems;
    private readonly _exportPathOrder: string[];
    private readonly _monoRepoPkgNameToPath: { [name: string]: string };
    private readonly _packageJson: PackageJSON;
    /**
     *  Recursively iterate over the TypeDoc JSON object and find all type names
     */
    private static _getAllTypeNames(node: any, typeNames: string[]): string[] {
        if (!_.isObject(node)) {
            return typeNames;
        }
        const typeKindStrings = ['Interface', 'Enumeration', 'Type alias'];
        if (_.includes(typeKindStrings, node.kindString)) {
            return [...typeNames, node.name];
        }
        let updatedTypeNames = typeNames;
        _.each(node, nodeValue => {
            if (_.isArray(nodeValue)) {
                _.each(nodeValue, aNode => {
                    updatedTypeNames = DocGenerateAndUploadUtils._getAllTypeNames(aNode, updatedTypeNames);
                });
            } else if (_.isObject(nodeValue)) {
                updatedTypeNames = DocGenerateAndUploadUtils._getAllTypeNames(nodeValue, updatedTypeNames);
            }
        });
        return updatedTypeNames;
    }
    /**
     * Recursively iterate over the TypeDoc JSON object and find all reference names (i.e types, classNames,
     * objectLiteral names, etc...)
     */
    private static _getAllReferenceNames(propertyName: string, node: any, referenceNames: string[]): string[] {
        if (!_.isObject(node)) {
            return referenceNames;
        }

        let updatedReferenceNames = referenceNames;
        // Some nodes of type reference are for subtypes, which we don't want to return.
        // We therefore filter them out.
        const SUB_TYPE_PROPERTY_NAMES = ['inheritedFrom', 'overwrites', 'extendedTypes', 'implementationOf'];
        const TS_MAPPED_TYPES = ['Partial', 'Promise', 'Readonly', 'Pick', 'Record'];
        if (
            !_.isUndefined(node.type) &&
            _.isString(node.type) &&
            node.type === 'reference' &&
            !_.includes(TS_MAPPED_TYPES, node.name) &&
            !_.includes(SUB_TYPE_PROPERTY_NAMES, propertyName)
        ) {
            updatedReferenceNames = _.uniq([...referenceNames, node.name]);
            return updatedReferenceNames;
        }
        _.each(node, (nodeValue, innerPropertyName) => {
            if (_.isArray(nodeValue)) {
                _.each(nodeValue, aNode => {
                    updatedReferenceNames = DocGenerateAndUploadUtils._getAllReferenceNames(
                        innerPropertyName,
                        aNode,
                        updatedReferenceNames,
                    );
                });
            } else if (_.isObject(nodeValue)) {
                updatedReferenceNames = DocGenerateAndUploadUtils._getAllReferenceNames(
                    innerPropertyName,
                    nodeValue,
                    updatedReferenceNames,
                );
            }
        });
        return _.uniq(updatedReferenceNames);
    }
    private static _getExportPathToExportedItems(filePath: string, omitExports?: string[]): ExportInfo {
        const sourceFile = ts.createSourceFile(
            'indexFile',
            readFileSync(filePath).toString(),
            ts.ScriptTarget.ES2017,
            /*setParentNodes */ true,
        );
        const exportPathToExportedItems: ExportPathToExportedItems = {};
        const exportPathOrder: string[] = [];
        const exportsToOmit = _.isUndefined(omitExports) ? [] : omitExports;

        processNode(sourceFile);

        function processNode(node: ts.Node): void {
            switch (node.kind) {
                case ts.SyntaxKind.ExportDeclaration: {
                    const exportClause = (node as any).exportClause;
                    if (_.isUndefined(exportClause)) {
                        return;
                    }
                    const exportPath = exportClause.parent.moduleSpecifier.text;
                    _.each(exportClause.elements, element => {
                        const exportItem = element.name.escapedText;
                        if (!_.includes(exportsToOmit, exportItem)) {
                            exportPathToExportedItems[exportPath] = _.isUndefined(exportPathToExportedItems[exportPath])
                                ? [exportItem]
                                : [...exportPathToExportedItems[exportPath], exportItem];
                        }
                    });
                    if (!_.isUndefined(exportPathToExportedItems[exportPath])) {
                        exportPathOrder.push(exportPath);
                    }
                    break;
                }

                case ts.SyntaxKind.ExportKeyword: {
                    const foundNode: any = node;
                    let exportPath = './index';
                    if (foundNode.parent && foundNode.parent.name) {
                        const exportItem = foundNode.parent.name.escapedText;
                        const isExportImportRequireStatement = !_.isUndefined(
                            _.get(foundNode, 'parent.moduleReference.expression.text'),
                        );
                        if (isExportImportRequireStatement) {
                            exportPath = foundNode.parent.moduleReference.expression.text;
                        }
                        if (!_.includes(exportsToOmit, exportItem)) {
                            exportPathToExportedItems[exportPath] = _.isUndefined(exportPathToExportedItems[exportPath])
                                ? [exportItem]
                                : [...exportPathToExportedItems[exportPath], exportItem];
                        }
                    }
                    if (
                        !_.includes(exportPathOrder, exportPath) &&
                        !_.isUndefined(exportPathToExportedItems[exportPath])
                    ) {
                        exportPathOrder.push(exportPath);
                    }
                    break;
                }
                default:
                    // noop
                    break;
            }

            ts.forEachChild(node, processNode);
        }
        const exportInfo = {
            exportPathToExportedItems,
            exportPathOrder,
        };
        return exportInfo;
    }
    constructor(packageName: string, isStaging: boolean, shouldUploadDocs: boolean) {
        this._isStaging = isStaging;
        this._packageName = packageName;
        this._shouldUploadDocs = shouldUploadDocs;
        this._packagePath = `${constants.monorepoRootPath}/packages/${packageName}`;

        this._monoRepoPkgNameToPath = {};
        const monorepoPackages = utils.getPackages(constants.monorepoRootPath);
        _.each(monorepoPackages, p => (this._monoRepoPkgNameToPath[p.packageJson.name] = p.location));

        const pkg = _.find(monorepoPackages, monorepoPackage => {
            return _.includes(monorepoPackage.packageJson.name, packageName);
        });
        if (_.isUndefined(pkg)) {
            throw new Error(`Couldn't find a package.json for ${packageName}`);
        }
        this._packageJson = pkg.packageJson;
        this._omitExports = _.get(this._packageJson, 'config.postpublish.docOmitExports', []);

        const indexPath = `${this._packagePath}/src/index.ts`;
        const exportInfo = DocGenerateAndUploadUtils._getExportPathToExportedItems(indexPath, this._omitExports);
        this._exportPathToExportedItems = exportInfo.exportPathToExportedItems;
        this._exportPathOrder = exportInfo.exportPathOrder;
    }
    public async generateAndUploadDocsAsync(): Promise<void> {
        // For each dep that is another one of our monorepo packages, we fetch it's index.ts
        // and see which specific files we must pass to TypeDoc, in order to generate a Doc JSON
        // the includes everything exported by the public interface.
        const typeDocExtraFileIncludes: string[] = this._getTypeDocFileIncludesForPackage();

        // In order to avoid TS errors, we need to pass TypeDoc the package's global.d.ts file
        // if it exists.
        const globalTypeDefinitionsPath = path.join(this._packagePath, 'src', 'globals.d.ts');
        if (existsSync(globalTypeDefinitionsPath)) {
            typeDocExtraFileIncludes.push(globalTypeDefinitionsPath);
        }

        utils.log(`GENERATE_UPLOAD_DOCS: Generating Typedoc JSON for ${this._packageName}...`);
        const jsonFilePath = path.join(this._packagePath, 'generated_docs', 'index.json');
        const projectFiles = typeDocExtraFileIncludes.join(' ');
        const cwd = path.join(constants.monorepoRootPath, 'packages', this._packageName);
        // HACK: For some reason calling `typedoc` command directly from here, even with `cwd` set to the
        // packages root dir, does not work. It only works when called via a `package.json` script located
        // in the package's root.
        await execAsync(`JSON_FILE_PATH=${jsonFilePath} PROJECT_FILES="${projectFiles}" yarn docs:json`, {
            cwd,
        });

        utils.log('GENERATE_UPLOAD_DOCS: Modifying Typedoc JSON to our custom format...');
        const typedocOutputString = readFileSync(jsonFilePath).toString();
        const typedocOutput = JSON.parse(typedocOutputString);
        const standardizedTypedocOutput = this._standardizeTypedocOutputTopLevelChildNames(typedocOutput);
        const modifiedTypedocOutput = this._pruneTypedocOutput(standardizedTypedocOutput);

        if (!_.includes(docGenConfigs.TYPES_ONLY_LIBRARIES, this._packageName)) {
            const propertyName = ''; // Root has no property name
            const referenceNames = DocGenerateAndUploadUtils._getAllReferenceNames(
                propertyName,
                modifiedTypedocOutput,
                [],
            );
            this._lookForUnusedExportedTypesThrowIfExists(referenceNames, modifiedTypedocOutput);
            this._lookForMissingReferenceExportsThrowIfExists(referenceNames);
        }

        // Some of our packages re-export external package exports in their index.ts
        // Typedoc is incapable of rendering these packages, so we need to special-case them
        const externalExportToLink: { [externalExport: string]: string } = {};
        const externalExportsWithoutLinks: string[] = [];
        const externalExports: string[] = this._getAllExternalExports();
        _.each(externalExports, externalExport => {
            const linkIfExists = docGenConfigs.EXTERNAL_EXPORT_TO_LINK[externalExport];
            if (_.isUndefined(linkIfExists)) {
                externalExportsWithoutLinks.push(externalExport);
                return;
            }
            externalExportToLink[externalExport] = linkIfExists;
        });
        if (!_.isEmpty(externalExportsWithoutLinks)) {
            throw new Error(
                `Found the following external exports in ${
                    this._packageName
                }'s index.ts:\n ${externalExportsWithoutLinks.join(
                    '\n',
                )}\nThey are missing from the EXTERNAL_EXPORT_TO_LINK mapping. Add them and try again.`,
            );
        }

        const exportPathToTypedocNames: ExportNameToTypedocNames = {};
        _.each(modifiedTypedocOutput.children, file => {
            const exportPath = this._findExportPathGivenTypedocName(file.name);
            exportPathToTypedocNames[exportPath] = _.isUndefined(exportPathToTypedocNames[exportPath])
                ? [file.name]
                : [...exportPathToTypedocNames[exportPath], file.name];
        });

        // Since we need additional metadata included in the doc JSON, we nest the TypeDoc JSON
        // within our own custom, versioned docsJson format.
        const docJson = {
            version: docGenConfigs.DOC_JSON_VERSION,
            metadata: {
                exportPathToTypedocNames,
                exportPathOrder: this._exportPathOrder,
                externalTypeToLink: docGenConfigs.EXTERNAL_TYPE_TO_LINK,
                externalExportToLink,
            },
            typedocJson: modifiedTypedocOutput,
        };
        utils.log(`GENERATE_UPLOAD_DOCS: Saving Doc JSON to: ${jsonFilePath}`);
        writeFileSync(jsonFilePath, JSON.stringify(docJson, null, 2));

        if (this._shouldUploadDocs) {
            await this._uploadDocsAsync(jsonFilePath, cwd);
        }
        utils.log(`GENERATE_UPLOAD_DOCS: Doc generation done for ${this._packageName}`);
    }
    private async _uploadDocsAsync(jsonFilePath: string, cwd: string): Promise<void> {
        const fileName = `v${this._packageJson.version}.json`;
        utils.log(`GENERATE_UPLOAD_DOCS: Doc generation successful, uploading docs... as ${fileName}`);

        const S3BucketPath = this._isStaging
            ? `s3://staging-doc-jsons/${this._packageName}/`
            : `s3://doc-jsons/${this._packageName}/`;
        const s3Url = `${S3BucketPath}${fileName}`;
        await execAsync(
            `aws s3 cp ${jsonFilePath} ${s3Url} --profile 0xproject --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers --content-type application/json`,
            {
                cwd,
            },
        );
        utils.log(`GENERATE_UPLOAD_DOCS: Docs uploaded to S3 bucket: ${S3BucketPath}`);
        // Remove the generated docs directory
        await execAsync(`rm -rf ${jsonFilePath}`, {
            cwd,
        });
    }
    /**
     *  Look for types that are used by the public interface but are missing from a package's index.ts
     */
    private _lookForMissingReferenceExportsThrowIfExists(referenceNames: string[]): void {
        const allExportedItems = _.flatten(_.values(this._exportPathToExportedItems));
        const missingReferences: string[] = [];
        _.each(referenceNames, referenceName => {
            if (
                !_.includes(allExportedItems, referenceName) &&
                _.isUndefined(docGenConfigs.EXTERNAL_TYPE_TO_LINK[referenceName])
            ) {
                missingReferences.push(referenceName);
            }
        });
        if (!_.isEmpty(missingReferences)) {
            throw new Error(
                `${this._packageName} package needs to export: \n${missingReferences.join(
                    '\n',
                )} \nFrom it\'s index.ts. If any are from external dependencies, then add them to the EXTERNAL_TYPE_TO_LINK mapping.`,
            );
        }
    }
    /**
     * Look for exported types that are not used by the package's public interface
     */
    private _lookForUnusedExportedTypesThrowIfExists(referenceNames: string[], typedocOutput: any): void {
        const exportedTypes = DocGenerateAndUploadUtils._getAllTypeNames(typedocOutput, []);
        const excessiveReferences = _.difference(exportedTypes, referenceNames);
        const excessiveReferencesExceptIgnored = _.difference(
            excessiveReferences,
            docGenConfigs.IGNORED_EXCESSIVE_TYPES,
        );
        if (!_.isEmpty(excessiveReferencesExceptIgnored)) {
            throw new Error(
                `${this._packageName} package exports BUT does not need: \n${excessiveReferencesExceptIgnored.join(
                    '\n',
                )} \nin it\'s index.ts. Remove them then try again OR if we still want them exported (e.g error enum types), then add them to the IGNORED_EXCESSIVE_TYPES array.`,
            );
        }
    }
    /**
     *  For each entry in the TypeDoc JSON, remove it if:
     * - it was not exported in index.ts
     * - the constructor is to be ignored
     * - it begins with an underscore (i.e is private)
     */
    private _pruneTypedocOutput(typedocOutput: any): any {
        const modifiedTypedocOutput = _.cloneDeep(typedocOutput);
        _.each(typedocOutput.children, (file, i) => {
            const exportPath = this._findExportPathGivenTypedocName(file.name);
            const exportItems = this._exportPathToExportedItems[exportPath];
            _.each(file.children, (child, j) => {
                const isNotExported = !_.includes(exportItems, child.name);
                if (isNotExported) {
                    delete modifiedTypedocOutput.children[i].children[j];
                    return;
                }

                const innerChildren = typedocOutput.children[i].children[j].children;
                _.each(innerChildren, (innerChild, k) => {
                    const isHiddenConstructor =
                        child.kindString === 'Class' &&
                        _.includes(docGenConfigs.CLASSES_WITH_HIDDEN_CONSTRUCTORS, child.name) &&
                        innerChild.kindString === 'Constructor';
                    const isPrivate = _.startsWith(innerChild.name, '_');
                    if (isHiddenConstructor || isPrivate) {
                        delete modifiedTypedocOutput.children[i].children[j].children[k];
                    }
                });
                modifiedTypedocOutput.children[i].children[j].children = _.compact(
                    modifiedTypedocOutput.children[i].children[j].children,
                );
            });
            modifiedTypedocOutput.children[i].children = _.compact(modifiedTypedocOutput.children[i].children);
        });
        return modifiedTypedocOutput;
    }
    /**
     * Unfortunately TypeDoc children names will only be prefixed with the name of the package _if_ we passed
     * TypeDoc files outside of the packages root path (i.e this package exports another package from our
     * monorepo). In order to enforce that the names are always prefixed with the package's name, we check and add
     * them here when necessary.
     */
    private _standardizeTypedocOutputTopLevelChildNames(typedocOutput: any): any {
        const modifiedTypedocOutput = _.cloneDeep(typedocOutput);
        _.each(typedocOutput.children, (child, i) => {
            if (!_.includes(child.name, '/src/')) {
                const nameWithoutQuotes = child.name.replace(/"/g, '');
                const standardizedName = `"${this._packageName}/src/${nameWithoutQuotes}"`;
                modifiedTypedocOutput.children[i].name = standardizedName;
            }
        });
        return modifiedTypedocOutput;
    }
    /**
     * Maps back each top-level TypeDoc JSON object name to the exportPath from which it was generated.
     */
    private _findExportPathGivenTypedocName(typedocName: string): string {
        const typeDocNameWithoutQuotes = _.replace(typedocName, /"/g, '');
        const sanitizedExportPathToExportPath: { [sanitizedName: string]: string } = {};
        const exportPaths = _.keys(this._exportPathToExportedItems);
        const sanitizedExportPaths = _.map(exportPaths, exportPath => {
            if (_.startsWith(exportPath, './')) {
                const sanitizedExportPath = path.join(this._packageName, 'src', exportPath);
                sanitizedExportPathToExportPath[sanitizedExportPath] = exportPath;
                return sanitizedExportPath;
            }
            const monorepoPrefix = '@0x/';
            if (_.startsWith(exportPath, monorepoPrefix)) {
                const sanitizedExportPath = exportPath.split(monorepoPrefix)[1];
                sanitizedExportPathToExportPath[sanitizedExportPath] = exportPath;
                return sanitizedExportPath;
            }
            sanitizedExportPathToExportPath[exportPath] = exportPath;
            return exportPath;
        });
        // We need to sort the exportPaths by length (longest first), so that the match finding will pick
        // longer matches before shorter matches, since it might match both, but the longer match is more
        // precisely what we are looking for.
        const sanitizedExportPathsSortedByLength = sanitizedExportPaths.sort((a: string, b: string) => {
            return b.length - a.length;
        });
        const matchingSanitizedExportPathIfExists = _.find(sanitizedExportPathsSortedByLength, p => {
            return _.startsWith(typeDocNameWithoutQuotes, p);
        });
        if (_.isUndefined(matchingSanitizedExportPathIfExists)) {
            throw new Error(`Didn't find an exportPath for ${typeDocNameWithoutQuotes}`);
        }
        const matchingExportPath = sanitizedExportPathToExportPath[matchingSanitizedExportPathIfExists];
        return matchingExportPath;
    }
    private _getAllExternalExports(): string[] {
        const externalExports: string[] = [];
        _.each(this._exportPathToExportedItems, (exportedItems, exportPath) => {
            const pathIfExists = this._monoRepoPkgNameToPath[exportPath];
            if (_.isUndefined(pathIfExists) && !_.startsWith(exportPath, './')) {
                _.each(exportedItems, exportedItem => {
                    externalExports.push(exportedItem);
                });
                return; // It's an external package
            }
        });
        return externalExports;
    }
    private _getTypeDocFileIncludesForPackage(): string[] {
        let typeDocExtraFileIncludes: string[] = [];
        _.each(this._exportPathToExportedItems, (exportedItems, exportPath) => {
            const isInternalToPkg = _.startsWith(exportPath, '.');
            if (isInternalToPkg) {
                const pathToInternalPkg = path.join(this._packagePath, 'src', `${exportPath}.ts`);
                typeDocExtraFileIncludes.push(pathToInternalPkg);
                return;
            }

            const pathIfExists = this._monoRepoPkgNameToPath[exportPath];
            if (_.isUndefined(pathIfExists)) {
                return; // It's an external package
            }

            const typeDocSourceIncludes = new Set();
            const pathToIndex = `${pathIfExists}/src/index.ts`;
            const exportInfo = DocGenerateAndUploadUtils._getExportPathToExportedItems(pathToIndex);
            const innerExportPathToExportedItems = exportInfo.exportPathToExportedItems;
            _.each(exportedItems, exportName => {
                _.each(innerExportPathToExportedItems, (innerExportItems, innerExportPath) => {
                    if (!_.includes(innerExportItems, exportName)) {
                        return;
                    }
                    if (!_.startsWith(innerExportPath, './')) {
                        throw new Error(
                            `GENERATE_UPLOAD_DOCS: WARNING - ${
                                this._packageName
                            } is exporting one of ${innerExportItems} which is
                            itself exported from an external package. To fix this, export the external dependency directly,
                            not indirectly through ${innerExportPath}.`,
                        );
                    } else {
                        const absoluteSrcPath = path.join(pathIfExists, 'src', `${innerExportPath}.ts`);
                        typeDocSourceIncludes.add(absoluteSrcPath);
                    }
                });
            });

            // @0x/types & ethereum-types are examples of packages where their index.ts exports types
            // directly, meaning no internal paths will exist to follow. Other packages also have direct exports
            // in their index.ts, so we always add it to the source files passed to TypeDoc
            if (typeDocSourceIncludes.size === 0) {
                typeDocSourceIncludes.add(pathToIndex);
            }

            typeDocExtraFileIncludes = [...typeDocExtraFileIncludes, ...Array.from(typeDocSourceIncludes)];
        });
        return typeDocExtraFileIncludes;
    }
}