diff options
author | Remco Bloemen <remco@wicked.ventures> | 2018-11-09 01:32:40 +0800 |
---|---|---|
committer | Remco Bloemen <remco@wicked.ventures> | 2018-11-09 01:32:40 +0800 |
commit | d71362af993d3797dbdbfcac245ad57f0086bce3 (patch) | |
tree | 888826fe23c2d06d6c9191fb3a238e14f9fe4aac /packages/monorepo-scripts | |
parent | a5665a68756c905637c551fc48c9b7011a55c237 (diff) | |
parent | f6abc007ffb249e4bbf85b8a7a77309d43e0a147 (diff) | |
download | dexon-sol-tools-d71362af993d3797dbdbfcac245ad57f0086bce3.tar dexon-sol-tools-d71362af993d3797dbdbfcac245ad57f0086bce3.tar.gz dexon-sol-tools-d71362af993d3797dbdbfcac245ad57f0086bce3.tar.bz2 dexon-sol-tools-d71362af993d3797dbdbfcac245ad57f0086bce3.tar.lz dexon-sol-tools-d71362af993d3797dbdbfcac245ad57f0086bce3.tar.xz dexon-sol-tools-d71362af993d3797dbdbfcac245ad57f0086bce3.tar.zst dexon-sol-tools-d71362af993d3797dbdbfcac245ad57f0086bce3.zip |
Merge remote-tracking branch 'origin/development' into feature/utils/prettybignum
Diffstat (limited to 'packages/monorepo-scripts')
19 files changed, 888 insertions, 278 deletions
diff --git a/packages/monorepo-scripts/CHANGELOG.json b/packages/monorepo-scripts/CHANGELOG.json index 3b8684fab..170a97a33 100644 --- a/packages/monorepo-scripts/CHANGELOG.json +++ b/packages/monorepo-scripts/CHANGELOG.json @@ -1,5 +1,22 @@ [ { + "version": "1.0.6", + "changes": [ + { + "note": "Render date formats in UTC to prevent conflicts when publishing in different timezones.", + "pr": 1143 + }, + { + "note": "Add AssetBuyerError to the IGNORED_EXCESSIVE_TYPES array", + "pr": 1139 + }, + { + "note": "Add ForwarderError to the IGNORED_EXCESSIVE_TYPES array", + "pr": 1147 + } + ] + }, + { "timestamp": 1534210131, "version": "1.0.5", "changes": [ diff --git a/packages/monorepo-scripts/README.md b/packages/monorepo-scripts/README.md index d979e27dc..0673098b5 100644 --- a/packages/monorepo-scripts/README.md +++ b/packages/monorepo-scripts/README.md @@ -47,13 +47,13 @@ yarn install To build this package and all other monorepo packages that it depends on, run the following from the monorepo root directory: ```bash -PKG=@0xproject/monorepo-scripts yarn build +PKG=@0x/monorepo-scripts yarn build ``` Or continuously rebuild on change: ```bash -PKG=@0xproject/monorepo-scripts yarn watch +PKG=@0x/monorepo-scripts yarn watch ``` ### Clean diff --git a/packages/monorepo-scripts/package.json b/packages/monorepo-scripts/package.json index 87135fd93..a83f90516 100644 --- a/packages/monorepo-scripts/package.json +++ b/packages/monorepo-scripts/package.json @@ -1,6 +1,7 @@ { - "name": "@0xproject/monorepo-scripts", - "version": "1.0.5", + "private": true, + "name": "@0x/monorepo-scripts", + "version": "1.0.12", "engines": { "node": ">=6.12" }, @@ -8,16 +9,18 @@ "main": "lib/index.js", "types": "lib/index.d.ts", "scripts": { - "watch_without_deps": "tsc -w", - "build": "tsc", - "lint": "tslint --project .", + "build": "tsc -b", + "build:ci": "yarn build", + "lint": "tslint --format stylish --project .", "clean": "shx rm -rf lib", "test:publish": "run-s build script:publish", "find_unused_deps": "run-s build script:find_unused_deps", "script:deps_versions": "node ./lib/deps_versions.js", "script:prepublish_checks": "node ./lib/prepublish_checks.js", "script:publish": "IS_DRY_RUN=true node ./lib/publish.js", - "script:find_unused_deps": "node ./lib/find_unused_dependencies.js" + "script:find_unused_deps": "node ./lib/find_unused_dependencies.js", + "script:doc_generate_and_upload": "node ./lib/doc_generate_and_upload.js", + "script:publish_release_notes": "node ./lib/publish_release_notes.js" }, "repository": { "type": "git", @@ -31,10 +34,11 @@ "devDependencies": { "@types/glob": "^5.0.33", "@types/mkdirp": "^0.5.2", - "@types/node": "^8.0.53", + "@types/node": "*", "@types/opn": "^5.1.0", "@types/rimraf": "^2.0.2", "@types/semver": "5.5.0", + "@types/yargs": "^10.0.0", "depcheck": "^0.6.9", "make-promises-safe": "^1.1.0", "npm-run-all": "^4.1.2", @@ -60,7 +64,9 @@ "rimraf": "^2.6.2", "semver": "5.5.0", "semver-diff": "^2.1.0", - "semver-sort": "0.0.4" + "semver-sort": "0.0.4", + "typedoc": "0.13.0", + "yargs": "^10.0.3" }, "publishConfig": { "access": "public" diff --git a/packages/monorepo-scripts/src/constants.ts b/packages/monorepo-scripts/src/constants.ts index e5d3348bd..acb4b211e 100644 --- a/packages/monorepo-scripts/src/constants.ts +++ b/packages/monorepo-scripts/src/constants.ts @@ -5,4 +5,5 @@ export const constants = { stagingWebsite: 'http://staging-0xproject.s3-website-us-east-1.amazonaws.com', lernaExecutable: path.join('node_modules', '@0x-lerna-fork', 'lerna', 'cli.js'), githubPersonalAccessToken: process.env.GITHUB_PERSONAL_ACCESS_TOKEN_0X_JS, + dependenciesUpdatedMessage: 'Dependencies updated', }; diff --git a/packages/monorepo-scripts/src/doc_gen_configs.ts b/packages/monorepo-scripts/src/doc_gen_configs.ts new file mode 100644 index 000000000..7a14f8664 --- /dev/null +++ b/packages/monorepo-scripts/src/doc_gen_configs.ts @@ -0,0 +1,64 @@ +import { DocGenConfigs } from './types'; + +export const docGenConfigs: DocGenConfigs = { + // Versions our doc JSON format so we can handle breaking changes intelligently + DOC_JSON_VERSION: '0.0.1', + // Some types that are exposed by our package's public interface are external types. As such, we won't + // be able to render their definitions. Instead we link to them using this lookup. + EXTERNAL_TYPE_TO_LINK: { + Array: 'https://developer.mozilla.org/pt-PT/docs/Web/JavaScript/Reference/Global_Objects/Array', + BigNumber: 'http://mikemcl.github.io/bignumber.js', + Error: 'https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error', + Buffer: 'https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/node/v9/index.d.ts#L262', + 'solc.StandardContractOutput': + 'https://solidity.readthedocs.io/en/v0.4.24/using-the-compiler.html#output-description', + 'solc.CompilerSettings': 'https://solidity.readthedocs.io/en/v0.4.24/using-the-compiler.html#input-description', + Schema: + 'https://github.com/tdegrunt/jsonschema/blob/5c2edd4baba149964aec0f23c87ad12c25a50dfb/lib/index.d.ts#L49', + Uint8Array: 'https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Uint8Array', + // HACK: CI can handle these without the namespace but some local setups (Jacob) require the namespace prefix + // This is duplicated until we can discover the source of the issue. + GanacheOpts: 'https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/ganache-core/index.d.ts#L8', + keystore: 'https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/eth-lightwallet/index.d.ts#L36', + 'Ganache.GanacheOpts': + 'https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/ganache-core/index.d.ts#L8', + 'lightwallet.keystore': + 'https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/eth-lightwallet/index.d.ts#L36', + }, + // If a 0x package re-exports an external package, we should add a link to it's exported items here + EXTERNAL_EXPORT_TO_LINK: { + Web3ProviderEngine: 'https://www.npmjs.com/package/web3-provider-engine', + BigNumber: 'https://www.npmjs.com/package/bignumber.js', + Schema: 'https://github.com/tdegrunt/jsonschema/blob/v1.2.4/lib/index.d.ts#L49', + ValidatorResult: 'https://github.com/tdegrunt/jsonschema/blob/v1.2.4/lib/helpers.js#L31', + }, + // Sometimes we want to hide a constructor from rendering in our docs. An example is when our library has a + // factory method which instantiates an instance of a class, but we don't want users instantiating it themselves + // and getting confused. Any class name in this list will not have it's constructor rendered in our docs. + CLASSES_WITH_HIDDEN_CONSTRUCTORS: [ + 'AssetBuyer', + 'ERC20ProxyWrapper', + 'ERC20TokenWrapper', + 'ERC721ProxyWrapper', + 'ERC721TokenWrapper', + 'EtherTokenWrapper', + 'ExchangeWrapper', + 'ForwarderWrapper', + 'OrderValidatorWrapper', + 'TransactionEncoder', + ], + // Some types are not explicitly part of the public interface like params, return values, etc... But we still + // want them exported. E.g error enum types that can be thrown by methods. These must be manually added to this + // config + IGNORED_EXCESSIVE_TYPES: [ + 'NonceSubproviderErrors', + 'Web3WrapperErrors', + 'ContractWrappersError', + 'OrderError', + 'AssetBuyerError', + 'ForwarderWrapperError', + ], + // Some libraries only export types. In those cases, we cannot check if the exported types are part of the + // "exported public interface". Thus we add them here and skip those checks. + TYPES_ONLY_LIBRARIES: ['ethereum-types', 'types'], +}; diff --git a/packages/monorepo-scripts/src/doc_generate_and_upload.ts b/packages/monorepo-scripts/src/doc_generate_and_upload.ts new file mode 100644 index 000000000..4c4a72701 --- /dev/null +++ b/packages/monorepo-scripts/src/doc_generate_and_upload.ts @@ -0,0 +1,37 @@ +import * as yargs from 'yargs'; + +import { DocGenerateAndUploadUtils } from './utils/doc_generate_and_upload_utils'; +import { utils } from './utils/utils'; + +const args = yargs + .option('package', { + describe: 'Monorepo sub-package for which to generate DocJSON', + type: 'string', + demandOption: true, + }) + .option('isStaging', { + describe: 'Whether we wish to publish docs to staging or production', + type: 'boolean', + demandOption: true, + }) + .option('shouldUpload', { + describe: 'Whether we wish to upload the docs to S3 or not', + type: 'boolean', + demandOption: false, + default: true, + }) + .example("$0 --package '0x.js' --isStaging true", 'Full usage example').argv; + +(async () => { + const packageName = args.package; + const isStaging = args.isStaging; + const shouldUploadDocs = args.shouldUpload; + + const docGenerateAndUploadUtils = new DocGenerateAndUploadUtils(packageName, isStaging, shouldUploadDocs); + await docGenerateAndUploadUtils.generateAndUploadDocsAsync(); + + process.exit(0); +})().catch(err => { + utils.log(err); + process.exit(1); +}); diff --git a/packages/monorepo-scripts/src/find_unused_dependencies.ts b/packages/monorepo-scripts/src/find_unused_dependencies.ts index 4bb4b7dc5..42b4b7890 100644 --- a/packages/monorepo-scripts/src/find_unused_dependencies.ts +++ b/packages/monorepo-scripts/src/find_unused_dependencies.ts @@ -7,7 +7,7 @@ import { constants } from './constants'; import { utils } from './utils/utils'; // For some reason, `depcheck` hangs on some packages. Add them here. -const IGNORE_PACKAGES = ['@0xproject/sol-compiler']; +const IGNORE_PACKAGES = ['@0x/sol-compiler']; (async () => { utils.log('*** NOTE: Not all deps listed here are actually not required. ***'); diff --git a/packages/monorepo-scripts/src/index.ts b/packages/monorepo-scripts/src/index.ts index 95c96ebe8..e69de29bb 100644 --- a/packages/monorepo-scripts/src/index.ts +++ b/packages/monorepo-scripts/src/index.ts @@ -1 +0,0 @@ -export { postpublishUtils } from './postpublish_utils'; diff --git a/packages/monorepo-scripts/src/postpublish_utils.ts b/packages/monorepo-scripts/src/postpublish_utils.ts deleted file mode 100644 index 8e445a045..000000000 --- a/packages/monorepo-scripts/src/postpublish_utils.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { execAsync } from 'async-child-process'; -import * as promisify from 'es6-promisify'; -import * as fs from 'fs'; -import * as _ from 'lodash'; -import * as path from 'path'; -import * as publishRelease from 'publish-release'; - -import { constants } from './constants'; -import { configs } from './utils/configs'; -import { utils } from './utils/utils'; - -const publishReleaseAsync = promisify(publishRelease); -const generatedDocsDirectoryName = 'generated_docs'; - -export interface PostpublishConfigs { - cwd: string; - packageName: string; - version: string; - assets: string[]; - docPublishConfigs: DocPublishConfigs; -} - -export interface DocPublishConfigs { - fileIncludes: string[]; - s3BucketPath: string; - s3StagingBucketPath: string; -} - -export const postpublishUtils = { - generateConfig(packageJSON: any, tsConfigJSON: any, cwd: string): PostpublishConfigs { - if (_.isUndefined(packageJSON.name)) { - throw new Error('name field required in package.json. Cannot publish release notes to Github.'); - } - if (_.isUndefined(packageJSON.version)) { - throw new Error('version field required in package.json. Cannot publish release notes to Github.'); - } - const postpublishConfig = _.get(packageJSON, 'config.postpublish', {}); - const postpublishConfigs: PostpublishConfigs = { - cwd, - packageName: packageJSON.name, - version: packageJSON.version, - assets: _.get(postpublishConfig, 'assets', []), - docPublishConfigs: { - fileIncludes: [ - ...tsConfigJSON.include, - ..._.get(postpublishConfig, 'docPublishConfigs.extraFileIncludes', []), - ], - s3BucketPath: _.get(postpublishConfig, 'docPublishConfigs.s3BucketPath'), - s3StagingBucketPath: _.get(postpublishConfig, 'docPublishConfigs.s3StagingBucketPath'), - }, - }; - return postpublishConfigs; - }, - async runAsync(packageJSON: any, tsConfigJSON: any, cwd: string): Promise<void> { - if (configs.IS_LOCAL_PUBLISH) { - return; - } - const postpublishConfigs = postpublishUtils.generateConfig(packageJSON, tsConfigJSON, cwd); - await postpublishUtils.publishReleaseNotesAsync( - postpublishConfigs.packageName, - postpublishConfigs.version, - postpublishConfigs.assets, - ); - if ( - !_.isUndefined(postpublishConfigs.docPublishConfigs.s3BucketPath) || - !_.isUndefined(postpublishConfigs.docPublishConfigs.s3StagingBucketPath) - ) { - utils.log('POSTPUBLISH: Release successful, generating docs...'); - await postpublishUtils.generateAndUploadDocsAsync( - postpublishConfigs.cwd, - postpublishConfigs.docPublishConfigs.fileIncludes, - postpublishConfigs.version, - postpublishConfigs.docPublishConfigs.s3BucketPath, - ); - } else { - utils.log(`POSTPUBLISH: No S3Bucket config found for ${packageJSON.name}. Skipping doc JSON generation.`); - } - }, - async publishDocsToStagingAsync(packageJSON: any, tsConfigJSON: any, cwd: string): Promise<void> { - const postpublishConfigs = postpublishUtils.generateConfig(packageJSON, tsConfigJSON, cwd); - if (_.isUndefined(postpublishConfigs.docPublishConfigs.s3StagingBucketPath)) { - utils.log('config.postpublish.docPublishConfigs.s3StagingBucketPath entry in package.json not found!'); - return; - } - - utils.log('POSTPUBLISH: Generating docs...'); - await postpublishUtils.generateAndUploadDocsAsync( - postpublishConfigs.cwd, - postpublishConfigs.docPublishConfigs.fileIncludes, - postpublishConfigs.version, - postpublishConfigs.docPublishConfigs.s3StagingBucketPath, - ); - }, - async publishReleaseNotesAsync(packageName: string, version: string, assets: string[]): Promise<void> { - const notes = postpublishUtils.getReleaseNotes(packageName, version); - const releaseName = postpublishUtils.getReleaseName(packageName, version); - const tag = postpublishUtils.getTag(packageName, version); - postpublishUtils.adjustAssetPaths(assets); - utils.log('POSTPUBLISH: Releasing ', releaseName, '...'); - await publishReleaseAsync({ - token: constants.githubPersonalAccessToken, - owner: '0xProject', - repo: '0x-monorepo', - tag, - name: releaseName, - notes, - draft: false, - prerelease: false, - reuseRelease: true, - reuseDraftOnly: false, - assets, - }); - }, - getReleaseNotes(packageName: string, version: string): string { - const packageNameWithNamespace = packageName.replace('@0xproject/', ''); - const changelogJSONPath = path.join( - constants.monorepoRootPath, - 'packages', - packageNameWithNamespace, - 'CHANGELOG.json', - ); - const changelogJSON = fs.readFileSync(changelogJSONPath, 'utf-8'); - const changelogs = JSON.parse(changelogJSON); - const latestLog = changelogs[0]; - // We sanity check that the version for the changelog notes we are about to publish to Github - // correspond to the new version of the package. - if (version !== latestLog.version) { - throw new Error('Expected CHANGELOG.json latest entry version to coincide with published version.'); - } - let notes = ''; - _.each(latestLog.changes, change => { - notes += `* ${change.note}`; - if (change.pr) { - notes += ` (#${change.pr})`; - } - notes += `\n`; - }); - return notes; - }, - getTag(packageName: string, version: string): string { - return `${packageName}@${version}`; - }, - getReleaseName(subPackageName: string, version: string): string { - const releaseName = `${subPackageName} v${version}`; - return releaseName; - }, - // Asset paths should described from the monorepo root. This method prefixes - // the supplied path with the absolute path to the monorepo root. - adjustAssetPaths(assets: string[]): string[] { - const finalAssets: string[] = []; - _.each(assets, (asset: string) => { - finalAssets.push(`${constants.monorepoRootPath}/${asset}`); - }); - return finalAssets; - }, - adjustFileIncludePaths(fileIncludes: string[], cwd: string): string[] { - const fileIncludesAdjusted = _.map(fileIncludes, fileInclude => { - let includePath = _.startsWith(fileInclude, './') - ? `${cwd}/${fileInclude.substr(2)}` - : `${cwd}/${fileInclude}`; - - // HACK: tsconfig.json needs wildcard directory endings as `/**/*` - // but TypeDoc needs it as `/**` in order to pick up files at the root - if (_.endsWith(includePath, '/**/*')) { - // tslint:disable-next-line:custom-no-magic-numbers - includePath = includePath.slice(0, -2); - } - return includePath; - }); - return fileIncludesAdjusted; - }, - async generateAndUploadDocsAsync( - cwd: string, - fileIncludes: string[], - version: string, - S3BucketPath: string, - ): Promise<void> { - const fileIncludesAdjusted = postpublishUtils.adjustFileIncludePaths(fileIncludes, cwd); - const projectFiles = fileIncludesAdjusted.join(' '); - const jsonFilePath = `${cwd}/${generatedDocsDirectoryName}/index.json`; - const result = await execAsync( - `JSON_FILE_PATH=${jsonFilePath} PROJECT_FILES="${projectFiles}" yarn docs:json`, - { - cwd, - }, - ); - if (!_.isEmpty(result.stderr)) { - throw new Error(result.stderr); - } - const fileName = `v${version}.json`; - utils.log(`POSTPUBLISH: Doc generation successful, uploading docs... as ${fileName}`); - const s3Url = S3BucketPath + fileName; - await execAsync(`S3_URL=${s3Url} yarn upload_docs_json`, { - cwd, - }); - // Remove the generated docs directory - await execAsync(`rm -rf ${generatedDocsDirectoryName}`, { - cwd, - }); - utils.log(`POSTPUBLISH: Docs uploaded to S3 bucket: ${S3BucketPath}`); - }, -}; diff --git a/packages/monorepo-scripts/src/prepublish_checks.ts b/packages/monorepo-scripts/src/prepublish_checks.ts index 683c26094..5f603ebc7 100644 --- a/packages/monorepo-scripts/src/prepublish_checks.ts +++ b/packages/monorepo-scripts/src/prepublish_checks.ts @@ -11,7 +11,7 @@ import { utils } from './utils/utils'; async function prepublishChecksAsync(): Promise<void> { const shouldIncludePrivate = false; - const updatedPublicPackages = await utils.getUpdatedPackagesAsync(shouldIncludePrivate); + const updatedPublicPackages = await utils.getPackagesToPublishAsync(shouldIncludePrivate); await checkCurrentVersionMatchesLatestPublishedNPMPackageAsync(updatedPublicPackages); await checkChangelogFormatAsync(updatedPublicPackages); diff --git a/packages/monorepo-scripts/src/publish.ts b/packages/monorepo-scripts/src/publish.ts index 6ff0c9bef..854a72b86 100644 --- a/packages/monorepo-scripts/src/publish.ts +++ b/packages/monorepo-scripts/src/publish.ts @@ -1,6 +1,7 @@ #!/usr/bin/env node import * as promisify from 'es6-promisify'; +import * as fs from 'fs'; import * as _ from 'lodash'; import * as moment from 'moment'; import opn = require('opn'); @@ -13,23 +14,12 @@ import { constants } from './constants'; import { Package, PackageToNextVersion, VersionChangelog } from './types'; import { changelogUtils } from './utils/changelog_utils'; import { configs } from './utils/configs'; +import { DocGenerateAndUploadUtils } from './utils/doc_generate_and_upload_utils'; +import { publishReleaseNotesAsync } from './utils/github_release_utils'; import { utils } from './utils/utils'; -const DOC_GEN_COMMAND = 'docs:json'; -const NPM_NAMESPACE = '@0xproject/'; +const NPM_NAMESPACE = '@0x/'; const TODAYS_TIMESTAMP = moment().unix(); -const packageNameToWebsitePath: { [name: string]: string } = { - '0x.js': '0xjs', - 'web3-wrapper': 'web3_wrapper', - contracts: 'contracts', - connect: 'connect', - 'json-schemas': 'json-schemas', - 'sol-compiler': 'sol-compiler', - 'sol-cov': 'sol-cov', - subproviders: 'subproviders', - 'order-utils': 'order-utils', - 'ethereum-types': 'ethereum-types', -}; async function confirmAsync(message: string): Promise<void> { prompt.start(); @@ -44,22 +34,27 @@ async function confirmAsync(message: string): Promise<void> { (async () => { // Fetch public, updated Lerna packages const shouldIncludePrivate = true; - const allUpdatedPackages = await utils.getUpdatedPackagesAsync(shouldIncludePrivate); + const allPackagesToPublish = await utils.getPackagesToPublishAsync(shouldIncludePrivate); + if (_.isEmpty(allPackagesToPublish)) { + utils.log('No packages need publishing'); + process.exit(0); + } + const packagesWithDocs = getPackagesWithDocs(allPackagesToPublish); if (!configs.IS_LOCAL_PUBLISH) { await confirmAsync( 'THIS IS NOT A TEST PUBLISH! You are about to publish one or more packages to npm. Are you sure you want to continue? (y/n)', ); - await confirmDocPagesRenderAsync(allUpdatedPackages); + await confirmDocPagesRenderAsync(packagesWithDocs); } // Update CHANGELOGs - const updatedPublicPackages = _.filter(allUpdatedPackages, pkg => !pkg.packageJson.private); + const updatedPublicPackages = _.filter(allPackagesToPublish, pkg => !pkg.packageJson.private); const updatedPublicPackageNames = _.map(updatedPublicPackages, pkg => pkg.packageJson.name); utils.log(`Will update CHANGELOGs and publish: \n${updatedPublicPackageNames.join('\n')}\n`); const packageToNextVersion = await updateChangeLogsAsync(updatedPublicPackages); - const updatedPrivatePackages = _.filter(allUpdatedPackages, pkg => pkg.packageJson.private); + const updatedPrivatePackages = _.filter(allPackagesToPublish, pkg => pkg.packageJson.private); _.each(updatedPrivatePackages, pkg => { const currentVersion = pkg.packageJson.version; const packageName = pkg.packageJson.name; @@ -83,44 +78,69 @@ async function confirmAsync(message: string): Promise<void> { }); utils.log(`Calling 'lerna publish'...`); await lernaPublishAsync(packageToNextVersion); + if (!configs.IS_LOCAL_PUBLISH) { + const isStaging = false; + const shouldUploadDocs = true; + await generateAndUploadDocJsonsAsync(packagesWithDocs, isStaging, shouldUploadDocs); + } + const isDryRun = configs.IS_LOCAL_PUBLISH; + await publishReleaseNotesAsync(updatedPublicPackages, isDryRun); })().catch(err => { utils.log(err); process.exit(1); }); -async function confirmDocPagesRenderAsync(packages: Package[]): Promise<void> { +function getPackagesWithDocs(allUpdatedPackages: Package[]): Package[] { + const rootPackageJsonPath = `${constants.monorepoRootPath}/package.json`; + const rootPackageJson = JSON.parse(fs.readFileSync(rootPackageJsonPath).toString()); + const packagesWithDocPagesStringIfExist = _.get(rootPackageJson, 'config.packagesWithDocPages', undefined); + if (_.isUndefined(packagesWithDocPagesStringIfExist)) { + return []; // None to generate & publish + } + const packagesWithDocPages = packagesWithDocPagesStringIfExist.split(' '); + const updatedPackagesWithDocPages: Package[] = []; + _.each(allUpdatedPackages, pkg => { + const nameWithoutPrefix = pkg.packageJson.name.replace('@0x/', ''); + if (_.includes(packagesWithDocPages, nameWithoutPrefix)) { + updatedPackagesWithDocPages.push(pkg); + } + }); + return updatedPackagesWithDocPages; +} + +async function generateAndUploadDocJsonsAsync( + packagesWithDocs: Package[], + isStaging: boolean, + shouldUploadDocs: boolean, +): Promise<void> { + for (const pkg of packagesWithDocs) { + const nameWithoutPrefix = pkg.packageJson.name.replace('@0x/', ''); + const docGenerateAndUploadUtils = new DocGenerateAndUploadUtils(nameWithoutPrefix, isStaging, shouldUploadDocs); + await docGenerateAndUploadUtils.generateAndUploadDocsAsync(); + } +} + +async function confirmDocPagesRenderAsync(packagesWithDocs: Package[]): Promise<void> { // push docs to staging utils.log("Upload all docJson's to S3 staging..."); - await execAsync(`yarn stage_docs`, { cwd: constants.monorepoRootPath }); + const isStaging = true; + const shouldUploadDocs = true; + await generateAndUploadDocJsonsAsync(packagesWithDocs, isStaging, shouldUploadDocs); // deploy website to staging utils.log('Deploy website to staging...'); const pathToWebsite = `${constants.monorepoRootPath}/packages/website`; await execAsync(`yarn deploy_staging`, { cwd: pathToWebsite }); - const packagesWithDocs = _.filter(packages, pkg => { - const scriptsIfExists = pkg.packageJson.scripts; - if (_.isUndefined(scriptsIfExists)) { - throw new Error('Found a public package without any scripts in package.json'); - } - return !_.isUndefined(scriptsIfExists[DOC_GEN_COMMAND]); - }); _.each(packagesWithDocs, pkg => { const name = pkg.packageJson.name; - const nameWithoutPrefix = _.startsWith(name, NPM_NAMESPACE) ? name.split('@0xproject/')[1] : name; - const docSegmentIfExists = packageNameToWebsitePath[nameWithoutPrefix]; - if (_.isUndefined(docSegmentIfExists)) { - throw new Error( - `Found package '${name}' with doc commands but no corresponding docSegment in monorepo_scripts -package.ts. Please add an entry for it and try again.`, - ); - } - const link = `${constants.stagingWebsite}/docs/${docSegmentIfExists}`; + const nameWithoutPrefix = _.startsWith(name, NPM_NAMESPACE) ? name.split('@0x/')[1] : name; + const link = `${constants.stagingWebsite}/docs/${nameWithoutPrefix}`; // tslint:disable-next-line:no-floating-promises opn(link); }); - await confirmAsync('Do all the doc pages render properly? (y/n)'); + await confirmAsync('Do all the doc pages render? (y/n)'); } async function pushChangelogsToGithubAsync(): Promise<void> { @@ -153,7 +173,7 @@ async function updateChangeLogsAsync(updatedPublicPackages: Package[]): Promise< version: nextPatchVersionIfValid, changes: [ { - note: 'Dependencies updated', + note: constants.dependenciesUpdatedMessage, }, ], }; diff --git a/packages/monorepo-scripts/src/publish_release_notes.ts b/packages/monorepo-scripts/src/publish_release_notes.ts new file mode 100644 index 000000000..d2082521c --- /dev/null +++ b/packages/monorepo-scripts/src/publish_release_notes.ts @@ -0,0 +1,36 @@ +import * as _ from 'lodash'; +import * as yargs from 'yargs'; + +import { publishReleaseNotesAsync } from './utils/github_release_utils'; +import { utils } from './utils/utils'; + +const args = yargs + .option('isDryRun', { + describe: 'Whether we wish to do a dry run, not committing anything to Github', + type: 'boolean', + demandOption: true, + }) + .option('packages', { + describe: + 'Space-separated list of packages to generated release notes for. If not supplied, it does all `Lerna updated` packages.', + type: 'string', + }) + .example('$0 --isDryRun true --packages "0x.js @0x/web3-wrapper"', 'Full usage example').argv; + +(async () => { + const isDryRun = args.isDryRun; + let packages; + if (_.isUndefined(args.packages)) { + const shouldIncludePrivate = false; + packages = await utils.getPackagesToPublishAsync(shouldIncludePrivate); + } else { + const packageNames = args.packages.split(' '); + packages = await utils.getPackagesByNameAsync(packageNames); + } + + await publishReleaseNotesAsync(packages, isDryRun); + process.exit(0); +})().catch(err => { + utils.log(err); + process.exit(1); +}); diff --git a/packages/monorepo-scripts/src/test_installation.ts b/packages/monorepo-scripts/src/test_installation.ts index 87c4ad1d7..96875d0f9 100644 --- a/packages/monorepo-scripts/src/test_installation.ts +++ b/packages/monorepo-scripts/src/test_installation.ts @@ -12,12 +12,7 @@ import { Package } from './types'; import { utils } from './utils/utils'; // Packages might not be runnable if they are command-line tools or only run in browsers. -const UNRUNNABLE_PACKAGES = [ - '@0xproject/abi-gen', - '@0xproject/sra-report', - '@0xproject/react-shared', - '@0xproject/react-docs', -]; +const UNRUNNABLE_PACKAGES = ['@0x/abi-gen', '@0x/react-shared', '@0x/react-docs']; const mkdirpAsync = promisify(mkdirp); const rimrafAsync = promisify(rimraf); @@ -85,11 +80,13 @@ function logIfDefined(x: any): void { logIfDefined(packageError.error.stdout); logIfDefined(packageError.error.stack); }); + process.exit(1); + } else { process.exit(0); } })().catch(err => { utils.log(`Unexpected error: ${err.message}`); - process.exit(0); + process.exit(1); }); async function testInstallPackageAsync( @@ -102,13 +99,10 @@ async function testInstallPackageAsync( const packageName = installablePackage.packageJson.name; utils.log(`Testing ${packageName}@${lastChangelogVersion}`); const packageDirName = path.join(...(packageName + '-test').split('/')); - const testDirectory = path.join( - monorepoRootPath, - 'packages', - 'monorepo-scripts', - '.installation-test', - packageDirName, - ); + // NOTE(fabio): The `testDirectory` needs to be somewhere **outside** the monorepo root directory. + // Otherwise, it will have access to the hoisted `node_modules` directory and the Typescript missing + // type errors will not be caught. + const testDirectory = path.join(monorepoRootPath, '..', '.installation-test', packageDirName); await rimrafAsync(testDirectory); await mkdirpAsync(testDirectory); await execAsync('yarn init --yes', { cwd: testDirectory }); @@ -122,7 +116,7 @@ async function testInstallPackageAsync( await writeFileAsync(indexFilePath, `import * as Package from '${packageName}';\nconsole.log(Package);\n`); const tsConfig = { compilerOptions: { - typeRoots: ['node_modules/@0xproject/typescript-typings/types', 'node_modules/@types'], + typeRoots: ['node_modules/@0x/typescript-typings/types', 'node_modules/@types'], module: 'commonjs', target: 'es5', lib: ['es2017', 'dom'], @@ -130,6 +124,7 @@ async function testInstallPackageAsync( noImplicitReturns: true, pretty: true, strict: true, + resolveJsonModule: true, }, include: ['index.ts'], }; @@ -144,7 +139,7 @@ async function testInstallPackageAsync( const transpiledIndexFilePath = path.join(testDirectory, 'index.js'); utils.log(`Running test script with ${packageName} imported`); await execAsync(`node ${transpiledIndexFilePath}`); - utils.log(`Successfilly ran test script with ${packageName} imported`); + utils.log(`Successfully ran test script with ${packageName} imported`); } await rimrafAsync(testDirectory); } diff --git a/packages/monorepo-scripts/src/types.ts b/packages/monorepo-scripts/src/types.ts index d9e1dfabb..3c2ec5069 100644 --- a/packages/monorepo-scripts/src/types.ts +++ b/packages/monorepo-scripts/src/types.ts @@ -49,3 +49,25 @@ export interface Package { location: string; packageJson: PackageJSON; } + +export interface DocGenConfigs { + DOC_JSON_VERSION: string; + EXTERNAL_TYPE_TO_LINK: { [externalType: string]: string }; + EXTERNAL_EXPORT_TO_LINK: { [externalExport: string]: string }; + CLASSES_WITH_HIDDEN_CONSTRUCTORS: string[]; + IGNORED_EXCESSIVE_TYPES: string[]; + TYPES_ONLY_LIBRARIES: string[]; +} + +export interface ExportPathToExportedItems { + [pkgName: string]: string[]; +} + +export interface ExportInfo { + exportPathToExportedItems: ExportPathToExportedItems; + exportPathOrder: string[]; +} + +export interface ExportNameToTypedocNames { + [exportName: string]: string[]; +} diff --git a/packages/monorepo-scripts/src/utils/changelog_utils.ts b/packages/monorepo-scripts/src/utils/changelog_utils.ts index 4781b3b7d..0b46bf670 100644 --- a/packages/monorepo-scripts/src/utils/changelog_utils.ts +++ b/packages/monorepo-scripts/src/utils/changelog_utils.ts @@ -19,12 +19,8 @@ CHANGELOG export const changelogUtils = { getChangelogMdTitle(versionChangelog: VersionChangelog): string { - if (_.isUndefined(versionChangelog.timestamp)) { - throw new Error( - 'All CHANGELOG.json entries must be updated to include a timestamp before generating their MD version', - ); - } - const date = moment(`${versionChangelog.timestamp}`, 'X').format('MMMM D, YYYY'); + // Use UTC rather than the local machines time (formatted date time is +0:00) + const date = moment.utc(`${versionChangelog.timestamp}`, 'X').format('MMMM D, YYYY'); const title = `\n## v${versionChangelog.version} - _${date}_\n\n`; return title; }, diff --git a/packages/monorepo-scripts/src/utils/doc_generate_and_upload_utils.ts b/packages/monorepo-scripts/src/utils/doc_generate_and_upload_utils.ts new file mode 100644 index 000000000..1a4294e9c --- /dev/null +++ b/packages/monorepo-scripts/src/utils/doc_generate_and_upload_utils.ts @@ -0,0 +1,492 @@ +import { existsSync, readFileSync, writeFileSync } from 'fs'; +import * as _ from 'lodash'; +import * as path from 'path'; +import { exec as execAsync } from 'promisify-child-process'; +import * as ts from 'typescript'; + +import { constants } from '../constants'; +import { docGenConfigs } from '../doc_gen_configs'; +import { ExportInfo, ExportNameToTypedocNames, ExportPathToExportedItems, PackageJSON } from '../types'; + +import { utils } from './utils'; + +export class DocGenerateAndUploadUtils { + private readonly _isStaging: boolean; + private readonly _shouldUploadDocs: boolean; + private readonly _packageName: string; + private readonly _omitExports: string[]; + private readonly _packagePath: string; + private readonly _exportPathToExportedItems: ExportPathToExportedItems; + private readonly _exportPathOrder: string[]; + private readonly _monoRepoPkgNameToPath: { [name: string]: string }; + private readonly _packageJson: PackageJSON; + /** + * Recursively iterate over the TypeDoc JSON object and find all type names + */ + private static _getAllTypeNames(node: any, typeNames: string[]): string[] { + if (!_.isObject(node)) { + return typeNames; + } + const typeKindStrings = ['Interface', 'Enumeration', 'Type alias']; + if (_.includes(typeKindStrings, node.kindString)) { + return [...typeNames, node.name]; + } + let updatedTypeNames = typeNames; + _.each(node, nodeValue => { + if (_.isArray(nodeValue)) { + _.each(nodeValue, aNode => { + updatedTypeNames = DocGenerateAndUploadUtils._getAllTypeNames(aNode, updatedTypeNames); + }); + } else if (_.isObject(nodeValue)) { + updatedTypeNames = DocGenerateAndUploadUtils._getAllTypeNames(nodeValue, updatedTypeNames); + } + }); + return updatedTypeNames; + } + /** + * Recursively iterate over the TypeDoc JSON object and find all reference names (i.e types, classNames, + * objectLiteral names, etc...) + */ + private static _getAllReferenceNames(propertyName: string, node: any, referenceNames: string[]): string[] { + if (!_.isObject(node)) { + return referenceNames; + } + + let updatedReferenceNames = referenceNames; + // Some nodes of type reference are for subtypes, which we don't want to return. + // We therefore filter them out. + const SUB_TYPE_PROPERTY_NAMES = ['inheritedFrom', 'overwrites', 'extendedTypes', 'implementationOf']; + const TS_MAPPED_TYPES = ['Partial', 'Promise', 'Readonly', 'Pick', 'Record']; + if ( + !_.isUndefined(node.type) && + _.isString(node.type) && + node.type === 'reference' && + !_.includes(TS_MAPPED_TYPES, node.name) && + !_.includes(SUB_TYPE_PROPERTY_NAMES, propertyName) + ) { + updatedReferenceNames = _.uniq([...referenceNames, node.name]); + return updatedReferenceNames; + } + _.each(node, (nodeValue, innerPropertyName) => { + if (_.isArray(nodeValue)) { + _.each(nodeValue, aNode => { + updatedReferenceNames = DocGenerateAndUploadUtils._getAllReferenceNames( + innerPropertyName, + aNode, + updatedReferenceNames, + ); + }); + } else if (_.isObject(nodeValue)) { + updatedReferenceNames = DocGenerateAndUploadUtils._getAllReferenceNames( + innerPropertyName, + nodeValue, + updatedReferenceNames, + ); + } + }); + return _.uniq(updatedReferenceNames); + } + private static _getExportPathToExportedItems(filePath: string, omitExports?: string[]): ExportInfo { + const sourceFile = ts.createSourceFile( + 'indexFile', + readFileSync(filePath).toString(), + ts.ScriptTarget.ES2017, + /*setParentNodes */ true, + ); + const exportPathToExportedItems: ExportPathToExportedItems = {}; + const exportPathOrder: string[] = []; + const exportsToOmit = _.isUndefined(omitExports) ? [] : omitExports; + + processNode(sourceFile); + + function processNode(node: ts.Node): void { + switch (node.kind) { + case ts.SyntaxKind.ExportDeclaration: { + const exportClause = (node as any).exportClause; + if (_.isUndefined(exportClause)) { + return; + } + const exportPath = exportClause.parent.moduleSpecifier.text; + _.each(exportClause.elements, element => { + const exportItem = element.name.escapedText; + if (!_.includes(exportsToOmit, exportItem)) { + exportPathToExportedItems[exportPath] = _.isUndefined(exportPathToExportedItems[exportPath]) + ? [exportItem] + : [...exportPathToExportedItems[exportPath], exportItem]; + } + }); + if (!_.isUndefined(exportPathToExportedItems[exportPath])) { + exportPathOrder.push(exportPath); + } + break; + } + + case ts.SyntaxKind.ExportKeyword: { + const foundNode: any = node; + let exportPath = './index'; + if (foundNode.parent && foundNode.parent.name) { + const exportItem = foundNode.parent.name.escapedText; + const isExportImportRequireStatement = !_.isUndefined( + _.get(foundNode, 'parent.moduleReference.expression.text'), + ); + if (isExportImportRequireStatement) { + exportPath = foundNode.parent.moduleReference.expression.text; + } + if (!_.includes(exportsToOmit, exportItem)) { + exportPathToExportedItems[exportPath] = _.isUndefined(exportPathToExportedItems[exportPath]) + ? [exportItem] + : [...exportPathToExportedItems[exportPath], exportItem]; + } + } + if ( + !_.includes(exportPathOrder, exportPath) && + !_.isUndefined(exportPathToExportedItems[exportPath]) + ) { + exportPathOrder.push(exportPath); + } + break; + } + default: + // noop + break; + } + + ts.forEachChild(node, processNode); + } + const exportInfo = { + exportPathToExportedItems, + exportPathOrder, + }; + return exportInfo; + } + constructor(packageName: string, isStaging: boolean, shouldUploadDocs: boolean) { + this._isStaging = isStaging; + this._packageName = packageName; + this._shouldUploadDocs = shouldUploadDocs; + this._packagePath = `${constants.monorepoRootPath}/packages/${packageName}`; + + this._monoRepoPkgNameToPath = {}; + const monorepoPackages = utils.getPackages(constants.monorepoRootPath); + _.each(monorepoPackages, p => (this._monoRepoPkgNameToPath[p.packageJson.name] = p.location)); + + const pkg = _.find(monorepoPackages, monorepoPackage => { + return _.includes(monorepoPackage.packageJson.name, packageName); + }); + if (_.isUndefined(pkg)) { + throw new Error(`Couldn't find a package.json for ${packageName}`); + } + this._packageJson = pkg.packageJson; + this._omitExports = _.get(this._packageJson, 'config.postpublish.docOmitExports', []); + + const indexPath = `${this._packagePath}/src/index.ts`; + const exportInfo = DocGenerateAndUploadUtils._getExportPathToExportedItems(indexPath, this._omitExports); + this._exportPathToExportedItems = exportInfo.exportPathToExportedItems; + this._exportPathOrder = exportInfo.exportPathOrder; + } + public async generateAndUploadDocsAsync(): Promise<void> { + // For each dep that is another one of our monorepo packages, we fetch it's index.ts + // and see which specific files we must pass to TypeDoc, in order to generate a Doc JSON + // the includes everything exported by the public interface. + const typeDocExtraFileIncludes: string[] = this._getTypeDocFileIncludesForPackage(); + + // In order to avoid TS errors, we need to pass TypeDoc the package's global.d.ts file + // if it exists. + const globalTypeDefinitionsPath = path.join(this._packagePath, 'src', 'globals.d.ts'); + if (existsSync(globalTypeDefinitionsPath)) { + typeDocExtraFileIncludes.push(globalTypeDefinitionsPath); + } + + utils.log(`GENERATE_UPLOAD_DOCS: Generating Typedoc JSON for ${this._packageName}...`); + const jsonFilePath = path.join(this._packagePath, 'generated_docs', 'index.json'); + const projectFiles = typeDocExtraFileIncludes.join(' '); + const cwd = path.join(constants.monorepoRootPath, 'packages', this._packageName); + // HACK: For some reason calling `typedoc` command directly from here, even with `cwd` set to the + // packages root dir, does not work. It only works when called via a `package.json` script located + // in the package's root. + await execAsync(`JSON_FILE_PATH=${jsonFilePath} PROJECT_FILES="${projectFiles}" yarn docs:json`, { + cwd, + }); + + utils.log('GENERATE_UPLOAD_DOCS: Modifying Typedoc JSON to our custom format...'); + const typedocOutputString = readFileSync(jsonFilePath).toString(); + const typedocOutput = JSON.parse(typedocOutputString); + const standardizedTypedocOutput = this._standardizeTypedocOutputTopLevelChildNames(typedocOutput); + const modifiedTypedocOutput = this._pruneTypedocOutput(standardizedTypedocOutput); + + if (!_.includes(docGenConfigs.TYPES_ONLY_LIBRARIES, this._packageName)) { + const propertyName = ''; // Root has no property name + const referenceNames = DocGenerateAndUploadUtils._getAllReferenceNames( + propertyName, + modifiedTypedocOutput, + [], + ); + this._lookForUnusedExportedTypesThrowIfExists(referenceNames, modifiedTypedocOutput); + this._lookForMissingReferenceExportsThrowIfExists(referenceNames); + } + + // Some of our packages re-export external package exports in their index.ts + // Typedoc is incapable of rendering these packages, so we need to special-case them + const externalExportToLink: { [externalExport: string]: string } = {}; + const externalExportsWithoutLinks: string[] = []; + const externalExports: string[] = this._getAllExternalExports(); + _.each(externalExports, externalExport => { + const linkIfExists = docGenConfigs.EXTERNAL_EXPORT_TO_LINK[externalExport]; + if (_.isUndefined(linkIfExists)) { + externalExportsWithoutLinks.push(externalExport); + return; + } + externalExportToLink[externalExport] = linkIfExists; + }); + if (!_.isEmpty(externalExportsWithoutLinks)) { + throw new Error( + `Found the following external exports in ${ + this._packageName + }'s index.ts:\n ${externalExportsWithoutLinks.join( + '\n', + )}\nThey are missing from the EXTERNAL_EXPORT_TO_LINK mapping. Add them and try again.`, + ); + } + + const exportPathToTypedocNames: ExportNameToTypedocNames = {}; + _.each(modifiedTypedocOutput.children, file => { + const exportPath = this._findExportPathGivenTypedocName(file.name); + exportPathToTypedocNames[exportPath] = _.isUndefined(exportPathToTypedocNames[exportPath]) + ? [file.name] + : [...exportPathToTypedocNames[exportPath], file.name]; + }); + + // Since we need additional metadata included in the doc JSON, we nest the TypeDoc JSON + // within our own custom, versioned docsJson format. + const docJson = { + version: docGenConfigs.DOC_JSON_VERSION, + metadata: { + exportPathToTypedocNames, + exportPathOrder: this._exportPathOrder, + externalTypeToLink: docGenConfigs.EXTERNAL_TYPE_TO_LINK, + externalExportToLink, + }, + typedocJson: modifiedTypedocOutput, + }; + utils.log(`GENERATE_UPLOAD_DOCS: Saving Doc JSON to: ${jsonFilePath}`); + writeFileSync(jsonFilePath, JSON.stringify(docJson, null, 2)); + + if (this._shouldUploadDocs) { + await this._uploadDocsAsync(jsonFilePath, cwd); + } + utils.log(`GENERATE_UPLOAD_DOCS: Doc generation done for ${this._packageName}`); + } + private async _uploadDocsAsync(jsonFilePath: string, cwd: string): Promise<void> { + const fileName = `v${this._packageJson.version}.json`; + utils.log(`GENERATE_UPLOAD_DOCS: Doc generation successful, uploading docs... as ${fileName}`); + + const S3BucketPath = this._isStaging + ? `s3://staging-doc-jsons/${this._packageName}/` + : `s3://doc-jsons/${this._packageName}/`; + const s3Url = `${S3BucketPath}${fileName}`; + await execAsync( + `aws s3 cp ${jsonFilePath} ${s3Url} --profile 0xproject --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers --content-type application/json`, + { + cwd, + }, + ); + utils.log(`GENERATE_UPLOAD_DOCS: Docs uploaded to S3 bucket: ${S3BucketPath}`); + // Remove the generated docs directory + await execAsync(`rm -rf ${jsonFilePath}`, { + cwd, + }); + } + /** + * Look for types that are used by the public interface but are missing from a package's index.ts + */ + private _lookForMissingReferenceExportsThrowIfExists(referenceNames: string[]): void { + const allExportedItems = _.flatten(_.values(this._exportPathToExportedItems)); + const missingReferences: string[] = []; + _.each(referenceNames, referenceName => { + if ( + !_.includes(allExportedItems, referenceName) && + _.isUndefined(docGenConfigs.EXTERNAL_TYPE_TO_LINK[referenceName]) + ) { + missingReferences.push(referenceName); + } + }); + if (!_.isEmpty(missingReferences)) { + throw new Error( + `${this._packageName} package needs to export: \n${missingReferences.join( + '\n', + )} \nFrom it\'s index.ts. If any are from external dependencies, then add them to the EXTERNAL_TYPE_TO_LINK mapping.`, + ); + } + } + /** + * Look for exported types that are not used by the package's public interface + */ + private _lookForUnusedExportedTypesThrowIfExists(referenceNames: string[], typedocOutput: any): void { + const exportedTypes = DocGenerateAndUploadUtils._getAllTypeNames(typedocOutput, []); + const excessiveReferences = _.difference(exportedTypes, referenceNames); + const excessiveReferencesExceptIgnored = _.difference( + excessiveReferences, + docGenConfigs.IGNORED_EXCESSIVE_TYPES, + ); + if (!_.isEmpty(excessiveReferencesExceptIgnored)) { + throw new Error( + `${this._packageName} package exports BUT does not need: \n${excessiveReferencesExceptIgnored.join( + '\n', + )} \nin it\'s index.ts. Remove them then try again OR if we still want them exported (e.g error enum types), then add them to the IGNORED_EXCESSIVE_TYPES array.`, + ); + } + } + /** + * For each entry in the TypeDoc JSON, remove it if: + * - it was not exported in index.ts + * - the constructor is to be ignored + * - it begins with an underscore (i.e is private) + */ + private _pruneTypedocOutput(typedocOutput: any): any { + const modifiedTypedocOutput = _.cloneDeep(typedocOutput); + _.each(typedocOutput.children, (file, i) => { + const exportPath = this._findExportPathGivenTypedocName(file.name); + const exportItems = this._exportPathToExportedItems[exportPath]; + _.each(file.children, (child, j) => { + const isNotExported = !_.includes(exportItems, child.name); + if (isNotExported) { + delete modifiedTypedocOutput.children[i].children[j]; + return; + } + + const innerChildren = typedocOutput.children[i].children[j].children; + _.each(innerChildren, (innerChild, k) => { + const isHiddenConstructor = + child.kindString === 'Class' && + _.includes(docGenConfigs.CLASSES_WITH_HIDDEN_CONSTRUCTORS, child.name) && + innerChild.kindString === 'Constructor'; + const isPrivate = _.startsWith(innerChild.name, '_'); + if (isHiddenConstructor || isPrivate) { + delete modifiedTypedocOutput.children[i].children[j].children[k]; + } + }); + modifiedTypedocOutput.children[i].children[j].children = _.compact( + modifiedTypedocOutput.children[i].children[j].children, + ); + }); + modifiedTypedocOutput.children[i].children = _.compact(modifiedTypedocOutput.children[i].children); + }); + return modifiedTypedocOutput; + } + /** + * Unfortunately TypeDoc children names will only be prefixed with the name of the package _if_ we passed + * TypeDoc files outside of the packages root path (i.e this package exports another package from our + * monorepo). In order to enforce that the names are always prefixed with the package's name, we check and add + * them here when necessary. + */ + private _standardizeTypedocOutputTopLevelChildNames(typedocOutput: any): any { + const modifiedTypedocOutput = _.cloneDeep(typedocOutput); + _.each(typedocOutput.children, (child, i) => { + if (!_.includes(child.name, '/src/')) { + const nameWithoutQuotes = child.name.replace(/"/g, ''); + const standardizedName = `"${this._packageName}/src/${nameWithoutQuotes}"`; + modifiedTypedocOutput.children[i].name = standardizedName; + } + }); + return modifiedTypedocOutput; + } + /** + * Maps back each top-level TypeDoc JSON object name to the exportPath from which it was generated. + */ + private _findExportPathGivenTypedocName(typedocName: string): string { + const typeDocNameWithoutQuotes = _.replace(typedocName, /"/g, ''); + const sanitizedExportPathToExportPath: { [sanitizedName: string]: string } = {}; + const exportPaths = _.keys(this._exportPathToExportedItems); + const sanitizedExportPaths = _.map(exportPaths, exportPath => { + if (_.startsWith(exportPath, './')) { + const sanitizedExportPath = path.join(this._packageName, 'src', exportPath); + sanitizedExportPathToExportPath[sanitizedExportPath] = exportPath; + return sanitizedExportPath; + } + const monorepoPrefix = '@0x/'; + if (_.startsWith(exportPath, monorepoPrefix)) { + const sanitizedExportPath = exportPath.split(monorepoPrefix)[1]; + sanitizedExportPathToExportPath[sanitizedExportPath] = exportPath; + return sanitizedExportPath; + } + sanitizedExportPathToExportPath[exportPath] = exportPath; + return exportPath; + }); + // We need to sort the exportPaths by length (longest first), so that the match finding will pick + // longer matches before shorter matches, since it might match both, but the longer match is more + // precisely what we are looking for. + const sanitizedExportPathsSortedByLength = sanitizedExportPaths.sort((a: string, b: string) => { + return b.length - a.length; + }); + const matchingSanitizedExportPathIfExists = _.find(sanitizedExportPathsSortedByLength, p => { + return _.startsWith(typeDocNameWithoutQuotes, p); + }); + if (_.isUndefined(matchingSanitizedExportPathIfExists)) { + throw new Error(`Didn't find an exportPath for ${typeDocNameWithoutQuotes}`); + } + const matchingExportPath = sanitizedExportPathToExportPath[matchingSanitizedExportPathIfExists]; + return matchingExportPath; + } + private _getAllExternalExports(): string[] { + const externalExports: string[] = []; + _.each(this._exportPathToExportedItems, (exportedItems, exportPath) => { + const pathIfExists = this._monoRepoPkgNameToPath[exportPath]; + if (_.isUndefined(pathIfExists) && !_.startsWith(exportPath, './')) { + _.each(exportedItems, exportedItem => { + externalExports.push(exportedItem); + }); + return; // It's an external package + } + }); + return externalExports; + } + private _getTypeDocFileIncludesForPackage(): string[] { + let typeDocExtraFileIncludes: string[] = []; + _.each(this._exportPathToExportedItems, (exportedItems, exportPath) => { + const isInternalToPkg = _.startsWith(exportPath, '.'); + if (isInternalToPkg) { + const pathToInternalPkg = path.join(this._packagePath, 'src', `${exportPath}.ts`); + typeDocExtraFileIncludes.push(pathToInternalPkg); + return; + } + + const pathIfExists = this._monoRepoPkgNameToPath[exportPath]; + if (_.isUndefined(pathIfExists)) { + return; // It's an external package + } + + const typeDocSourceIncludes = new Set(); + const pathToIndex = `${pathIfExists}/src/index.ts`; + const exportInfo = DocGenerateAndUploadUtils._getExportPathToExportedItems(pathToIndex); + const innerExportPathToExportedItems = exportInfo.exportPathToExportedItems; + _.each(exportedItems, exportName => { + _.each(innerExportPathToExportedItems, (innerExportItems, innerExportPath) => { + if (!_.includes(innerExportItems, exportName)) { + return; + } + if (!_.startsWith(innerExportPath, './')) { + throw new Error( + `GENERATE_UPLOAD_DOCS: WARNING - ${ + this._packageName + } is exporting one of ${innerExportItems} which is + itself exported from an external package. To fix this, export the external dependency directly, + not indirectly through ${innerExportPath}.`, + ); + } else { + const absoluteSrcPath = path.join(pathIfExists, 'src', `${innerExportPath}.ts`); + typeDocSourceIncludes.add(absoluteSrcPath); + } + }); + }); + + // @0x/types & ethereum-types are examples of packages where their index.ts exports types + // directly, meaning no internal paths will exist to follow. Other packages also have direct exports + // in their index.ts, so we always add it to the source files passed to TypeDoc + if (typeDocSourceIncludes.size === 0) { + typeDocSourceIncludes.add(pathToIndex); + } + + typeDocExtraFileIncludes = [...typeDocExtraFileIncludes, ...Array.from(typeDocSourceIncludes)]; + }); + return typeDocExtraFileIncludes; + } +} diff --git a/packages/monorepo-scripts/src/utils/github_release_utils.ts b/packages/monorepo-scripts/src/utils/github_release_utils.ts new file mode 100644 index 000000000..7434d397e --- /dev/null +++ b/packages/monorepo-scripts/src/utils/github_release_utils.ts @@ -0,0 +1,119 @@ +import * as promisify from 'es6-promisify'; +import { readFileSync } from 'fs'; +import * as _ from 'lodash'; +import * as path from 'path'; +import { exec as execAsync } from 'promisify-child-process'; +import * as publishRelease from 'publish-release'; + +import { constants } from '../constants'; +import { Package } from '../types'; + +import { utils } from './utils'; + +const publishReleaseAsync = promisify(publishRelease); +// tslint:disable-next-line:completed-docs +export async function publishReleaseNotesAsync(packagesToPublish: Package[], isDryRun: boolean): Promise<void> { + // Git push a tag representing this publish (publish-{commit-hash}) (truncate hash) + const result = await execAsync('git log -n 1 --pretty=format:"%H"', { cwd: constants.monorepoRootPath }); + const latestGitCommit = result.stdout; + const prefixLength = 7; + const shortenedGitCommit = latestGitCommit.slice(0, prefixLength); + const tagName = `monorepo@${shortenedGitCommit}`; + + if (!isDryRun) { + try { + await execAsync(`git tag ${tagName}`); + } catch (err) { + if (_.includes(err.message, 'already exists')) { + // Noop tag creation since already exists + } else { + throw err; + } + } + const { stdout } = await execAsync(`git ls-remote --tags origin refs/tags/${tagName}`); + if (_.isEmpty(stdout)) { + await execAsync(`git push origin ${tagName}`); + } + } + + const releaseName = `0x monorepo - ${shortenedGitCommit}`; + + let assets: string[] = []; + let aggregateNotes = ''; + _.each(packagesToPublish, pkg => { + aggregateNotes += getReleaseNotesForPackage(pkg.packageJson.name); + + const packageAssets = _.get(pkg.packageJson, 'config.postpublish.assets'); + if (!_.isUndefined(packageAssets)) { + assets = [...assets, ...packageAssets]; + } + }); + const finalAssets = adjustAssetPaths(assets); + + const publishReleaseConfigs = { + token: constants.githubPersonalAccessToken, + owner: '0xProject', + tag: tagName, + repo: '0x-monorepo', + name: releaseName, + notes: aggregateNotes, + draft: false, + prerelease: false, + reuseRelease: true, + reuseDraftOnly: false, + // TODO: Currently publish-release doesn't let you specify the labels for each asset uploaded + // Ideally we would like to name the assets after the package they are from + // Source: https://github.com/remixz/publish-release/issues/39 + assets: finalAssets, + }; + + if (isDryRun) { + utils.log(`Dry run: stopping short of publishing release notes to github`); + utils.log(`Would publish with configs:\n${JSON.stringify(publishReleaseConfigs, null, '\t')}`); + return; + } + + utils.log('Publishing release notes ', releaseName, '...'); + await publishReleaseAsync(publishReleaseConfigs); +} + +// Asset paths should described from the monorepo root. This method prefixes +// the supplied path with the absolute path to the monorepo root. +function adjustAssetPaths(assets: string[]): string[] { + const finalAssets: string[] = []; + _.each(assets, (asset: string) => { + const finalAsset = `${constants.monorepoRootPath}/${asset}`; + finalAssets.push(finalAsset); + }); + return finalAssets; +} + +function getReleaseNotesForPackage(packageName: string): string { + const packageNameWithoutNamespace = packageName.replace('@0x/', ''); + const changelogJSONPath = path.join( + constants.monorepoRootPath, + 'packages', + packageNameWithoutNamespace, + 'CHANGELOG.json', + ); + const changelogJSON = readFileSync(changelogJSONPath, 'utf-8'); + const changelogs = JSON.parse(changelogJSON); + const latestLog = changelogs[0]; + // If only has a `Dependencies updated` changelog, we don't include it in release notes + if (latestLog.changes.length === 1 && latestLog.changes[0].note === constants.dependenciesUpdatedMessage) { + return ''; + } + let notes = ''; + _.each(latestLog.changes, change => { + notes += `* ${change.note}`; + if (change.pr) { + notes += ` (#${change.pr})`; + } + notes += `\n`; + }); + if (_.isEmpty(notes)) { + return ''; // don't include it + } + const releaseNotesSection = `### ${packageName}@${latestLog.version}\n${notes}\n\n`; + return releaseNotesSection; +} diff --git a/packages/monorepo-scripts/src/utils/utils.ts b/packages/monorepo-scripts/src/utils/utils.ts index 26ac801bd..44ff971e8 100644 --- a/packages/monorepo-scripts/src/utils/utils.ts +++ b/packages/monorepo-scripts/src/utils/utils.ts @@ -48,13 +48,20 @@ export const utils = { }; packages.push(pkg); } catch (err) { - utils.log(`Couldn't find a 'package.json' for ${subpackageName}. Skipping.`); + // Couldn't find a 'package.json' for package. Skipping. } } } return packages; }, - async getUpdatedPackagesAsync(shouldIncludePrivate: boolean): Promise<Package[]> { + async getPackagesByNameAsync(packageNames: string[]): Promise<Package[]> { + const allPackages = utils.getPackages(constants.monorepoRootPath); + const updatedPackages = _.filter(allPackages, pkg => { + return _.includes(packageNames, pkg.packageJson.name); + }); + return updatedPackages; + }, + async getPackagesToPublishAsync(shouldIncludePrivate: boolean): Promise<Package[]> { const updatedPublicPackages = await utils.getLernaUpdatedPackagesAsync(shouldIncludePrivate); const updatedPackageNames = _.map(updatedPublicPackages, pkg => pkg.name); diff --git a/packages/monorepo-scripts/tsconfig.json b/packages/monorepo-scripts/tsconfig.json index 332d3a5e1..c8b1d23e5 100644 --- a/packages/monorepo-scripts/tsconfig.json +++ b/packages/monorepo-scripts/tsconfig.json @@ -2,7 +2,8 @@ "extends": "../../tsconfig", "compilerOptions": { "typeRoots": ["../../node_modules/@types", "node_modules/@types"], - "outDir": "lib" + "outDir": "lib", + "rootDir": "src" }, "include": ["./src/**/*"] } |