Merge branch 'development'

* development: (939 commits)
  Add asset-buyer to published packages section in README
  Publish
  Updated CHANGELOGS
  Update BuyQuote interface
  force re-build
  Add website build to instructions
  Revert format and re-add changes
  Build website in parallel with other tests since no other test relies on it being built to run
  Add back sourceMap support for both dev/prod
  Upgrade webpack
  Add missing default options
  Remove unused constants
  Add fee order with a takerFee
  Add additional order factory methods and refactor test to use them
  Add comments about buy quote calculation
  Update CHANGELOG
  Fix linter
  Add additional test for slippage
  Add buy_quote_calculator_test
  Add 0x Instant to bundle analysis
  ...

1 files changed, 290 insertions, 105 deletions

diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts
index 3620a3ec1..7eefc1474 100644
--- a/packages/sol-compiler/src/compiler.ts
+++ b/packages/sol-compiler/src/compiler.ts
@@ -10,6 +10,7 @@ import {
 } from '@0xproject/sol-resolver';
 import { fetchAsync, logUtils } from '@0xproject/utils';
 import chalk from 'chalk';
+import { CompilerOptions, ContractArtifact, ContractVersionData, StandardOutput } from 'ethereum-types';
 import * as ethUtil from 'ethereumjs-util';
 import * as fs from 'fs';
 import * as _ from 'lodash';
@@ -29,7 +30,6 @@ import {
 } from './utils/compiler';
 import { constants } from './utils/constants';
 import { fsWrapper } from './utils/fs_wrapper';
-import { CompilerOptions, ContractArtifact, ContractVersionData } from './utils/types';
 import { utils } from './utils/utils';
 
 type TYPE_ALL_FILES_IDENTIFIER = '*';
@@ -53,6 +53,23 @@ const DEFAULT_COMPILER_SETTINGS: solc.CompilerSettings = {
 };
 const CONFIG_FILE = 'compiler.json';
 
+interface VersionToInputs {
+    [solcVersion: string]: {
+        standardInput: solc.StandardInput;
+        contractsToCompile: string[];
+    };
+}
+
+interface ContractPathToData {
+    [contractPath: string]: ContractData;
+}
+
+interface ContractData {
+    currentArtifactIfExists: ContractArtifact | void;
+    sourceTreeHashHex: string;
+    contractName: string;
+}
+
 /**
  * The Compiler facilitates compiling Solidity smart contracts and saves the results
  * to artifact files.
@@ -65,8 +82,52 @@ export class Compiler {
     private readonly _artifactsDir: string;
     private readonly _solcVersionIfExists: string | undefined;
     private readonly _specifiedContracts: string[] | TYPE_ALL_FILES_IDENTIFIER;
+    private static async _getSolcAsync(
+        solcVersion: string,
+    ): Promise<{ solcInstance: solc.SolcInstance; fullSolcVersion: string }> {
+        const fullSolcVersion = binPaths[solcVersion];
+        if (_.isUndefined(fullSolcVersion)) {
+            throw new Error(`${solcVersion} is not a known compiler version`);
+        }
+        const compilerBinFilename = path.join(SOLC_BIN_DIR, fullSolcVersion);
+        let solcjs: string;
+        if (await fsWrapper.doesFileExistAsync(compilerBinFilename)) {
+            solcjs = (await fsWrapper.readFileAsync(compilerBinFilename)).toString();
+        } else {
+            logUtils.warn(`Downloading ${fullSolcVersion}...`);
+            const url = `${constants.BASE_COMPILER_URL}${fullSolcVersion}`;
+            const response = await fetchAsync(url);
+            const SUCCESS_STATUS = 200;
+            if (response.status !== SUCCESS_STATUS) {
+                throw new Error(`Failed to load ${fullSolcVersion}`);
+            }
+            solcjs = await response.text();
+            await fsWrapper.writeFileAsync(compilerBinFilename, solcjs);
+        }
+        if (solcjs.length === 0) {
+            throw new Error('No compiler available');
+        }
+        const solcInstance = solc.setupMethods(requireFromString(solcjs, compilerBinFilename));
+        return { solcInstance, fullSolcVersion };
+    }
+    private static _addHexPrefixToContractBytecode(compiledContract: solc.StandardContractOutput): void {
+        if (!_.isUndefined(compiledContract.evm)) {
+            if (!_.isUndefined(compiledContract.evm.bytecode) && !_.isUndefined(compiledContract.evm.bytecode.object)) {
+                compiledContract.evm.bytecode.object = ethUtil.addHexPrefix(compiledContract.evm.bytecode.object);
+            }
+            if (
+                !_.isUndefined(compiledContract.evm.deployedBytecode) &&
+                !_.isUndefined(compiledContract.evm.deployedBytecode.object)
+            ) {
+                compiledContract.evm.deployedBytecode.object = ethUtil.addHexPrefix(
+                    compiledContract.evm.deployedBytecode.object,
+                );
+            }
+        }
+    }
     /**
      * Instantiates a new instance of the Compiler class.
+     * @param opts Optional compiler options
      * @return An instance of the Compiler class.
      */
     constructor(opts?: CompilerOptions) {
@@ -98,129 +159,153 @@ export class Compiler {
     public async compileAsync(): Promise<void> {
         await createDirIfDoesNotExistAsync(this._artifactsDir);
         await createDirIfDoesNotExistAsync(SOLC_BIN_DIR);
-        let contractNamesToCompile: string[] = [];
+        await this._compileContractsAsync(this._getContractNamesToCompile(), true);
+    }
+    /**
+     * Compiles Solidity files specified during instantiation, and returns the
+     * compiler output given by solc.  Return value is an array of outputs:
+     * Solidity modules are batched together by version required, and each
+     * element of the returned array corresponds to a compiler version, and
+     * each element contains the output for all of the modules compiled with
+     * that version.
+     */
+    public async getCompilerOutputsAsync(): Promise<StandardOutput[]> {
+        const promisedOutputs = this._compileContractsAsync(this._getContractNamesToCompile(), false);
+        return promisedOutputs;
+    }
+    private _getContractNamesToCompile(): string[] {
+        let contractNamesToCompile;
         if (this._specifiedContracts === ALL_CONTRACTS_IDENTIFIER) {
             const allContracts = this._nameResolver.getAll();
             contractNamesToCompile = _.map(allContracts, contractSource =>
                 path.basename(contractSource.path, constants.SOLIDITY_FILE_EXTENSION),
             );
         } else {
-            contractNamesToCompile = this._specifiedContracts;
-        }
-        for (const contractNameToCompile of contractNamesToCompile) {
-            await this._compileContractAsync(contractNameToCompile);
+            contractNamesToCompile = this._specifiedContracts.map(specifiedContract =>
+                path.basename(specifiedContract, constants.SOLIDITY_FILE_EXTENSION),
+            );
         }
+        return contractNamesToCompile;
     }
     /**
-     * Compiles contract and saves artifact to artifactsDir.
+     * Compiles contracts, and, if `shouldPersist` is true, saves artifacts to artifactsDir.
      * @param fileName Name of contract with '.sol' extension.
+     * @return an array of compiler outputs, where each element corresponds to a different version of solc-js.
      */
-    private async _compileContractAsync(contractName: string): Promise<void> {
-        const contractSource = this._resolver.resolve(contractName);
-        const absoluteContractPath = path.join(this._contractsDir, contractSource.path);
-        const currentArtifactIfExists = await getContractArtifactIfExistsAsync(this._artifactsDir, contractName);
-        const sourceTreeHashHex = `0x${this._getSourceTreeHash(absoluteContractPath).toString('hex')}`;
-        let shouldCompile = false;
-        if (_.isUndefined(currentArtifactIfExists)) {
-            shouldCompile = true;
-        } else {
-            const currentArtifact = currentArtifactIfExists as ContractArtifact;
-            const isUserOnLatestVersion = currentArtifact.schemaVersion === constants.LATEST_ARTIFACT_VERSION;
-            const didCompilerSettingsChange = !_.isEqual(currentArtifact.compiler.settings, this._compilerSettings);
-            const didSourceChange = currentArtifact.sourceTreeHashHex !== sourceTreeHashHex;
-            shouldCompile = !isUserOnLatestVersion || didCompilerSettingsChange || didSourceChange;
-        }
-        if (!shouldCompile) {
-            return;
-        }
-        let solcVersion = this._solcVersionIfExists;
-        if (_.isUndefined(solcVersion)) {
-            const solcVersionRange = parseSolidityVersionRange(contractSource.source);
-            const availableCompilerVersions = _.keys(binPaths);
-            solcVersion = semver.maxSatisfying(availableCompilerVersions, solcVersionRange);
-        }
-        const fullSolcVersion = binPaths[solcVersion];
-        const compilerBinFilename = path.join(SOLC_BIN_DIR, fullSolcVersion);
-        let solcjs: string;
-        const isCompilerAvailableLocally = fs.existsSync(compilerBinFilename);
-        if (isCompilerAvailableLocally) {
-            solcjs = fs.readFileSync(compilerBinFilename).toString();
-        } else {
-            logUtils.log(`Downloading ${fullSolcVersion}...`);
-            const url = `${constants.BASE_COMPILER_URL}${fullSolcVersion}`;
-            const response = await fetchAsync(url);
-            const SUCCESS_STATUS = 200;
-            if (response.status !== SUCCESS_STATUS) {
-                throw new Error(`Failed to load ${fullSolcVersion}`);
-            }
-            solcjs = await response.text();
-            fs.writeFileSync(compilerBinFilename, solcjs);
-        }
-        const solcInstance = solc.setupMethods(requireFromString(solcjs, compilerBinFilename));
+    private async _compileContractsAsync(contractNames: string[], shouldPersist: boolean): Promise<StandardOutput[]> {
+        // batch input contracts together based on the version of the compiler that they require.
+        const versionToInputs: VersionToInputs = {};
 
-        logUtils.log(`Compiling ${contractName} with Solidity v${solcVersion}...`);
-        const standardInput: solc.StandardInput = {
-            language: 'Solidity',
-            sources: {
-                [contractSource.path]: {
-                    content: contractSource.source,
-                },
-            },
-            settings: this._compilerSettings,
-        };
-        const compiled: solc.StandardOutput = JSON.parse(
-            solcInstance.compileStandardWrapper(JSON.stringify(standardInput), importPath => {
-                const sourceCodeIfExists = this._resolver.resolve(importPath);
-                return { contents: sourceCodeIfExists.source };
-            }),
-        );
+        // map contract paths to data about them for later verification and persistence
+        const contractPathToData: ContractPathToData = {};
 
-        if (!_.isUndefined(compiled.errors)) {
-            const SOLIDITY_WARNING = 'warning';
-            const errors = _.filter(compiled.errors, entry => entry.severity !== SOLIDITY_WARNING);
-            const warnings = _.filter(compiled.errors, entry => entry.severity === SOLIDITY_WARNING);
-            if (!_.isEmpty(errors)) {
-                errors.forEach(error => {
-                    const normalizedErrMsg = getNormalizedErrMsg(error.formattedMessage || error.message);
-                    logUtils.log(chalk.red(normalizedErrMsg));
-                });
-                process.exit(1);
-            } else {
-                warnings.forEach(warning => {
-                    const normalizedWarningMsg = getNormalizedErrMsg(warning.formattedMessage || warning.message);
-                    logUtils.log(chalk.yellow(normalizedWarningMsg));
-                });
+        for (const contractName of contractNames) {
+            const contractSource = this._resolver.resolve(contractName);
+            const contractData = {
+                contractName,
+                currentArtifactIfExists: await getContractArtifactIfExistsAsync(this._artifactsDir, contractName),
+                sourceTreeHashHex: `0x${this._getSourceTreeHash(
+                    path.join(this._contractsDir, contractSource.path),
+                ).toString('hex')}`,
+            };
+            if (!this._shouldCompile(contractData)) {
+                continue;
             }
+            contractPathToData[contractSource.path] = contractData;
+            const solcVersion = _.isUndefined(this._solcVersionIfExists)
+                ? semver.maxSatisfying(_.keys(binPaths), parseSolidityVersionRange(contractSource.source))
+                : this._solcVersionIfExists;
+            const isFirstContractWithThisVersion = _.isUndefined(versionToInputs[solcVersion]);
+            if (isFirstContractWithThisVersion) {
+                versionToInputs[solcVersion] = {
+                    standardInput: {
+                        language: 'Solidity',
+                        sources: {},
+                        settings: this._compilerSettings,
+                    },
+                    contractsToCompile: [],
+                };
+            }
+            // add input to the right version batch
+            versionToInputs[solcVersion].standardInput.sources[contractSource.path] = {
+                content: contractSource.source,
+            };
+            versionToInputs[solcVersion].contractsToCompile.push(contractSource.path);
         }
-        const compiledData = compiled.contracts[contractSource.path][contractName];
-        if (_.isUndefined(compiledData)) {
-            throw new Error(
-                `Contract ${contractName} not found in ${
-                    contractSource.path
-                }. Please make sure your contract has the same name as it's file name`,
+
+        const compilerOutputs: StandardOutput[] = [];
+
+        const solcVersions = _.keys(versionToInputs);
+        for (const solcVersion of solcVersions) {
+            const input = versionToInputs[solcVersion];
+            logUtils.warn(
+                `Compiling ${input.contractsToCompile.length} contracts (${
+                    input.contractsToCompile
+                }) with Solidity v${solcVersion}...`,
             );
-        }
-        if (!_.isUndefined(compiledData.evm)) {
-            if (!_.isUndefined(compiledData.evm.bytecode) && !_.isUndefined(compiledData.evm.bytecode.object)) {
-                compiledData.evm.bytecode.object = ethUtil.addHexPrefix(compiledData.evm.bytecode.object);
-            }
-            if (
-                !_.isUndefined(compiledData.evm.deployedBytecode) &&
-                !_.isUndefined(compiledData.evm.deployedBytecode.object)
-            ) {
-                compiledData.evm.deployedBytecode.object = ethUtil.addHexPrefix(
-                    compiledData.evm.deployedBytecode.object,
-                );
+
+            const { solcInstance, fullSolcVersion } = await Compiler._getSolcAsync(solcVersion);
+
+            const compilerOutput = this._compile(solcInstance, input.standardInput);
+            compilerOutputs.push(compilerOutput);
+
+            for (const contractPath of input.contractsToCompile) {
+                const contractName = contractPathToData[contractPath].contractName;
+
+                const compiledContract = compilerOutput.contracts[contractPath][contractName];
+                if (_.isUndefined(compiledContract)) {
+                    throw new Error(
+                        `Contract ${contractName} not found in ${contractPath}. Please make sure your contract has the same name as it's file name`,
+                    );
+                }
+
+                Compiler._addHexPrefixToContractBytecode(compiledContract);
+
+                if (shouldPersist) {
+                    await this._persistCompiledContractAsync(
+                        contractPath,
+                        contractPathToData[contractPath].currentArtifactIfExists,
+                        contractPathToData[contractPath].sourceTreeHashHex,
+                        contractName,
+                        fullSolcVersion,
+                        compilerOutput,
+                    );
+                }
             }
         }
 
-        const sourceCodes = _.mapValues(
-            compiled.sources,
-            (_1, sourceFilePath) => this._resolver.resolve(sourceFilePath).source,
-        );
+        return compilerOutputs;
+    }
+    private _shouldCompile(contractData: ContractData): boolean {
+        if (_.isUndefined(contractData.currentArtifactIfExists)) {
+            return true;
+        } else {
+            const currentArtifact = contractData.currentArtifactIfExists as ContractArtifact;
+            const isUserOnLatestVersion = currentArtifact.schemaVersion === constants.LATEST_ARTIFACT_VERSION;
+            const didCompilerSettingsChange = !_.isEqual(currentArtifact.compiler.settings, this._compilerSettings);
+            const didSourceChange = currentArtifact.sourceTreeHashHex !== contractData.sourceTreeHashHex;
+            return !isUserOnLatestVersion || didCompilerSettingsChange || didSourceChange;
+        }
+    }
+    private async _persistCompiledContractAsync(
+        contractPath: string,
+        currentArtifactIfExists: ContractArtifact | void,
+        sourceTreeHashHex: string,
+        contractName: string,
+        fullSolcVersion: string,
+        compilerOutput: solc.StandardOutput,
+    ): Promise<void> {
+        const compiledContract = compilerOutput.contracts[contractPath][contractName];
+
+        // need to gather sourceCodes for this artifact, but compilerOutput.sources (the list of contract modules)
+        // contains listings for for every contract compiled during the compiler invocation that compiled the contract
+        // to be persisted, which could include many that are irrelevant to the contract at hand.  So, gather up only
+        // the relevant sources:
+        const { sourceCodes, sources } = this._getSourcesWithDependencies(contractPath, compilerOutput.sources);
+
         const contractVersion: ContractVersionData = {
-            compilerOutput: compiledData,
-            sources: compiled.sources,
+            compilerOutput: compiledContract,
+            sources,
             sourceCodes,
             sourceTreeHashHex,
             compiler: {
@@ -249,7 +334,107 @@ export class Compiler {
         const artifactString = utils.stringifyWithFormatting(newArtifact);
         const currentArtifactPath = `${this._artifactsDir}/${contractName}.json`;
         await fsWrapper.writeFileAsync(currentArtifactPath, artifactString);
-        logUtils.log(`${contractName} artifact saved!`);
+        logUtils.warn(`${contractName} artifact saved!`);
+    }
+    /**
+     * For the given @param contractPath, populates JSON objects to be used in the ContractVersionData interface's
+     * properties `sources` (source code file names mapped to ID numbers) and `sourceCodes` (source code content of
+     * contracts) for that contract.  The source code pointed to by contractPath is read and parsed directly (via
+     * `this._resolver.resolve().source`), as are its imports, recursively.  The ID numbers for @return `sources` are
+     * taken from the corresponding ID's in @param fullSources, and the content for @return sourceCodes is read from
+     * disk (via the aforementioned `resolver.source`).
+     */
+    private _getSourcesWithDependencies(
+        contractPath: string,
+        fullSources: { [sourceName: string]: { id: number } },
+    ): { sourceCodes: { [sourceName: string]: string }; sources: { [sourceName: string]: { id: number } } } {
+        const sources = { [contractPath]: { id: fullSources[contractPath].id } };
+        const sourceCodes = { [contractPath]: this._resolver.resolve(contractPath).source };
+        this._recursivelyGatherDependencySources(
+            contractPath,
+            sourceCodes[contractPath],
+            fullSources,
+            sources,
+            sourceCodes,
+        );
+        return { sourceCodes, sources };
+    }
+    private _recursivelyGatherDependencySources(
+        contractPath: string,
+        contractSource: string,
+        fullSources: { [sourceName: string]: { id: number } },
+        sourcesToAppendTo: { [sourceName: string]: { id: number } },
+        sourceCodesToAppendTo: { [sourceName: string]: string },
+    ): void {
+        const importStatementMatches = contractSource.match(/\nimport[^;]*;/g);
+        if (importStatementMatches === null) {
+            return;
+        }
+        for (const importStatementMatch of importStatementMatches) {
+            const importPathMatches = importStatementMatch.match(/\"([^\"]*)\"/);
+            if (importPathMatches === null || importPathMatches.length === 0) {
+                continue;
+            }
+
+            let importPath = importPathMatches[1];
+            // HACK(ablrow): We have, e.g.:
+            //
+            //      importPath   = "../../utils/LibBytes/LibBytes.sol"
+            //      contractPath = "2.0.0/protocol/AssetProxyOwner/AssetProxyOwner.sol"
+            //
+            // Resolver doesn't understand "../" so we want to pass
+            // "2.0.0/utils/LibBytes/LibBytes.sol" to resolver.
+            //
+            // This hack involves using path.resolve. But path.resolve returns
+            // absolute directories by default. We trick it into thinking that
+            // contractPath is a root directory by prepending a '/' and then
+            // removing the '/' the end.
+            //
+            //      path.resolve("/a/b/c", ""../../d/e") === "/a/d/e"
+            //
+            const lastPathSeparatorPos = contractPath.lastIndexOf('/');
+            const contractFolder = lastPathSeparatorPos === -1 ? '' : contractPath.slice(0, lastPathSeparatorPos + 1);
+            importPath = path.resolve('/' + contractFolder, importPath).replace('/', '');
+
+            if (_.isUndefined(sourcesToAppendTo[importPath])) {
+                sourcesToAppendTo[importPath] = { id: fullSources[importPath].id };
+                sourceCodesToAppendTo[importPath] = this._resolver.resolve(importPath).source;
+
+                this._recursivelyGatherDependencySources(
+                    importPath,
+                    this._resolver.resolve(importPath).source,
+                    fullSources,
+                    sourcesToAppendTo,
+                    sourceCodesToAppendTo,
+                );
+            }
+        }
+    }
+    private _compile(solcInstance: solc.SolcInstance, standardInput: solc.StandardInput): solc.StandardOutput {
+        const compiled: solc.StandardOutput = JSON.parse(
+            solcInstance.compileStandardWrapper(JSON.stringify(standardInput), importPath => {
+                const sourceCodeIfExists = this._resolver.resolve(importPath);
+                return { contents: sourceCodeIfExists.source };
+            }),
+        );
+        if (!_.isUndefined(compiled.errors)) {
+            const SOLIDITY_WARNING = 'warning';
+            const errors = _.filter(compiled.errors, entry => entry.severity !== SOLIDITY_WARNING);
+            const warnings = _.filter(compiled.errors, entry => entry.severity === SOLIDITY_WARNING);
+            if (!_.isEmpty(errors)) {
+                errors.forEach(error => {
+                    const normalizedErrMsg = getNormalizedErrMsg(error.formattedMessage || error.message);
+                    logUtils.warn(chalk.red(normalizedErrMsg));
+                });
+                throw new Error('Compilation errors encountered');
+            } else {
+                warnings.forEach(warning => {
+                    const normalizedWarningMsg = getNormalizedErrMsg(warning.formattedMessage || warning.message);
+                    logUtils.warn(chalk.yellow(normalizedWarningMsg));
+                });
+            }
+        }
+        return compiled;
     }
     /**
      * Gets the source tree hash for a file and its dependencies.