From 7fb66bf71a0a86c693a0411c6e03d81982b9054e Mon Sep 17 00:00:00 2001 From: Amir Bandeali Date: Sat, 17 Feb 2018 15:07:05 -0700 Subject: Check dependencies when determining if should be recompiled --- packages/deployer/src/cli.ts | 5 +- packages/deployer/src/compiler.ts | 206 +++++++++++++++++++++---------- packages/deployer/src/deployer.ts | 8 +- packages/deployer/src/utils/constants.ts | 1 + packages/deployer/src/utils/types.ts | 15 ++- packages/deployer/test/deploy_test.ts | 8 +- 6 files changed, 163 insertions(+), 80 deletions(-) (limited to 'packages/deployer') diff --git a/packages/deployer/src/cli.ts b/packages/deployer/src/cli.ts index 3c6d042c0..b093dd71b 100644 --- a/packages/deployer/src/cli.ts +++ b/packages/deployer/src/cli.ts @@ -6,6 +6,7 @@ import * as Web3 from 'web3'; import * as yargs from 'yargs'; import { commands } from './commands'; +import { constants } from './utils/constants'; import { CliOptions, CompilerOptions, DeployerOptions } from './utils/types'; const DEFAULT_OPTIMIZER_ENABLED = false; @@ -15,7 +16,6 @@ const DEFAULT_NETWORK_ID = 50; const DEFAULT_JSONRPC_PORT = 8545; const DEFAULT_GAS_PRICE = (10 ** 9 * 2).toString(); const DEFAULT_CONTRACTS_LIST = '*'; - /** * Compiles all contracts with options passed in through CLI. * @param argv Instance of process.argv provided by yargs. @@ -101,7 +101,8 @@ function getContractsSetFromList(contracts: string): Set { const specifiedContracts = new Set(); const contractsArray = contracts.split(','); _.forEach(contractsArray, contractName => { - specifiedContracts.add(contractName); + const fileName = `${contractName}${constants.SOLIDITY_FILE_EXTENSION}`; + specifiedContracts.add(fileName); }); return specifiedContracts; } diff --git a/packages/deployer/src/compiler.ts b/packages/deployer/src/compiler.ts index 2b0b81c44..6eca4b57f 100644 --- a/packages/deployer/src/compiler.ts +++ b/packages/deployer/src/compiler.ts @@ -5,19 +5,25 @@ import solc = require('solc'); import * as Web3 from 'web3'; import { binPaths } from './solc/bin_paths'; +import { constants } from './utils/constants'; import { fsWrapper } from './utils/fs_wrapper'; import { CompilerOptions, ContractArtifact, - ContractData, + ContractNetworkData, ContractNetworks, + ContractSourceData, ContractSources, + ContractSpecificSourceData, ImportContents, } from './utils/types'; import { utils } from './utils/utils'; -const SOLIDITY_FILE_EXTENSION = '.sol'; const ALL_CONTRACTS_IDENTIFIER = '*'; +const SOLIDITY_VERSION_REGEX = /(?:solidity\s\^?)([0-9]{1,2}[.][0-9]{1,2}[.][0-9]{1,2})/; +const SOLIDITY_FILE_EXTENSION_REGEX = /(.*\.sol)/; +const IMPORT_REGEX = /(import\s)/; +const DEPENDENCY_PATH_REGEX = /"([^"]+)"/; export class Compiler { private _contractsDir: string; @@ -25,12 +31,13 @@ export class Compiler { private _optimizerEnabled: number; private _artifactsDir: string; private _contractSourcesIfExists?: ContractSources; - private _solcErrors: Set; - private _specifiedContracts: Set; + private _solcErrors: Set = new Set(); + private _specifiedContracts: Set = new Set(); + private _contractSourceData: ContractSourceData = {}; /** * Recursively retrieves Solidity source code from directory. * @param dirPath Directory to search. - * @return Mapping of contract name to contract source. + * @return Mapping of contract fileName to contract source. */ private static async _getContractSourcesAsync(dirPath: string): Promise { let dirContents: string[] = []; @@ -40,15 +47,16 @@ export class Compiler { throw new Error(`No directory found at ${dirPath}`); } let sources: ContractSources = {}; - for (const name of dirContents) { - const contentPath = `${dirPath}/${name}`; - if (path.extname(name) === SOLIDITY_FILE_EXTENSION) { + for (const fileName of dirContents) { + const contentPath = `${dirPath}/${fileName}`; + if (path.extname(fileName) === constants.SOLIDITY_FILE_EXTENSION) { try { const opts = { encoding: 'utf8', }; - sources[name] = await fsWrapper.readFileAsync(contentPath, opts); - utils.consoleLog(`Reading ${name} source...`); + const source = await fsWrapper.readFileAsync(contentPath, opts); + sources[fileName] = source; + utils.consoleLog(`Reading ${fileName} source...`); } catch (err) { utils.consoleLog(`Could not find file at ${contentPath}`); } @@ -60,19 +68,58 @@ export class Compiler { ...nestedSources, }; } catch (err) { - utils.consoleLog(`${contentPath} is not a directory or ${SOLIDITY_FILE_EXTENSION} file`); + utils.consoleLog(`${contentPath} is not a directory or ${constants.SOLIDITY_FILE_EXTENSION} file`); } } } return sources; } + /** + * Gets contract dependendencies and keccak256 hash from source. + * @param source Source code of contract. + * @return Object with contract dependencies and keccak256 hash of source. + */ + private static _getContractSpecificSourceData(source: string): ContractSpecificSourceData { + const dependencies: string[] = []; + const sourceHash = `0x${ethUtil.sha3(source).toString('hex')}`; + const solc_version = Compiler._parseSolidityVersion(source); + const contractSpecificSourceData: ContractSpecificSourceData = { + dependencies, + solc_version, + keccak256: sourceHash, + }; + const lines = source.split('\n'); + _.forEach(lines, line => { + if (!_.isNull(line.match(IMPORT_REGEX))) { + const dependencyMatch = line.match(DEPENDENCY_PATH_REGEX); + if (!_.isNull(dependencyMatch)) { + const dependencyPath = dependencyMatch[1]; + const fileName = path.basename(dependencyPath); + contractSpecificSourceData.dependencies.push(fileName); + } + } + }); + return contractSpecificSourceData; + } + /** + * Finds dependencies, keccak256 hashes, and compile flag for each contract. + * @param sources Mapping of contract file name to source code. + * @return Dependencies, keccak256 hash, and compile flag for each contract. + */ + private static _getContractSourceData(sources: ContractSources): ContractSourceData { + const contractSourceData: ContractSourceData = {}; + _.forIn(sources, (source, fileName) => { + contractSourceData[fileName] = Compiler._getContractSpecificSourceData(source); + }); + return contractSourceData; + } /** * Searches Solidity source code for compiler version. * @param source Source code of contract. * @return Solc compiler version. */ private static _parseSolidityVersion(source: string): string { - const solcVersionMatch = source.match(/(?:solidity\s\^?)([0-9]{1,2}[.][0-9]{1,2}[.][0-9]{1,2})/); + const solcVersionMatch = source.match(SOLIDITY_VERSION_REGEX); if (_.isNull(solcVersionMatch)) { throw new Error('Could not find Solidity version in source'); } @@ -88,7 +135,7 @@ export class Compiler { * @return The error message with directories truncated from the contract path. */ private static _getNormalizedErrMsg(errMsg: string): string { - const errPathMatch = errMsg.match(/(.*\.sol)/); + const errPathMatch = errMsg.match(SOLIDITY_FILE_EXTENSION_REGEX); if (_.isNull(errPathMatch)) { throw new Error('Could not find a path in error message'); } @@ -107,7 +154,6 @@ export class Compiler { this._networkId = opts.networkId; this._optimizerEnabled = opts.optimizerEnabled; this._artifactsDir = opts.artifactsDir; - this._solcErrors = new Set(); this._specifiedContracts = opts.specifiedContracts; } /** @@ -116,11 +162,12 @@ export class Compiler { public async compileAllAsync(): Promise { await this._createArtifactsDirIfDoesNotExistAsync(); this._contractSourcesIfExists = await Compiler._getContractSourcesAsync(this._contractsDir); - const contractBaseNames = _.keys(this._contractSourcesIfExists); - const compiledContractPromises = _.map(contractBaseNames, async (contractBaseName: string): Promise => { - return this._compileContractAsync(contractBaseName); - }); - await Promise.all(compiledContractPromises); + this._contractSourceData = Compiler._getContractSourceData(this._contractSourcesIfExists); + const fileNames = this._specifiedContracts.has(ALL_CONTRACTS_IDENTIFIER) + ? _.keys(this._contractSourcesIfExists) + : Array.from(this._specifiedContracts.values()); + await Promise.all(_.map(fileNames, async fileName => this._setCompileActionAsync(fileName))); + await Promise.all(_.map(fileNames, async fileName => this._compileContractAsync(fileName))); this._solcErrors.forEach(errMsg => { utils.consoleLog(errMsg); @@ -128,55 +175,27 @@ export class Compiler { } /** * Compiles contract and saves artifact to artifactsDir. - * @param contractBaseName Name of contract with '.sol' extension. + * @param fileName Name of contract with '.sol' extension. */ - private async _compileContractAsync(contractBaseName: string): Promise { + private async _compileContractAsync(fileName: string): Promise { if (_.isUndefined(this._contractSourcesIfExists)) { throw new Error('Contract sources not yet initialized'); } - - const source = this._contractSourcesIfExists[contractBaseName]; - const contractName = path.basename(contractBaseName, SOLIDITY_FILE_EXTENSION); - const currentArtifactPath = `${this._artifactsDir}/${contractName}.json`; - const sourceHash = `0x${ethUtil.sha3(source).toString('hex')}`; - const isContractSpecified = - this._specifiedContracts.has(ALL_CONTRACTS_IDENTIFIER) || this._specifiedContracts.has(contractName); - - let currentArtifactString: string; - let currentArtifact: ContractArtifact; - let oldNetworks: ContractNetworks; - let shouldCompile: boolean; - try { - const opts = { - encoding: 'utf8', - }; - currentArtifactString = await fsWrapper.readFileAsync(currentArtifactPath, opts); - currentArtifact = JSON.parse(currentArtifactString); - oldNetworks = currentArtifact.networks; - const oldNetwork: ContractData = oldNetworks[this._networkId]; - shouldCompile = - (_.isUndefined(oldNetwork) || - oldNetwork.keccak256 !== sourceHash || - oldNetwork.optimizer_enabled !== this._optimizerEnabled) && - isContractSpecified; - } catch (err) { - shouldCompile = isContractSpecified; - } - - if (!shouldCompile) { + const contractSpecificSourceData = this._contractSourceData[fileName]; + if (!contractSpecificSourceData.shouldCompile) { return; } - + const source = this._contractSourcesIfExists[fileName]; const input = { - [contractBaseName]: source, + [fileName]: source, }; - const solcVersion = Compiler._parseSolidityVersion(source); - const fullSolcVersion = binPaths[solcVersion]; + + const fullSolcVersion = binPaths[contractSpecificSourceData.solc_version]; const solcBinPath = `./solc/solc_bin/${fullSolcVersion}`; const solcBin = require(solcBinPath); const solcInstance = solc.setupMethods(solcBin); - utils.consoleLog(`Compiling ${contractBaseName}...`); + utils.consoleLog(`Compiling ${fileName}...`); const sourcesToCompile = { sources: input, }; @@ -187,19 +206,20 @@ export class Compiler { ); if (!_.isUndefined(compiled.errors)) { - _.each(compiled.errors, errMsg => { + _.forEach(compiled.errors, errMsg => { const normalizedErrMsg = Compiler._getNormalizedErrMsg(errMsg); this._solcErrors.add(normalizedErrMsg); }); } - const contractIdentifier = `${contractBaseName}:${contractName}`; + const contractName = path.basename(fileName, constants.SOLIDITY_FILE_EXTENSION); + const contractIdentifier = `${fileName}:${contractName}`; const abi: Web3.ContractAbi = JSON.parse(compiled.contracts[contractIdentifier].interface); const unlinked_binary = `0x${compiled.contracts[contractIdentifier].bytecode}`; const updated_at = Date.now(); - const contractData: ContractData = { - solc_version: solcVersion, - keccak256: sourceHash, + const contractNetworkData: ContractNetworkData = { + solc_version: contractSpecificSourceData.solc_version, + keccak256: contractSpecificSourceData.keccak256, optimizer_enabled: this._optimizerEnabled, abi, unlinked_binary, @@ -207,26 +227,55 @@ export class Compiler { }; let newArtifact: ContractArtifact; - if (!_.isUndefined(currentArtifactString)) { + const currentArtifact = (await this._getContractArtifactOrReturnAsync(fileName)) as ContractArtifact; + if (!_.isUndefined(currentArtifact)) { newArtifact = { ...currentArtifact, networks: { - ...oldNetworks, - [this._networkId]: contractData, + ...currentArtifact.networks, + [this._networkId]: contractNetworkData, }, }; } else { newArtifact = { contract_name: contractName, networks: { - [this._networkId]: contractData, + [this._networkId]: contractNetworkData, }, }; } const artifactString = utils.stringifyWithFormatting(newArtifact); + const currentArtifactPath = `${this._artifactsDir}/${contractName}.json`; await fsWrapper.writeFileAsync(currentArtifactPath, artifactString); - utils.consoleLog(`${contractBaseName} artifact saved!`); + utils.consoleLog(`${fileName} artifact saved!`); + } + /** + * Recursively sets the compile action for a specific contract and dependencies. + * @param fileName Name of contracts file. + */ + private async _setCompileActionAsync(fileName: string): Promise { + const contractSpecificSourceData = this._contractSourceData[fileName]; + if (_.isUndefined(contractSpecificSourceData)) { + throw new Error(`Contract data for ${fileName} not yet set`); + } + if (_.isUndefined(contractSpecificSourceData.shouldCompile)) { + const contractArtifact = (await this._getContractArtifactOrReturnAsync(fileName)) as ContractArtifact; + if (_.isUndefined(contractArtifact)) { + contractSpecificSourceData.shouldCompile = true; + } else { + const contractNetworkData = contractArtifact.networks[this._networkId]; + contractSpecificSourceData.shouldCompile = + contractNetworkData.keccak256 !== contractSpecificSourceData.keccak256 || + this._optimizerEnabled !== contractNetworkData.optimizer_enabled || + contractNetworkData.solc_version !== contractSpecificSourceData.solc_version; + } + } + _.forEach(contractSpecificSourceData.dependencies, async dependency => { + await this._setCompileActionAsync(dependency); + contractSpecificSourceData.shouldCompile = + contractSpecificSourceData.shouldCompile || this._contractSourceData[dependency].shouldCompile; + }); } /** * Callback to resolve dependencies with `solc.compile`. @@ -238,8 +287,8 @@ export class Compiler { if (_.isUndefined(this._contractSourcesIfExists)) { throw new Error('Contract sources not yet initialized'); } - const contractBaseName = path.basename(importPath); - const source = this._contractSourcesIfExists[contractBaseName]; + const fileName = path.basename(importPath); + const source = this._contractSourcesIfExists[fileName]; const importContents: ImportContents = { contents: source, }; @@ -254,4 +303,25 @@ export class Compiler { await fsWrapper.mkdirAsync(this._artifactsDir); } } + /** + * Gets contract data on network or returns if an artifact does not exist. + * @param fileName Name of contracts file. + * @return Contract data on network or undefined. + */ + private async _getContractArtifactOrReturnAsync(fileName: string): Promise { + let contractArtifact; + const contractName = path.basename(fileName, constants.SOLIDITY_FILE_EXTENSION); + const currentArtifactPath = `${this._artifactsDir}/${contractName}.json`; + try { + const opts = { + encoding: 'utf8', + }; + const contractArtifactString = await fsWrapper.readFileAsync(currentArtifactPath, opts); + contractArtifact = JSON.parse(contractArtifactString); + return contractArtifact; + } catch (err) { + utils.consoleLog(`Artifact for ${fileName} does not exist`); + return contractArtifact; + } + } } diff --git a/packages/deployer/src/deployer.ts b/packages/deployer/src/deployer.ts index 6f03581e8..b14401050 100644 --- a/packages/deployer/src/deployer.ts +++ b/packages/deployer/src/deployer.ts @@ -6,7 +6,7 @@ import * as Web3 from 'web3'; import { Contract } from './utils/contract'; import { encoder } from './utils/encoder'; import { fsWrapper } from './utils/fs_wrapper'; -import { ContractArtifact, ContractData, DeployerOptions } from './utils/types'; +import { ContractArtifact, ContractNetworkData, DeployerOptions } from './utils/types'; import { utils } from './utils/utils'; // Gas added to gas estimate to make sure there is sufficient gas for deployment. @@ -36,7 +36,7 @@ export class Deployer { */ public async deployAsync(contractName: string, args: any[] = []): Promise { const contractArtifact: ContractArtifact = this._loadContractArtifactIfExists(contractName); - const contractData: ContractData = this._getContractDataFromArtifactIfExists(contractArtifact); + const contractData: ContractNetworkData = this._getContractDataFromArtifactIfExists(contractArtifact); const data = contractData.unlinked_binary; const from = await this._getFromAddressAsync(); const gas = await this._getAllowableGasEstimateAsync(data); @@ -101,7 +101,7 @@ export class Deployer { args: any[], ): Promise { const contractArtifact: ContractArtifact = this._loadContractArtifactIfExists(contractName); - const contractData: ContractData = this._getContractDataFromArtifactIfExists(contractArtifact); + const contractData: ContractNetworkData = this._getContractDataFromArtifactIfExists(contractArtifact); const abi = contractData.abi; const encodedConstructorArgs = encoder.encodeConstructorArgsFromAbi(args, abi); const newContractData = { @@ -139,7 +139,7 @@ export class Deployer { * @param contractArtifact The contract artifact. * @return Network specific contract data. */ - private _getContractDataFromArtifactIfExists(contractArtifact: ContractArtifact): ContractData { + private _getContractDataFromArtifactIfExists(contractArtifact: ContractArtifact): ContractNetworkData { const contractData = contractArtifact.networks[this._networkId]; if (_.isUndefined(contractData)) { throw new Error(`Data not found in artifact for contract: ${contractArtifact.contract_name}`); diff --git a/packages/deployer/src/utils/constants.ts b/packages/deployer/src/utils/constants.ts index 8871a470d..57f30dec8 100644 --- a/packages/deployer/src/utils/constants.ts +++ b/packages/deployer/src/utils/constants.ts @@ -1,3 +1,4 @@ export const constants = { NULL_BYTES: '0x', + SOLIDITY_FILE_EXTENSION: '.sol', }; diff --git a/packages/deployer/src/utils/types.ts b/packages/deployer/src/utils/types.ts index 46481828e..0a70c4f3b 100644 --- a/packages/deployer/src/utils/types.ts +++ b/packages/deployer/src/utils/types.ts @@ -15,10 +15,10 @@ export interface ContractArtifact { } export interface ContractNetworks { - [key: number]: ContractData; + [key: number]: ContractNetworkData; } -export interface ContractData { +export interface ContractNetworkData { solc_version: string; optimizer_enabled: number; keccak256: string; @@ -64,6 +64,17 @@ export interface ContractSources { [key: string]: string; } +export interface ContractSourceData { + [key: string]: ContractSpecificSourceData; +} + +export interface ContractSpecificSourceData { + dependencies: string[]; + solc_version: string; + keccak256: string; + shouldCompile?: boolean; +} + export interface ImportContents { contents: string; } diff --git a/packages/deployer/test/deploy_test.ts b/packages/deployer/test/deploy_test.ts index 6a8397982..422d3763e 100644 --- a/packages/deployer/test/deploy_test.ts +++ b/packages/deployer/test/deploy_test.ts @@ -4,7 +4,7 @@ import 'mocha'; import { Compiler } from '../src/compiler'; import { Deployer } from '../src/deployer'; import { fsWrapper } from '../src/utils/fs_wrapper'; -import { CompilerOptions, ContractArtifact, ContractData, DoneCallback } from '../src/utils/types'; +import { CompilerOptions, ContractArtifact, ContractNetworkData, DoneCallback } from '../src/utils/types'; import { constructor_args, exchange_binary } from './fixtures/exchange_bin'; import { constants } from './util/constants'; @@ -51,7 +51,7 @@ describe('#Compiler', () => { }; const exchangeArtifactString = await fsWrapper.readFileAsync(exchangeArtifactPath, opts); const exchangeArtifact: ContractArtifact = JSON.parse(exchangeArtifactString); - const exchangeContractData: ContractData = exchangeArtifact.networks[constants.networkId]; + const exchangeContractData: ContractNetworkData = exchangeArtifact.networks[constants.networkId]; // The last 43 bytes of the binaries are metadata which may not be equivalent const unlinkedBinaryWithoutMetadata = exchangeContractData.unlinked_binary.slice(0, -86); const exchangeBinaryWithoutMetadata = exchange_binary.slice(0, -86); @@ -68,7 +68,7 @@ describe('#Deployer', () => { }; const exchangeArtifactString = await fsWrapper.readFileAsync(exchangeArtifactPath, opts); const exchangeArtifact: ContractArtifact = JSON.parse(exchangeArtifactString); - const exchangeContractData: ContractData = exchangeArtifact.networks[constants.networkId]; + const exchangeContractData: ContractNetworkData = exchangeArtifact.networks[constants.networkId]; const exchangeAddress = exchangeContractInstance.address; expect(exchangeAddress).to.not.equal(undefined); expect(exchangeContractData.address).to.equal(undefined); @@ -84,7 +84,7 @@ describe('#Deployer', () => { }; const exchangeArtifactString = await fsWrapper.readFileAsync(exchangeArtifactPath, opts); const exchangeArtifact: ContractArtifact = JSON.parse(exchangeArtifactString); - const exchangeContractData: ContractData = exchangeArtifact.networks[constants.networkId]; + const exchangeContractData: ContractNetworkData = exchangeArtifact.networks[constants.networkId]; const exchangeAddress = exchangeContractInstance.address; expect(exchangeAddress).to.be.equal(exchangeContractData.address); expect(constructor_args).to.be.equal(exchangeContractData.constructor_args); -- cgit v1.2.3 From d770e462082e9bd62c1bb1fda5be1b0799026081 Mon Sep 17 00:00:00 2001 From: Amir Bandeali Date: Sat, 17 Feb 2018 16:38:47 -0700 Subject: Update CHANGELOG --- packages/deployer/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'packages/deployer') diff --git a/packages/deployer/CHANGELOG.md b/packages/deployer/CHANGELOG.md index 1d28cbd41..b184f41b9 100644 --- a/packages/deployer/CHANGELOG.md +++ b/packages/deployer/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## v0.2.0 - _??_ + + * Check dependencies when determining if contracts should be recompiled. + ## v0.1.0 - _February 16, 2018_ * Add the ability to pass in specific contracts to compile in CLI (#400) -- cgit v1.2.3 From 6685cb3fba85d5052c67b19ad3fd33c4ccfe5b67 Mon Sep 17 00:00:00 2001 From: Amir Bandeali Date: Sat, 17 Feb 2018 18:51:19 -0700 Subject: Fix race condition --- packages/deployer/CHANGELOG.md | 2 +- packages/deployer/src/compiler.ts | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) (limited to 'packages/deployer') diff --git a/packages/deployer/CHANGELOG.md b/packages/deployer/CHANGELOG.md index b184f41b9..f1e9d38ee 100644 --- a/packages/deployer/CHANGELOG.md +++ b/packages/deployer/CHANGELOG.md @@ -3,7 +3,7 @@ ## v0.2.0 - _??_ * Check dependencies when determining if contracts should be recompiled. - + ## v0.1.0 - _February 16, 2018_ * Add the ability to pass in specific contracts to compile in CLI (#400) diff --git a/packages/deployer/src/compiler.ts b/packages/deployer/src/compiler.ts index 6eca4b57f..5004d4bb4 100644 --- a/packages/deployer/src/compiler.ts +++ b/packages/deployer/src/compiler.ts @@ -168,7 +168,6 @@ export class Compiler { : Array.from(this._specifiedContracts.values()); await Promise.all(_.map(fileNames, async fileName => this._setCompileActionAsync(fileName))); await Promise.all(_.map(fileNames, async fileName => this._compileContractAsync(fileName))); - this._solcErrors.forEach(errMsg => { utils.consoleLog(errMsg); }); @@ -185,10 +184,6 @@ export class Compiler { if (!contractSpecificSourceData.shouldCompile) { return; } - const source = this._contractSourcesIfExists[fileName]; - const input = { - [fileName]: source, - }; const fullSolcVersion = binPaths[contractSpecificSourceData.solc_version]; const solcBinPath = `./solc/solc_bin/${fullSolcVersion}`; @@ -196,6 +191,10 @@ export class Compiler { const solcInstance = solc.setupMethods(solcBin); utils.consoleLog(`Compiling ${fileName}...`); + const source = this._contractSourcesIfExists[fileName]; + const input = { + [fileName]: source, + }; const sourcesToCompile = { sources: input, }; @@ -271,10 +270,14 @@ export class Compiler { contractNetworkData.solc_version !== contractSpecificSourceData.solc_version; } } - _.forEach(contractSpecificSourceData.dependencies, async dependency => { - await this._setCompileActionAsync(dependency); + await Promise.all( + _.map(contractSpecificSourceData.dependencies, async dependency => this._setCompileActionAsync(dependency)), + ); + _.forEach(contractSpecificSourceData.dependencies, dependency => { contractSpecificSourceData.shouldCompile = - contractSpecificSourceData.shouldCompile || this._contractSourceData[dependency].shouldCompile; + contractSpecificSourceData.shouldCompile || + (this._contractSourceData[dependency].shouldCompile && + (this._specifiedContracts.has('*') || this._specifiedContracts.has(dependency))); }); } /** -- cgit v1.2.3 From c1bbcaba73b1798d5e336492acc00cfa300fc05f Mon Sep 17 00:00:00 2001 From: Amir Bandeali Date: Mon, 19 Feb 2018 19:15:57 -0800 Subject: Use source tree hash instead of compile flag --- packages/deployer/src/compiler.ts | 80 ++++++++++++++++++------------------ packages/deployer/src/utils/types.ts | 5 ++- 2 files changed, 42 insertions(+), 43 deletions(-) (limited to 'packages/deployer') diff --git a/packages/deployer/src/compiler.ts b/packages/deployer/src/compiler.ts index 5004d4bb4..783bc0ea3 100644 --- a/packages/deployer/src/compiler.ts +++ b/packages/deployer/src/compiler.ts @@ -81,12 +81,12 @@ export class Compiler { */ private static _getContractSpecificSourceData(source: string): ContractSpecificSourceData { const dependencies: string[] = []; - const sourceHash = `0x${ethUtil.sha3(source).toString('hex')}`; + const sourceHash = ethUtil.sha3(source); const solc_version = Compiler._parseSolidityVersion(source); const contractSpecificSourceData: ContractSpecificSourceData = { dependencies, solc_version, - keccak256: sourceHash, + sourceHash, }; const lines = source.split('\n'); _.forEach(lines, line => { @@ -101,18 +101,6 @@ export class Compiler { }); return contractSpecificSourceData; } - /** - * Finds dependencies, keccak256 hashes, and compile flag for each contract. - * @param sources Mapping of contract file name to source code. - * @return Dependencies, keccak256 hash, and compile flag for each contract. - */ - private static _getContractSourceData(sources: ContractSources): ContractSourceData { - const contractSourceData: ContractSourceData = {}; - _.forIn(sources, (source, fileName) => { - contractSourceData[fileName] = Compiler._getContractSpecificSourceData(source); - }); - return contractSourceData; - } /** * Searches Solidity source code for compiler version. * @param source Source code of contract. @@ -162,11 +150,15 @@ export class Compiler { public async compileAllAsync(): Promise { await this._createArtifactsDirIfDoesNotExistAsync(); this._contractSourcesIfExists = await Compiler._getContractSourcesAsync(this._contractsDir); - this._contractSourceData = Compiler._getContractSourceData(this._contractSourcesIfExists); + _.forIn(this._contractSourcesIfExists, (source, fileName) => { + this._contractSourceData[fileName] = Compiler._getContractSpecificSourceData(source); + }); const fileNames = this._specifiedContracts.has(ALL_CONTRACTS_IDENTIFIER) ? _.keys(this._contractSourcesIfExists) : Array.from(this._specifiedContracts.values()); - await Promise.all(_.map(fileNames, async fileName => this._setCompileActionAsync(fileName))); + _.forEach(fileNames, fileName => { + this._setSourceTreeHash(fileName); + }); await Promise.all(_.map(fileNames, async fileName => this._compileContractAsync(fileName))); this._solcErrors.forEach(errMsg => { utils.consoleLog(errMsg); @@ -181,7 +173,15 @@ export class Compiler { throw new Error('Contract sources not yet initialized'); } const contractSpecificSourceData = this._contractSourceData[fileName]; - if (!contractSpecificSourceData.shouldCompile) { + const currentArtifact = (await this._getContractArtifactOrReturnAsync(fileName)) as ContractArtifact; + const sourceHash = `0x${contractSpecificSourceData.sourceHash.toString('hex')}`; + const sourceTreeHash = `0x${contractSpecificSourceData.sourceTreeHash.toString('hex')}`; + + const shouldCompile = + _.isUndefined(currentArtifact) || + currentArtifact.networks[this._networkId].optimizer_enabled !== this._optimizerEnabled || + currentArtifact.networks[this._networkId].source_tree_hash !== sourceTreeHash; + if (!shouldCompile) { return; } @@ -218,7 +218,8 @@ export class Compiler { const updated_at = Date.now(); const contractNetworkData: ContractNetworkData = { solc_version: contractSpecificSourceData.solc_version, - keccak256: contractSpecificSourceData.keccak256, + keccak256: sourceHash, + source_tree_hash: sourceTreeHash, optimizer_enabled: this._optimizerEnabled, abi, unlinked_binary, @@ -226,7 +227,6 @@ export class Compiler { }; let newArtifact: ContractArtifact; - const currentArtifact = (await this._getContractArtifactOrReturnAsync(fileName)) as ContractArtifact; if (!_.isUndefined(currentArtifact)) { newArtifact = { ...currentArtifact, @@ -250,35 +250,33 @@ export class Compiler { utils.consoleLog(`${fileName} artifact saved!`); } /** - * Recursively sets the compile action for a specific contract and dependencies. - * @param fileName Name of contracts file. + * Sets the source tree hash for a file and its dependencies. + * @param fileName Name of contract file. */ - private async _setCompileActionAsync(fileName: string): Promise { + private _setSourceTreeHash(fileName: string) { const contractSpecificSourceData = this._contractSourceData[fileName]; if (_.isUndefined(contractSpecificSourceData)) { throw new Error(`Contract data for ${fileName} not yet set`); } - if (_.isUndefined(contractSpecificSourceData.shouldCompile)) { - const contractArtifact = (await this._getContractArtifactOrReturnAsync(fileName)) as ContractArtifact; - if (_.isUndefined(contractArtifact)) { - contractSpecificSourceData.shouldCompile = true; + if (_.isUndefined(contractSpecificSourceData.sourceTreeHash)) { + const dependencies = contractSpecificSourceData.dependencies; + if (dependencies.length === 0) { + contractSpecificSourceData.sourceTreeHash = contractSpecificSourceData.sourceHash; } else { - const contractNetworkData = contractArtifact.networks[this._networkId]; - contractSpecificSourceData.shouldCompile = - contractNetworkData.keccak256 !== contractSpecificSourceData.keccak256 || - this._optimizerEnabled !== contractNetworkData.optimizer_enabled || - contractNetworkData.solc_version !== contractSpecificSourceData.solc_version; + _.forEach(dependencies, dependency => { + this._setSourceTreeHash(dependency); + }); + const dependencySourceTreeHashes = _.map( + dependencies, + dependency => this._contractSourceData[dependency].sourceTreeHash, + ); + const sourceTreeHashesBuffer = Buffer.concat([ + contractSpecificSourceData.sourceHash, + ...dependencySourceTreeHashes, + ]); + contractSpecificSourceData.sourceTreeHash = ethUtil.sha3(sourceTreeHashesBuffer); } } - await Promise.all( - _.map(contractSpecificSourceData.dependencies, async dependency => this._setCompileActionAsync(dependency)), - ); - _.forEach(contractSpecificSourceData.dependencies, dependency => { - contractSpecificSourceData.shouldCompile = - contractSpecificSourceData.shouldCompile || - (this._contractSourceData[dependency].shouldCompile && - (this._specifiedContracts.has('*') || this._specifiedContracts.has(dependency))); - }); } /** * Callback to resolve dependencies with `solc.compile`. @@ -308,7 +306,7 @@ export class Compiler { } /** * Gets contract data on network or returns if an artifact does not exist. - * @param fileName Name of contracts file. + * @param fileName Name of contract file. * @return Contract data on network or undefined. */ private async _getContractArtifactOrReturnAsync(fileName: string): Promise { diff --git a/packages/deployer/src/utils/types.ts b/packages/deployer/src/utils/types.ts index 0a70c4f3b..166fc15dd 100644 --- a/packages/deployer/src/utils/types.ts +++ b/packages/deployer/src/utils/types.ts @@ -22,6 +22,7 @@ export interface ContractNetworkData { solc_version: string; optimizer_enabled: number; keccak256: string; + source_tree_hash: string; abi: Web3.ContractAbi; unlinked_binary: string; address?: string; @@ -71,8 +72,8 @@ export interface ContractSourceData { export interface ContractSpecificSourceData { dependencies: string[]; solc_version: string; - keccak256: string; - shouldCompile?: boolean; + sourceHash: Buffer; + sourceTreeHash?: Buffer; } export interface ImportContents { -- cgit v1.2.3 From 67f28645018244a0aeda6404b7fd4ea33c67110f Mon Sep 17 00:00:00 2001 From: Amir Bandeali Date: Tue, 20 Feb 2018 13:42:35 -0800 Subject: Address feedback --- packages/deployer/CHANGELOG.md | 4 +-- packages/deployer/src/cli.ts | 1 + packages/deployer/src/compiler.ts | 62 ++++++++++++++++++------------------ packages/deployer/src/deployer.ts | 32 +++++++++++-------- packages/deployer/src/utils/types.ts | 4 +-- 5 files changed, 54 insertions(+), 49 deletions(-) (limited to 'packages/deployer') diff --git a/packages/deployer/CHANGELOG.md b/packages/deployer/CHANGELOG.md index f1e9d38ee..a63d9cf3b 100644 --- a/packages/deployer/CHANGELOG.md +++ b/packages/deployer/CHANGELOG.md @@ -1,8 +1,8 @@ # CHANGELOG -## v0.2.0 - _??_ +## v0.2.0 - _TBD, 2018_ - * Check dependencies when determining if contracts should be recompiled. + * Check dependencies when determining if contracts should be recompiled (#408). ## v0.1.0 - _February 16, 2018_ diff --git a/packages/deployer/src/cli.ts b/packages/deployer/src/cli.ts index b093dd71b..ba156ac20 100644 --- a/packages/deployer/src/cli.ts +++ b/packages/deployer/src/cli.ts @@ -16,6 +16,7 @@ const DEFAULT_NETWORK_ID = 50; const DEFAULT_JSONRPC_PORT = 8545; const DEFAULT_GAS_PRICE = (10 ** 9 * 2).toString(); const DEFAULT_CONTRACTS_LIST = '*'; + /** * Compiles all contracts with options passed in through CLI. * @param argv Instance of process.argv provided by yargs. diff --git a/packages/deployer/src/compiler.ts b/packages/deployer/src/compiler.ts index 783bc0ea3..149ca5d6d 100644 --- a/packages/deployer/src/compiler.ts +++ b/packages/deployer/src/compiler.ts @@ -20,17 +20,17 @@ import { import { utils } from './utils/utils'; const ALL_CONTRACTS_IDENTIFIER = '*'; -const SOLIDITY_VERSION_REGEX = /(?:solidity\s\^?)([0-9]{1,2}[.][0-9]{1,2}[.][0-9]{1,2})/; +const SOLIDITY_VERSION_REGEX = /(?:solidity\s\^?)(\d+\.\d+\.\d+)/; const SOLIDITY_FILE_EXTENSION_REGEX = /(.*\.sol)/; const IMPORT_REGEX = /(import\s)/; -const DEPENDENCY_PATH_REGEX = /"([^"]+)"/; +const DEPENDENCY_PATH_REGEX = /"([^"]+)"/; // Source: https://github.com/BlockChainCompany/soljitsu/blob/master/lib/shared.js export class Compiler { private _contractsDir: string; private _networkId: number; private _optimizerEnabled: number; private _artifactsDir: string; - private _contractSourcesIfExists?: ContractSources; + private _contractSources?: ContractSources; private _solcErrors: Set = new Set(); private _specifiedContracts: Set = new Set(); private _contractSourceData: ContractSourceData = {}; @@ -82,10 +82,10 @@ export class Compiler { private static _getContractSpecificSourceData(source: string): ContractSpecificSourceData { const dependencies: string[] = []; const sourceHash = ethUtil.sha3(source); - const solc_version = Compiler._parseSolidityVersion(source); + const solcVersion = Compiler._parseSolidityVersion(source); const contractSpecificSourceData: ContractSpecificSourceData = { dependencies, - solc_version, + solcVersion, sourceHash, }; const lines = source.split('\n'); @@ -149,12 +149,12 @@ export class Compiler { */ public async compileAllAsync(): Promise { await this._createArtifactsDirIfDoesNotExistAsync(); - this._contractSourcesIfExists = await Compiler._getContractSourcesAsync(this._contractsDir); - _.forIn(this._contractSourcesIfExists, (source, fileName) => { + this._contractSources = await Compiler._getContractSourcesAsync(this._contractsDir); + _.forIn(this._contractSources, (source, fileName) => { this._contractSourceData[fileName] = Compiler._getContractSpecificSourceData(source); }); const fileNames = this._specifiedContracts.has(ALL_CONTRACTS_IDENTIFIER) - ? _.keys(this._contractSourcesIfExists) + ? _.keys(this._contractSources) : Array.from(this._specifiedContracts.values()); _.forEach(fileNames, fileName => { this._setSourceTreeHash(fileName); @@ -169,29 +169,29 @@ export class Compiler { * @param fileName Name of contract with '.sol' extension. */ private async _compileContractAsync(fileName: string): Promise { - if (_.isUndefined(this._contractSourcesIfExists)) { + if (_.isUndefined(this._contractSources)) { throw new Error('Contract sources not yet initialized'); } const contractSpecificSourceData = this._contractSourceData[fileName]; - const currentArtifact = (await this._getContractArtifactOrReturnAsync(fileName)) as ContractArtifact; + const currentArtifactIfExists = (await this._getContractArtifactIfExistsAsync(fileName)) as ContractArtifact; const sourceHash = `0x${contractSpecificSourceData.sourceHash.toString('hex')}`; - const sourceTreeHash = `0x${contractSpecificSourceData.sourceTreeHash.toString('hex')}`; + const sourceTreeHash = `0x${contractSpecificSourceData.sourceTreeHashIfExists.toString('hex')}`; const shouldCompile = - _.isUndefined(currentArtifact) || - currentArtifact.networks[this._networkId].optimizer_enabled !== this._optimizerEnabled || - currentArtifact.networks[this._networkId].source_tree_hash !== sourceTreeHash; + _.isUndefined(currentArtifactIfExists) || + currentArtifactIfExists.networks[this._networkId].optimizer_enabled !== this._optimizerEnabled || + currentArtifactIfExists.networks[this._networkId].source_tree_hash !== sourceTreeHash; if (!shouldCompile) { return; } - const fullSolcVersion = binPaths[contractSpecificSourceData.solc_version]; + const fullSolcVersion = binPaths[contractSpecificSourceData.solcVersion]; const solcBinPath = `./solc/solc_bin/${fullSolcVersion}`; const solcBin = require(solcBinPath); const solcInstance = solc.setupMethods(solcBin); utils.consoleLog(`Compiling ${fileName}...`); - const source = this._contractSourcesIfExists[fileName]; + const source = this._contractSources[fileName]; const input = { [fileName]: source, }; @@ -217,7 +217,7 @@ export class Compiler { const unlinked_binary = `0x${compiled.contracts[contractIdentifier].bytecode}`; const updated_at = Date.now(); const contractNetworkData: ContractNetworkData = { - solc_version: contractSpecificSourceData.solc_version, + solc_version: contractSpecificSourceData.solcVersion, keccak256: sourceHash, source_tree_hash: sourceTreeHash, optimizer_enabled: this._optimizerEnabled, @@ -227,11 +227,11 @@ export class Compiler { }; let newArtifact: ContractArtifact; - if (!_.isUndefined(currentArtifact)) { + if (!_.isUndefined(currentArtifactIfExists)) { newArtifact = { - ...currentArtifact, + ...currentArtifactIfExists, networks: { - ...currentArtifact.networks, + ...currentArtifactIfExists.networks, [this._networkId]: contractNetworkData, }, }; @@ -253,28 +253,28 @@ export class Compiler { * Sets the source tree hash for a file and its dependencies. * @param fileName Name of contract file. */ - private _setSourceTreeHash(fileName: string) { + private _setSourceTreeHash(fileName: string): void { const contractSpecificSourceData = this._contractSourceData[fileName]; if (_.isUndefined(contractSpecificSourceData)) { throw new Error(`Contract data for ${fileName} not yet set`); } - if (_.isUndefined(contractSpecificSourceData.sourceTreeHash)) { + if (_.isUndefined(contractSpecificSourceData.sourceTreeHashIfExists)) { const dependencies = contractSpecificSourceData.dependencies; if (dependencies.length === 0) { - contractSpecificSourceData.sourceTreeHash = contractSpecificSourceData.sourceHash; + contractSpecificSourceData.sourceTreeHashIfExists = contractSpecificSourceData.sourceHash; } else { _.forEach(dependencies, dependency => { this._setSourceTreeHash(dependency); }); const dependencySourceTreeHashes = _.map( dependencies, - dependency => this._contractSourceData[dependency].sourceTreeHash, + dependency => this._contractSourceData[dependency].sourceTreeHashIfExists, ); const sourceTreeHashesBuffer = Buffer.concat([ contractSpecificSourceData.sourceHash, ...dependencySourceTreeHashes, ]); - contractSpecificSourceData.sourceTreeHash = ethUtil.sha3(sourceTreeHashesBuffer); + contractSpecificSourceData.sourceTreeHashIfExists = ethUtil.sha3(sourceTreeHashesBuffer); } } } @@ -285,11 +285,11 @@ export class Compiler { * @return Import contents object containing source code of dependency. */ private _findImportsIfSourcesExist(importPath: string): ImportContents { - if (_.isUndefined(this._contractSourcesIfExists)) { - throw new Error('Contract sources not yet initialized'); - } const fileName = path.basename(importPath); - const source = this._contractSourcesIfExists[fileName]; + const source = this._contractSources[fileName]; + if (_.isUndefined(source)) { + throw new Error(`Contract source not found for ${fileName}`); + } const importContents: ImportContents = { contents: source, }; @@ -309,7 +309,7 @@ export class Compiler { * @param fileName Name of contract file. * @return Contract data on network or undefined. */ - private async _getContractArtifactOrReturnAsync(fileName: string): Promise { + private async _getContractArtifactIfExistsAsync(fileName: string): Promise { let contractArtifact; const contractName = path.basename(fileName, constants.SOLIDITY_FILE_EXTENSION); const currentArtifactPath = `${this._artifactsDir}/${contractName}.json`; @@ -322,7 +322,7 @@ export class Compiler { return contractArtifact; } catch (err) { utils.consoleLog(`Artifact for ${fileName} does not exist`); - return contractArtifact; + return undefined; } } } diff --git a/packages/deployer/src/deployer.ts b/packages/deployer/src/deployer.ts index b14401050..021645fd1 100644 --- a/packages/deployer/src/deployer.ts +++ b/packages/deployer/src/deployer.ts @@ -35,9 +35,11 @@ export class Deployer { * @return Deployed contract instance. */ public async deployAsync(contractName: string, args: any[] = []): Promise { - const contractArtifact: ContractArtifact = this._loadContractArtifactIfExists(contractName); - const contractData: ContractNetworkData = this._getContractDataFromArtifactIfExists(contractArtifact); - const data = contractData.unlinked_binary; + const contractArtifactIfExists: ContractArtifact = this._loadContractArtifactIfExists(contractName); + const contractNetworkDataIfExists: ContractNetworkData = this._getContractNetworkDataFromArtifactIfExists( + contractArtifactIfExists, + ); + const data = contractNetworkDataIfExists.unlinked_binary; const from = await this._getFromAddressAsync(); const gas = await this._getAllowableGasEstimateAsync(data); const txData = { @@ -46,7 +48,7 @@ export class Deployer { data, gas, }; - const abi = contractData.abi; + const abi = contractNetworkDataIfExists.abi; const web3ContractInstance = await this._deployFromAbiAsync(abi, args, txData); utils.consoleLog(`${contractName}.sol successfully deployed at ${web3ContractInstance.address}`); const contractInstance = new Contract(web3ContractInstance, this._defaults); @@ -100,19 +102,21 @@ export class Deployer { contractAddress: string, args: any[], ): Promise { - const contractArtifact: ContractArtifact = this._loadContractArtifactIfExists(contractName); - const contractData: ContractNetworkData = this._getContractDataFromArtifactIfExists(contractArtifact); - const abi = contractData.abi; + const contractArtifactIfExists: ContractArtifact = this._loadContractArtifactIfExists(contractName); + const contractNetworkDataIfExists: ContractNetworkData = this._getContractNetworkDataFromArtifactIfExists( + contractArtifactIfExists, + ); + const abi = contractNetworkDataIfExists.abi; const encodedConstructorArgs = encoder.encodeConstructorArgsFromAbi(args, abi); const newContractData = { - ...contractData, + ...contractNetworkDataIfExists, address: contractAddress, constructor_args: encodedConstructorArgs, }; const newArtifact = { - ...contractArtifact, + ...contractArtifactIfExists, networks: { - ...contractArtifact.networks, + ...contractArtifactIfExists.networks, [this._networkId]: newContractData, }, }; @@ -139,12 +143,12 @@ export class Deployer { * @param contractArtifact The contract artifact. * @return Network specific contract data. */ - private _getContractDataFromArtifactIfExists(contractArtifact: ContractArtifact): ContractNetworkData { - const contractData = contractArtifact.networks[this._networkId]; - if (_.isUndefined(contractData)) { + private _getContractNetworkDataFromArtifactIfExists(contractArtifact: ContractArtifact): ContractNetworkData { + const contractNetworkDataIfExists = contractArtifact.networks[this._networkId]; + if (_.isUndefined(contractNetworkDataIfExists)) { throw new Error(`Data not found in artifact for contract: ${contractArtifact.contract_name}`); } - return contractData; + return contractNetworkDataIfExists; } /** * Gets the address to use for sending a transaction. diff --git a/packages/deployer/src/utils/types.ts b/packages/deployer/src/utils/types.ts index 166fc15dd..a3f722976 100644 --- a/packages/deployer/src/utils/types.ts +++ b/packages/deployer/src/utils/types.ts @@ -71,9 +71,9 @@ export interface ContractSourceData { export interface ContractSpecificSourceData { dependencies: string[]; - solc_version: string; + solcVersion: string; sourceHash: Buffer; - sourceTreeHash?: Buffer; + sourceTreeHashIfExists?: Buffer; } export interface ImportContents { -- cgit v1.2.3