From 760bab8f866ec3d5fc7627ce9bbf5c2eaaef1f36 Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Fri, 8 Jun 2018 11:18:32 -0700 Subject: Implement SolidityProfiler & adapt sol-cov to work with Geth --- packages/sol-cov/CHANGELOG.json | 41 ++++ packages/sol-cov/package.json | 2 + .../sol_compiler_artifact_adapter.ts | 8 +- packages/sol-cov/src/coverage_manager.ts | 154 +++++------- packages/sol-cov/src/coverage_subprovider.ts | 192 +-------------- packages/sol-cov/src/index.ts | 2 + packages/sol-cov/src/profiler_manager.ts | 134 +++++++++++ packages/sol-cov/src/profiler_subprovider.ts | 36 +++ packages/sol-cov/src/trace.ts | 16 +- .../sol-cov/src/trace_collection_subprovider.ts | 258 +++++++++++++++++++++ packages/sol-cov/src/types.ts | 17 +- packages/sol-cov/src/utils.ts | 30 ++- 12 files changed, 605 insertions(+), 285 deletions(-) create mode 100644 packages/sol-cov/src/profiler_manager.ts create mode 100644 packages/sol-cov/src/profiler_subprovider.ts create mode 100644 packages/sol-cov/src/trace_collection_subprovider.ts (limited to 'packages/sol-cov') diff --git a/packages/sol-cov/CHANGELOG.json b/packages/sol-cov/CHANGELOG.json index 0d3303231..b92220a14 100644 --- a/packages/sol-cov/CHANGELOG.json +++ b/packages/sol-cov/CHANGELOG.json @@ -1,4 +1,45 @@ [ + { + "version": "0.2.0", + "changes": [ + { + "note": "Fixed a bug causing RegExp to crash if contract code is longer that 32767 characters", + "pr": 675 + }, + { + "note": "Fixed a bug caused by Geth debug trace depth being 1-indexed", + "pr": 675 + }, + { + "note": "Fixed a bug when the tool crashed on empty traces", + "pr": 675 + }, + { + "note": "Use `BlockchainLifecycle` to support reverst on Geth", + "pr": 675 + }, + { + "note": "Add `ProfilerSubprovider` as a hacky way to profile code using coverage tools", + "pr": 675 + }, + { + "note": "Collect traces from `estimate_gas` calls", + "pr": 675 + }, + { + "note": "Fix a race condition caused by not awaiting the transaction before getting a trace", + "pr": 675 + }, + { + "note": "Add `start`/`stop` functionality to `CoverageSubprovider` and `ProfilerSubprovider`", + "pr": 675 + }, + { + "note": "Skip interface artifacts with the warning instead of failing", + "pr": 675 + } + ] + }, { "timestamp": 1527009134, "version": "0.1.0", diff --git a/packages/sol-cov/package.json b/packages/sol-cov/package.json index 2b4c1db40..02235915a 100644 --- a/packages/sol-cov/package.json +++ b/packages/sol-cov/package.json @@ -54,6 +54,8 @@ "@0xproject/subproviders": "^0.10.2", "@0xproject/typescript-typings": "^0.3.2", "@0xproject/utils": "^0.6.2", + "@0xproject/web3-wrapper": "^0.6.4", + "@0xproject/dev-utils": "^0.4.2", "ethereum-types": "^0.0.1", "ethereumjs-util": "^5.1.1", "glob": "^7.1.2", diff --git a/packages/sol-cov/src/artifact_adapters/sol_compiler_artifact_adapter.ts b/packages/sol-cov/src/artifact_adapters/sol_compiler_artifact_adapter.ts index 6e385203b..0be661106 100644 --- a/packages/sol-cov/src/artifact_adapters/sol_compiler_artifact_adapter.ts +++ b/packages/sol-cov/src/artifact_adapters/sol_compiler_artifact_adapter.ts @@ -1,3 +1,5 @@ +import { ContractArtifact } from '@0xproject/sol-compiler'; +import { logUtils } from '@0xproject/utils'; import * as fs from 'fs'; import * as glob from 'glob'; import * as _ from 'lodash'; @@ -29,7 +31,11 @@ export class SolCompilerArtifactAdapter extends AbstractArtifactAdapter { const artifactFileNames = glob.sync(artifactsGlob, { absolute: true }); const contractsData: ContractData[] = []; for (const artifactFileName of artifactFileNames) { - const artifact = JSON.parse(fs.readFileSync(artifactFileName).toString()); + const artifact: ContractArtifact = JSON.parse(fs.readFileSync(artifactFileName).toString()); + if (_.isUndefined(artifact.compilerOutput.evm)) { + logUtils.warn(`${artifactFileName} doesn't contain bytecode. Skipping...`); + continue; + } let sources = _.keys(artifact.sources); sources = _.map(sources, relativeFilePath => path.resolve(this._sourcesPath, relativeFilePath)); const sourceCodes = _.map(sources, (source: string) => fs.readFileSync(source).toString()); diff --git a/packages/sol-cov/src/coverage_manager.ts b/packages/sol-cov/src/coverage_manager.ts index 064338a32..3ab363b52 100644 --- a/packages/sol-cov/src/coverage_manager.ts +++ b/packages/sol-cov/src/coverage_manager.ts @@ -21,6 +21,7 @@ import { SourceRange, StatementCoverage, StatementDescription, + Subtrace, TraceInfo, TraceInfoExistingContract, TraceInfoNewContract, @@ -29,21 +30,30 @@ import { utils } from './utils'; const mkdirpAsync = promisify(mkdirp); +/** + * CoverageManager is used by CoverageSubprovider to compute code coverage based on collected trace data. + */ export class CoverageManager { private _artifactAdapter: AbstractArtifactAdapter; private _logger: Logger; private _traceInfos: TraceInfo[] = []; - // tslint:disable-next-line:no-unused-variable - private _getContractCodeAsync: (address: string) => Promise; - private static _getSingleFileCoverageForTrace( + /** + * Computed partial coverage for a single file & subtrace + * @param contractData Contract metadata (source, srcMap, bytecode) + * @param subtrace A subset of a transcation/call trace that was executed within that contract + * @param pcToSourceRange A mapping from program counters to source ranges + * @param fileIndex Index of a file to compute coverage for + * @return Partial istanbul coverage for that file & subtrace + */ + private static _getSingleFileCoverageForSubtrace( contractData: ContractData, - coveredPcs: number[], + subtrace: Subtrace, pcToSourceRange: { [programCounter: number]: SourceRange }, fileIndex: number, ): Coverage { const absoluteFileName = contractData.sources[fileIndex]; const coverageEntriesDescription = collectCoverageEntries(contractData.sourceCodes[fileIndex]); - let sourceRanges = _.map(coveredPcs, coveredPc => pcToSourceRange[coveredPc]); + let sourceRanges = _.map(subtrace, structLog => pcToSourceRange[structLog.pc]); sourceRanges = _.compact(sourceRanges); // Some PC's don't map to a source range and we just ignore them. // By default lodash does a shallow object comparasion. We JSON.stringify them and compare as strings. sourceRanges = _.uniqBy(sourceRanges, s => JSON.stringify(s)); // We don't care if one PC was covered multiple times within a single transaction @@ -52,26 +62,32 @@ export class CoverageManager { const branchIds = _.keys(coverageEntriesDescription.branchMap); for (const branchId of branchIds) { const branchDescription = coverageEntriesDescription.branchMap[branchId]; - const isCoveredByBranchIndex = _.map(branchDescription.locations, location => - _.some(sourceRanges, range => utils.isRangeInside(range.location, location)), - ); - branchCoverage[branchId] = isCoveredByBranchIndex; + const isBranchCoveredByBranchIndex = _.map(branchDescription.locations, location => { + const isBranchCovered = _.some(sourceRanges, range => utils.isRangeInside(range.location, location)); + const timesBranchCovered = Number(isBranchCovered); + return timesBranchCovered; + }); + branchCoverage[branchId] = isBranchCoveredByBranchIndex; } const statementCoverage: StatementCoverage = {}; const statementIds = _.keys(coverageEntriesDescription.statementMap); for (const statementId of statementIds) { const statementDescription = coverageEntriesDescription.statementMap[statementId]; - const isCovered = _.some(sourceRanges, range => utils.isRangeInside(range.location, statementDescription)); - statementCoverage[statementId] = isCovered; + const isStatementCovered = _.some(sourceRanges, range => + utils.isRangeInside(range.location, statementDescription), + ); + const timesStatementCovered = Number(isStatementCovered); + statementCoverage[statementId] = timesStatementCovered; } const functionCoverage: FunctionCoverage = {}; const functionIds = _.keys(coverageEntriesDescription.fnMap); for (const fnId of functionIds) { const functionDescription = coverageEntriesDescription.fnMap[fnId]; - const isCovered = _.some(sourceRanges, range => + const isFunctionCovered = _.some(sourceRanges, range => utils.isRangeInside(range.location, functionDescription.loc), ); - functionCoverage[fnId] = isCovered; + const timesFunctionCovered = Number(isFunctionCovered); + functionCoverage[fnId] = timesFunctionCovered; } // HACK: Solidity doesn't emit any opcodes that map back to modifiers with no args, that's why we map back to the // function range and check if there is any covered statement within that range. @@ -95,12 +111,12 @@ export class CoverageManager { return isInsideTheModifierEnclosingFunction && isCovered; }, ); - statementCoverage[modifierStatementId] = isModifierCovered; + const timesModifierCovered = Number(isModifierCovered); + statementCoverage[modifierStatementId] = timesModifierCovered; } const partialCoverage = { [absoluteFileName]: { ...coverageEntriesDescription, - l: {}, // It's able to derive it from statement coverage path: absoluteFileName, f: functionCoverage, s: statementCoverage, @@ -109,37 +125,7 @@ export class CoverageManager { }; return partialCoverage; } - private static _bytecodeToBytecodeRegex(bytecode: string): string { - const bytecodeRegex = bytecode - // Library linking placeholder: __ConvertLib____________________________ - .replace(/_.*_/, '.*') - // Last 86 characters is solidity compiler metadata that's different between compilations - .replace(/.{86}$/, '') - // Libraries contain their own address at the beginning of the code and it's impossible to know it in advance - .replace(/^0x730000000000000000000000000000000000000000/, '0x73........................................'); - return bytecodeRegex; - } - private static _getContractDataIfExists(contractsData: ContractData[], bytecode: string): ContractData | undefined { - if (!bytecode.startsWith('0x')) { - throw new Error(`0x hex prefix missing: ${bytecode}`); - } - const contractData = _.find(contractsData, contractDataCandidate => { - const bytecodeRegex = CoverageManager._bytecodeToBytecodeRegex(contractDataCandidate.bytecode); - const runtimeBytecodeRegex = CoverageManager._bytecodeToBytecodeRegex( - contractDataCandidate.runtimeBytecode, - ); - // We use that function to find by bytecode or runtimeBytecode. Those are quasi-random strings so - // collisions are practically impossible and it allows us to reuse that code - return !_.isNull(bytecode.match(bytecodeRegex)) || !_.isNull(bytecode.match(runtimeBytecodeRegex)); - }); - return contractData; - } - constructor( - artifactAdapter: AbstractArtifactAdapter, - getContractCodeAsync: (address: string) => Promise, - isVerbose: boolean, - ) { - this._getContractCodeAsync = getContractCodeAsync; + constructor(artifactAdapter: AbstractArtifactAdapter, isVerbose: boolean) { this._artifactAdapter = artifactAdapter; this._logger = getLogger('sol-cov'); this._logger.setLevel(isVerbose ? levels.TRACE : levels.ERROR); @@ -157,56 +143,34 @@ export class CoverageManager { const contractsData = await this._artifactAdapter.collectContractsDataAsync(); const collector = new Collector(); for (const traceInfo of this._traceInfos) { - if (traceInfo.address !== constants.NEW_CONTRACT) { - // Runtime transaction - const runtimeBytecode = (traceInfo as TraceInfoExistingContract).runtimeBytecode; - const contractData = CoverageManager._getContractDataIfExists(contractsData, runtimeBytecode); - if (_.isUndefined(contractData)) { - this._logger.warn(`Transaction to an unknown address: ${traceInfo.address}`); - continue; - } - const bytecodeHex = stripHexPrefix(runtimeBytecode); - const sourceMap = contractData.sourceMapRuntime; - const pcToSourceRange = parseSourceMap( - contractData.sourceCodes, - sourceMap, - bytecodeHex, - contractData.sources, - ); - for (let fileIndex = 0; fileIndex < contractData.sources.length; fileIndex++) { - const singleFileCoverageForTrace = CoverageManager._getSingleFileCoverageForTrace( - contractData, - traceInfo.coveredPcs, - pcToSourceRange, - fileIndex, - ); - collector.add(singleFileCoverageForTrace); - } - } else { - // Contract creation transaction - const bytecode = (traceInfo as TraceInfoNewContract).bytecode; - const contractData = CoverageManager._getContractDataIfExists(contractsData, bytecode); - if (_.isUndefined(contractData)) { - this._logger.warn(`Unknown contract creation transaction`); - continue; - } - const bytecodeHex = stripHexPrefix(bytecode); - const sourceMap = contractData.sourceMap; - const pcToSourceRange = parseSourceMap( - contractData.sourceCodes, - sourceMap, - bytecodeHex, - contractData.sources, + const isContractCreation = traceInfo.address === constants.NEW_CONTRACT; + const bytecode = isContractCreation + ? (traceInfo as TraceInfoNewContract).bytecode + : (traceInfo as TraceInfoExistingContract).runtimeBytecode; + const contractData = utils.getContractDataIfExists(contractsData, bytecode); + if (_.isUndefined(contractData)) { + const errMsg = isContractCreation + ? `Unknown contract creation transaction` + : `Transaction to an unknown address: ${traceInfo.address}`; + this._logger.warn(errMsg); + continue; + } + const bytecodeHex = stripHexPrefix(bytecode); + const sourceMap = isContractCreation ? contractData.sourceMap : contractData.sourceMapRuntime; + const pcToSourceRange = parseSourceMap( + contractData.sourceCodes, + sourceMap, + bytecodeHex, + contractData.sources, + ); + for (let fileIndex = 0; fileIndex < contractData.sources.length; fileIndex++) { + const singleFileCoverageForTrace = CoverageManager._getSingleFileCoverageForSubtrace( + contractData, + traceInfo.subtrace, + pcToSourceRange, + fileIndex, ); - for (let fileIndex = 0; fileIndex < contractData.sources.length; fileIndex++) { - const singleFileCoverageForTrace = CoverageManager._getSingleFileCoverageForTrace( - contractData, - traceInfo.coveredPcs, - pcToSourceRange, - fileIndex, - ); - collector.add(singleFileCoverageForTrace); - } + collector.add(singleFileCoverageForTrace); } } return collector.getFinalCoverage(); diff --git a/packages/sol-cov/src/coverage_subprovider.ts b/packages/sol-cov/src/coverage_subprovider.ts index 13b76f537..174b7c6ac 100644 --- a/packages/sol-cov/src/coverage_subprovider.ts +++ b/packages/sol-cov/src/coverage_subprovider.ts @@ -1,31 +1,15 @@ -import { Callback, ErrorCallback, NextCallback, Subprovider } from '@0xproject/subproviders'; -import { BlockParam, CallData, JSONRPCRequestPayload, TransactionTrace, TxData } from 'ethereum-types'; import * as _ from 'lodash'; -import { Lock } from 'semaphore-async-await'; import { AbstractArtifactAdapter } from './artifact_adapters/abstract_artifact_adapter'; -import { constants } from './constants'; import { CoverageManager } from './coverage_manager'; -import { getTracesByContractAddress } from './trace'; -import { BlockParamLiteral, TraceInfoExistingContract, TraceInfoNewContract } from './types'; - -interface MaybeFakeTxData extends TxData { - isFakeTransaction?: boolean; -} - -// Because there is no notion of a call trace in the Ethereum rpc - we collect them in a rather non-obvious/hacky way. -// On each call - we create a snapshot, execute the call as a transaction, get the trace, revert the snapshot. -// That allows us to avoid influencing test behaviour. +import { TraceCollectionSubprovider } from './trace_collection_subprovider'; /** * This class implements the [web3-provider-engine](https://github.com/MetaMask/provider-engine) subprovider interface. - * It collects traces of all transactions that were sent and all calls that were executed through JSON RPC. + * It's used to compute your code coverage while running solidity tests. */ -export class CoverageSubprovider extends Subprovider { - // Lock is used to not accept normal transactions while doing call/snapshot magic because they'll be reverted later otherwise - private _lock: Lock; +export class CoverageSubprovider extends TraceCollectionSubprovider { private _coverageManager: CoverageManager; - private _defaultFromAddress: string; /** * Instantiates a CoverageSubprovider instance * @param artifactAdapter Adapter for used artifacts format (0x, truffle, giveth, etc.) @@ -33,172 +17,20 @@ export class CoverageSubprovider extends Subprovider { * @param isVerbose If true, we will log any unknown transactions. Otherwise we will ignore them */ constructor(artifactAdapter: AbstractArtifactAdapter, defaultFromAddress: string, isVerbose: boolean = true) { - super(); - this._lock = new Lock(); - this._defaultFromAddress = defaultFromAddress; - this._coverageManager = new CoverageManager(artifactAdapter, this._getContractCodeAsync.bind(this), isVerbose); + const traceCollectionSubproviderConfig = { + shouldCollectTransactionTraces: true, + shouldCollectGasEstimateTraces: true, + shouldCollectCallTraces: true, + }; + super(defaultFromAddress, traceCollectionSubproviderConfig); + this._coverageManager = new CoverageManager(artifactAdapter, isVerbose); } /** * Write the test coverage results to a file in Istanbul format. */ public async writeCoverageAsync(): Promise { + const traceInfos = this.getCollectedTraceInfos(); + _.forEach(traceInfos, traceInfo => this._coverageManager.appendTraceInfo(traceInfo)); await this._coverageManager.writeCoverageAsync(); } - /** - * This method conforms to the web3-provider-engine interface. - * It is called internally by the ProviderEngine when it is this subproviders - * turn to handle a JSON RPC request. - * @param payload JSON RPC payload - * @param next Callback to call if this subprovider decides not to handle the request - * @param end Callback to call if subprovider handled the request and wants to pass back the request. - */ - // tslint:disable-next-line:prefer-function-over-method async-suffix - public async handleRequest(payload: JSONRPCRequestPayload, next: NextCallback, end: ErrorCallback): Promise { - switch (payload.method) { - case 'eth_sendTransaction': - const txData = payload.params[0]; - next(this._onTransactionSentAsync.bind(this, txData)); - return; - - case 'eth_call': - const callData = payload.params[0]; - const blockNumber = payload.params[1]; - next(this._onCallExecutedAsync.bind(this, callData, blockNumber)); - return; - - default: - next(); - return; - } - } - private async _onTransactionSentAsync( - txData: MaybeFakeTxData, - err: Error | null, - txHash: string | undefined, - cb: Callback, - ): Promise { - if (!txData.isFakeTransaction) { - // This transaction is a usual ttransaction. Not a call executed as one. - // And we don't want it to be executed within a snapshotting period - await this._lock.acquire(); - } - if (_.isNull(err)) { - const toAddress = _.isUndefined(txData.to) || txData.to === '0x0' ? constants.NEW_CONTRACT : txData.to; - await this._recordTxTraceAsync(toAddress, txData.data, txHash as string); - } else { - const payload = { - method: 'eth_getBlockByNumber', - params: [BlockParamLiteral.Latest, true], - }; - const jsonRPCResponsePayload = await this.emitPayloadAsync(payload); - const transactions = jsonRPCResponsePayload.result.transactions; - for (const transaction of transactions) { - const toAddress = _.isUndefined(txData.to) || txData.to === '0x0' ? constants.NEW_CONTRACT : txData.to; - await this._recordTxTraceAsync(toAddress, transaction.data, transaction.hash); - } - } - if (!txData.isFakeTransaction) { - // This transaction is a usual ttransaction. Not a call executed as one. - // And we don't want it to be executed within a snapshotting period - this._lock.release(); - } - cb(); - } - private async _onCallExecutedAsync( - callData: Partial, - blockNumber: BlockParam, - err: Error | null, - callResult: string, - cb: Callback, - ): Promise { - await this._recordCallTraceAsync(callData, blockNumber); - cb(); - } - private async _recordTxTraceAsync(address: string, data: string | undefined, txHash: string): Promise { - let payload = { - method: 'debug_traceTransaction', - params: [txHash, { disableMemory: true, disableStack: false, disableStorage: true }], - }; - let jsonRPCResponsePayload = await this.emitPayloadAsync(payload); - const trace: TransactionTrace = jsonRPCResponsePayload.result; - const tracesByContractAddress = getTracesByContractAddress(trace.structLogs, address); - const subcallAddresses = _.keys(tracesByContractAddress); - if (address === constants.NEW_CONTRACT) { - for (const subcallAddress of subcallAddresses) { - let traceInfo: TraceInfoNewContract | TraceInfoExistingContract; - if (subcallAddress === 'NEW_CONTRACT') { - const traceForThatSubcall = tracesByContractAddress[subcallAddress]; - const coveredPcs = _.map(traceForThatSubcall, log => log.pc); - traceInfo = { - coveredPcs, - txHash, - address: constants.NEW_CONTRACT, - bytecode: data as string, - }; - } else { - payload = { method: 'eth_getCode', params: [subcallAddress, BlockParamLiteral.Latest] }; - jsonRPCResponsePayload = await this.emitPayloadAsync(payload); - const runtimeBytecode = jsonRPCResponsePayload.result; - const traceForThatSubcall = tracesByContractAddress[subcallAddress]; - const coveredPcs = _.map(traceForThatSubcall, log => log.pc); - traceInfo = { - coveredPcs, - txHash, - address: subcallAddress, - runtimeBytecode, - }; - } - this._coverageManager.appendTraceInfo(traceInfo); - } - } else { - for (const subcallAddress of subcallAddresses) { - payload = { method: 'eth_getCode', params: [subcallAddress, BlockParamLiteral.Latest] }; - jsonRPCResponsePayload = await this.emitPayloadAsync(payload); - const runtimeBytecode = jsonRPCResponsePayload.result; - const traceForThatSubcall = tracesByContractAddress[subcallAddress]; - const coveredPcs = _.map(traceForThatSubcall, log => log.pc); - const traceInfo: TraceInfoExistingContract = { - coveredPcs, - txHash, - address: subcallAddress, - runtimeBytecode, - }; - this._coverageManager.appendTraceInfo(traceInfo); - } - } - } - private async _recordCallTraceAsync(callData: Partial, blockNumber: BlockParam): Promise { - // We don't want other transactions to be exeucted during snashotting period, that's why we lock the - // transaction execution for all transactions except our fake ones. - await this._lock.acquire(); - const snapshotId = Number((await this.emitPayloadAsync({ method: 'evm_snapshot' })).result); - const fakeTxData: MaybeFakeTxData = { - isFakeTransaction: true, // This transaction (and only it) is allowed to come through when the lock is locked - ...callData, - from: callData.from || this._defaultFromAddress, - }; - try { - await this.emitPayloadAsync({ - method: 'eth_sendTransaction', - params: [fakeTxData], - }); - } catch (err) { - // Even if this transaction failed - we've already recorded it's trace. - } - const jsonRPCResponse = await this.emitPayloadAsync({ method: 'evm_revert', params: [snapshotId] }); - this._lock.release(); - const didRevert = jsonRPCResponse.result; - if (!didRevert) { - throw new Error('Failed to revert the snapshot'); - } - } - private async _getContractCodeAsync(address: string): Promise { - const payload = { - method: 'eth_getCode', - params: [address, BlockParamLiteral.Latest], - }; - const jsonRPCResponsePayload = await this.emitPayloadAsync(payload); - const contractCode: string = jsonRPCResponsePayload.result; - return contractCode; - } } diff --git a/packages/sol-cov/src/index.ts b/packages/sol-cov/src/index.ts index 7a2afbe80..10f6d9597 100644 --- a/packages/sol-cov/src/index.ts +++ b/packages/sol-cov/src/index.ts @@ -1,4 +1,6 @@ export { CoverageSubprovider } from './coverage_subprovider'; +// HACK: ProfilerSubprovider is a hacky way to do profiling using coverage tools. Not production ready +export { ProfilerSubprovider } from './profiler_subprovider'; export { SolCompilerArtifactAdapter } from './artifact_adapters/sol_compiler_artifact_adapter'; export { TruffleArtifactAdapter } from './artifact_adapters/truffle_artifact_adapter'; export { AbstractArtifactAdapter } from './artifact_adapters/abstract_artifact_adapter'; diff --git a/packages/sol-cov/src/profiler_manager.ts b/packages/sol-cov/src/profiler_manager.ts new file mode 100644 index 000000000..0ab0ea544 --- /dev/null +++ b/packages/sol-cov/src/profiler_manager.ts @@ -0,0 +1,134 @@ +import { promisify } from '@0xproject/utils'; +import { stripHexPrefix } from 'ethereumjs-util'; +import * as fs from 'fs'; +import { Collector } from 'istanbul'; +import * as _ from 'lodash'; +import { getLogger, levels, Logger } from 'loglevel'; +import * as mkdirp from 'mkdirp'; + +import { AbstractArtifactAdapter } from './artifact_adapters/abstract_artifact_adapter'; +import { collectCoverageEntries } from './collect_coverage_entries'; +import { constants } from './constants'; +import { parseSourceMap } from './source_maps'; +import { + ContractData, + Coverage, + SingleFileSourceRange, + SourceRange, + Subtrace, + TraceInfo, + TraceInfoExistingContract, + TraceInfoNewContract, +} from './types'; +import { utils } from './utils'; + +const mkdirpAsync = promisify(mkdirp); + +/** + * ProfilerManager is used by ProfilerSubprovider to profile code while running Solidity tests based on collected trace data. + * HACK: It's almost the exact copy of CoverageManager but instead of reporting how much times was each statement executed - it reports - how expensive it was gaswise. + */ +export class ProfilerManager { + private _artifactAdapter: AbstractArtifactAdapter; + private _logger: Logger; + private _traceInfos: TraceInfo[] = []; + /** + * Computed partial coverage for a single file & subtrace + * @param contractData Contract metadata (source, srcMap, bytecode) + * @param subtrace A subset of a transcation/call trace that was executed within that contract + * @param pcToSourceRange A mapping from program counters to source ranges + * @param fileIndex Index of a file to compute coverage for + * @return Partial istanbul coverage for that file & subtrace + */ + private static _getSingleFileCoverageForSubtrace( + contractData: ContractData, + subtrace: Subtrace, + pcToSourceRange: { [programCounter: number]: SourceRange }, + fileIndex: number, + ): Coverage { + const absoluteFileName = contractData.sources[fileIndex]; + const profilerEntriesDescription = collectCoverageEntries(contractData.sourceCodes[fileIndex]); + const gasConsumedByStatement: { [statementId: string]: number } = {}; + const statementIds = _.keys(profilerEntriesDescription.statementMap); + for (const statementId of statementIds) { + const statementDescription = profilerEntriesDescription.statementMap[statementId]; + const totalGasCost = _.sum( + _.map(subtrace, structLog => { + const sourceRange = pcToSourceRange[structLog.pc]; + if (_.isUndefined(sourceRange)) { + return 0; + } + if (sourceRange.fileName !== absoluteFileName) { + return 0; + } + if (utils.isRangeInside(sourceRange.location, statementDescription)) { + return structLog.gasCost; + } else { + return 0; + } + }), + ); + gasConsumedByStatement[statementId] = totalGasCost; + } + const partialProfilerOutput = { + [absoluteFileName]: { + ...profilerEntriesDescription, + path: absoluteFileName, + f: {}, // I's meaningless in profiling context + s: gasConsumedByStatement, + b: {}, // I's meaningless in profiling context + }, + }; + return partialProfilerOutput; + } + constructor(artifactAdapter: AbstractArtifactAdapter, isVerbose: boolean) { + this._artifactAdapter = artifactAdapter; + this._logger = getLogger('sol-cov'); + this._logger.setLevel(isVerbose ? levels.TRACE : levels.ERROR); + } + public appendTraceInfo(traceInfo: TraceInfo): void { + this._traceInfos.push(traceInfo); + } + public async writeProfilerOutputAsync(): Promise { + const finalCoverage = await this._computeCoverageAsync(); + const stringifiedCoverage = JSON.stringify(finalCoverage, null, '\t'); + await mkdirpAsync('coverage'); + fs.writeFileSync('coverage/coverage.json', stringifiedCoverage); + } + private async _computeCoverageAsync(): Promise { + const contractsData = await this._artifactAdapter.collectContractsDataAsync(); + const collector = new Collector(); + for (const traceInfo of this._traceInfos) { + const isContractCreation = traceInfo.address === constants.NEW_CONTRACT; + const bytecode = isContractCreation + ? (traceInfo as TraceInfoNewContract).bytecode + : (traceInfo as TraceInfoExistingContract).runtimeBytecode; + const contractData = utils.getContractDataIfExists(contractsData, bytecode); + if (_.isUndefined(contractData)) { + const errMsg = isContractCreation + ? `Unknown contract creation transaction` + : `Transaction to an unknown address: ${traceInfo.address}`; + this._logger.warn(errMsg); + continue; + } + const bytecodeHex = stripHexPrefix(bytecode); + const sourceMap = isContractCreation ? contractData.sourceMap : contractData.sourceMapRuntime; + const pcToSourceRange = parseSourceMap( + contractData.sourceCodes, + sourceMap, + bytecodeHex, + contractData.sources, + ); + for (let fileIndex = 0; fileIndex < contractData.sources.length; fileIndex++) { + const singleFileCoverageForTrace = ProfilerManager._getSingleFileCoverageForSubtrace( + contractData, + traceInfo.subtrace, + pcToSourceRange, + fileIndex, + ); + collector.add(singleFileCoverageForTrace); + } + } + return collector.getFinalCoverage(); + } +} diff --git a/packages/sol-cov/src/profiler_subprovider.ts b/packages/sol-cov/src/profiler_subprovider.ts new file mode 100644 index 000000000..ac878c070 --- /dev/null +++ b/packages/sol-cov/src/profiler_subprovider.ts @@ -0,0 +1,36 @@ +import * as _ from 'lodash'; + +import { AbstractArtifactAdapter } from './artifact_adapters/abstract_artifact_adapter'; +import { ProfilerManager } from './profiler_manager'; +import { TraceCollectionSubprovider } from './trace_collection_subprovider'; + +/** + * This class implements the [web3-provider-engine](https://github.com/MetaMask/provider-engine) subprovider interface. + * ProfilerSubprovider is used to profile Solidity code while running tests. + */ +export class ProfilerSubprovider extends TraceCollectionSubprovider { + private _profilerManager: ProfilerManager; + /** + * Instantiates a ProfilerSubprovider instance + * @param artifactAdapter Adapter for used artifacts format (0x, truffle, giveth, etc.) + * @param defaultFromAddress default from address to use when sending transactions + * @param isVerbose If true, we will log any unknown transactions. Otherwise we will ignore them + */ + constructor(artifactAdapter: AbstractArtifactAdapter, defaultFromAddress: string, isVerbose: boolean = true) { + const traceCollectionSubproviderConfig = { + shouldCollectTransactionTraces: true, + shouldCollectGasEstimateTraces: false, + shouldCollectCallTraces: false, + }; + super(defaultFromAddress, traceCollectionSubproviderConfig); + this._profilerManager = new ProfilerManager(artifactAdapter, isVerbose); + } + /** + * Write the test profiler results to a file in Istanbul format. + */ + public async writeProfilerOutputAsync(): Promise { + const traceInfos = this.getCollectedTraceInfos(); + _.forEach(traceInfos, traceInfo => this._profilerManager.appendTraceInfo(traceInfo)); + await this._profilerManager.writeProfilerOutputAsync(); + } +} diff --git a/packages/sol-cov/src/trace.ts b/packages/sol-cov/src/trace.ts index c725de3d0..45e45e9c5 100644 --- a/packages/sol-cov/src/trace.ts +++ b/packages/sol-cov/src/trace.ts @@ -16,6 +16,13 @@ export function getTracesByContractAddress(structLogs: StructLog[], startAddress const traceByContractAddress: TraceByContractAddress = {}; let currentTraceSegment = []; const callStack = [startAddress]; + if (_.isEmpty(structLogs)) { + return traceByContractAddress; + } + if (structLogs[0].depth === 1) { + // Geth uses 1-indexed depth counter whilst ganache starts from 0 + _.forEach(structLogs, structLog => structLog.depth--); + } // tslint:disable-next-line:prefer-for-of for (let i = 0; i < structLogs.length; i++) { const structLog = structLogs[i]; @@ -95,10 +102,15 @@ export function getTracesByContractAddress(structLogs: StructLog[], startAddress } } if (callStack.length !== 0) { - throw new Error('Malformed trace. Call stack non empty at the end'); + logUtils.warn('Malformed trace. Call stack non empty at the end'); } if (currentTraceSegment.length !== 0) { - throw new Error('Malformed trace. Current trace segment non empty at the end'); + const currentAddress = callStack.pop() as string; + traceByContractAddress[currentAddress] = (traceByContractAddress[currentAddress] || []).concat( + currentTraceSegment, + ); + currentTraceSegment = []; + logUtils.warn('Malformed trace. Current trace segment non empty at the end'); } return traceByContractAddress; } diff --git a/packages/sol-cov/src/trace_collection_subprovider.ts b/packages/sol-cov/src/trace_collection_subprovider.ts new file mode 100644 index 000000000..222561099 --- /dev/null +++ b/packages/sol-cov/src/trace_collection_subprovider.ts @@ -0,0 +1,258 @@ +import { BlockchainLifecycle } from '@0xproject/dev-utils'; +import { Callback, ErrorCallback, NextCallback, Subprovider } from '@0xproject/subproviders'; +import { Web3Wrapper } from '@0xproject/web3-wrapper'; +import { CallData, JSONRPCRequestPayload, Provider, TxData } from 'ethereum-types'; +import * as _ from 'lodash'; +import { Lock } from 'semaphore-async-await'; + +import { constants } from './constants'; +import { getTracesByContractAddress } from './trace'; +import { BlockParamLiteral, TraceInfo, TraceInfoExistingContract, TraceInfoNewContract } from './types'; + +interface MaybeFakeTxData extends TxData { + isFakeTransaction?: boolean; +} + +const BLOCK_GAS_LIMIT = 6000000; + +export interface TraceCollectionSubproviderConfig { + shouldCollectTransactionTraces: boolean; + shouldCollectCallTraces: boolean; + shouldCollectGasEstimateTraces: boolean; +} + +// Because there is no notion of a call trace in the Ethereum rpc - we collect them in a rather non-obvious/hacky way. +// On each call - we create a snapshot, execute the call as a transaction, get the trace, revert the snapshot. +// That allows us to avoid influencing test behaviour. + +/** + * This class implements the [web3-provider-engine](https://github.com/MetaMask/provider-engine) subprovider interface. + * It collects traces of all transactions that were sent and all calls that were executed through JSON RPC. + */ +export class TraceCollectionSubprovider extends Subprovider { + // Lock is used to not accept normal transactions while doing call/snapshot magic because they'll be reverted later otherwise + private _lock = new Lock(); + private _defaultFromAddress: string; + private _web3Wrapper!: Web3Wrapper; + private _traceInfos: TraceInfo[] = []; + private _isEnabled = true; + private _config: TraceCollectionSubproviderConfig; + /** + * Instantiates a TraceCollectionSubprovider instance + * @param defaultFromAddress default from address to use when sending transactions + */ + constructor(defaultFromAddress: string, config: TraceCollectionSubproviderConfig) { + super(); + this._defaultFromAddress = defaultFromAddress; + this._config = config; + } + /** + * Returns all trace infos collected by the subprovider so far + */ + public getCollectedTraceInfos(): TraceInfo[] { + return this._traceInfos; + } + /** + * Starts trace collection + */ + public start(): void { + this._isEnabled = true; + } + /** + * Stops trace collection + */ + public stop(): void { + this._isEnabled = false; + } + /** + * This method conforms to the web3-provider-engine interface. + * It is called internally by the ProviderEngine when it is this subproviders + * turn to handle a JSON RPC request. + * @param payload JSON RPC payload + * @param next Callback to call if this subprovider decides not to handle the request + * @param end Callback to call if subprovider handled the request and wants to pass back the request. + */ + // tslint:disable-next-line:prefer-function-over-method async-suffix + public async handleRequest(payload: JSONRPCRequestPayload, next: NextCallback, end: ErrorCallback): Promise { + if (this._isEnabled) { + switch (payload.method) { + case 'eth_sendTransaction': + if (!this._config.shouldCollectTransactionTraces) { + next(); + } else { + const txData = payload.params[0]; + next(this._onTransactionSentAsync.bind(this, txData)); + } + return; + + case 'eth_call': + if (!this._config.shouldCollectCallTraces) { + next(); + } else { + const callData = payload.params[0]; + next(this._onCallExecutedAsync.bind(this, callData)); + } + return; + + case 'eth_estimateGas': + if (!this._config.shouldCollectGasEstimateTraces) { + next(); + } else { + const estimateGasData = payload.params[0]; + next(this._onGasEstimateAsync.bind(this, estimateGasData)); + } + return; + + default: + next(); + return; + } + } else { + next(); + return; + } + } + /** + * Set's the subprovider's engine to the ProviderEngine it is added to. + * This is only called within the ProviderEngine source code, do not call + * directly. + */ + public setEngine(engine: Provider): void { + super.setEngine(engine); + this._web3Wrapper = new Web3Wrapper(engine); + } + private async _onTransactionSentAsync( + txData: MaybeFakeTxData, + err: Error | null, + txHash: string | undefined, + cb: Callback, + ): Promise { + if (!txData.isFakeTransaction) { + // This transaction is a usual ttransaction. Not a call executed as one. + // And we don't want it to be executed within a snapshotting period + await this._lock.acquire(); + } + if (_.isNull(err)) { + const toAddress = _.isUndefined(txData.to) || txData.to === '0x0' ? constants.NEW_CONTRACT : txData.to; + await this._recordTxTraceAsync(toAddress, txData.data, txHash as string); + } else { + const latestBlock = await this._web3Wrapper.getBlockWithTransactionDataAsync(BlockParamLiteral.Latest); + const transactions = latestBlock.transactions; + for (const transaction of transactions) { + const toAddress = _.isUndefined(txData.to) || txData.to === '0x0' ? constants.NEW_CONTRACT : txData.to; + await this._recordTxTraceAsync(toAddress, transaction.input, transaction.hash); + } + } + if (!txData.isFakeTransaction) { + // This transaction is a usual ttransaction. Not a call executed as one. + // And we don't want it to be executed within a snapshotting period + this._lock.release(); + } + cb(); + } + private async _onCallExecutedAsync( + callData: Partial, + err: Error | null, + callResult: string, + cb: Callback, + ): Promise { + await this._recordCallTraceAsync(callData); + cb(); + } + private async _onGasEstimateAsync( + estimateGasData: Partial, + err: Error | null, + estimateGasResult: string, + cb: Callback, + ): Promise { + await this._recordEstimateGasTraceAsync(estimateGasData); + cb(); + } + private async _recordTxTraceAsync(address: string, data: string | undefined, txHash: string): Promise { + await this._web3Wrapper.awaitTransactionMinedAsync(txHash); + const trace = await this._web3Wrapper.getTransactionTraceAsync(txHash, { + disableMemory: true, + disableStack: false, + disableStorage: true, + }); + const tracesByContractAddress = getTracesByContractAddress(trace.structLogs, address); + const subcallAddresses = _.keys(tracesByContractAddress); + if (address === constants.NEW_CONTRACT) { + for (const subcallAddress of subcallAddresses) { + let traceInfo: TraceInfoNewContract | TraceInfoExistingContract; + if (subcallAddress === 'NEW_CONTRACT') { + const traceForThatSubcall = tracesByContractAddress[subcallAddress]; + traceInfo = { + subtrace: traceForThatSubcall, + txHash, + address: subcallAddress, + bytecode: data as string, + }; + } else { + const runtimeBytecode = await this._web3Wrapper.getContractCodeAsync(subcallAddress); + const traceForThatSubcall = tracesByContractAddress[subcallAddress]; + traceInfo = { + subtrace: traceForThatSubcall, + txHash, + address: subcallAddress, + runtimeBytecode, + }; + } + this._traceInfos.push(traceInfo); + } + } else { + for (const subcallAddress of subcallAddresses) { + const runtimeBytecode = await this._web3Wrapper.getContractCodeAsync(subcallAddress); + const traceForThatSubcall = tracesByContractAddress[subcallAddress]; + const traceInfo: TraceInfoExistingContract = { + subtrace: traceForThatSubcall, + txHash, + address: subcallAddress, + runtimeBytecode, + }; + this._traceInfos.push(traceInfo); + } + } + } + private async _recordCallTraceAsync(callData: Partial): Promise { + // We don't want other transactions to be exeucted during snashotting period, that's why we lock the + // transaction execution for all transactions except our fake ones. + await this._lock.acquire(); + const blockchainLifecycle = new BlockchainLifecycle(this._web3Wrapper); + // debugPrinter.enterFunction('snapshot'); + await blockchainLifecycle.startAsync(); + const fakeTxData: MaybeFakeTxData = { + isFakeTransaction: true, // This transaction (and only it) is allowed to come through when the lock is locked + ...callData, + from: callData.from || this._defaultFromAddress, + }; + try { + await this._web3Wrapper.sendTransactionAsync(fakeTxData); + } catch (err) { + // Even if this transaction failed - we've already recorded it's trace. + } + // debugPrinter.leaveFunction('snapshot'); + await blockchainLifecycle.revertAsync(); + this._lock.release(); + } + private async _recordEstimateGasTraceAsync(estimateGasData: Partial): Promise { + // We don't want other transactions to be exeucted during snashotting period, that's why we lock the + // transaction execution for all transactions except our fake ones. + await this._lock.acquire(); + const blockchainLifecycle = new BlockchainLifecycle(this._web3Wrapper); + await blockchainLifecycle.startAsync(); + const fakeTxData: MaybeFakeTxData = { + isFakeTransaction: true, // This transaction (and only it) is allowed to come through when the lock is locked + ...estimateGasData, + from: estimateGasData.from || this._defaultFromAddress, + gas: BLOCK_GAS_LIMIT, + }; + try { + await this._web3Wrapper.sendTransactionAsync(fakeTxData); + } catch (err) { + // Even if this transaction failed - we've already recorded it's trace. + } + await blockchainLifecycle.revertAsync(); + this._lock.release(); + } +} diff --git a/packages/sol-cov/src/types.ts b/packages/sol-cov/src/types.ts index 4c3de55a1..896d4a7b5 100644 --- a/packages/sol-cov/src/types.ts +++ b/packages/sol-cov/src/types.ts @@ -1,3 +1,5 @@ +import { StructLog } from 'ethereum-types'; + export interface LineColumn { line: number; column: number; @@ -45,24 +47,24 @@ export interface StatementMap { } export interface LineCoverage { - [lineNo: number]: boolean; + [lineNo: number]: number; } export interface FunctionCoverage { - [functionId: string]: boolean; + [functionId: string]: number; } export interface StatementCoverage { - [statementId: string]: boolean; + [statementId: string]: number; } export interface BranchCoverage { - [branchId: string]: boolean[]; + [branchId: string]: number[]; } export interface Coverage { [fineName: string]: { - l: LineCoverage; + l?: LineCoverage; f: FunctionCoverage; s: StatementCoverage; b: BranchCoverage; @@ -82,8 +84,11 @@ export interface ContractData { sources: string[]; } +// Part of the trace executed within the same context +export type Subtrace = StructLog[]; + export interface TraceInfoBase { - coveredPcs: number[]; + subtrace: Subtrace; txHash: string; } diff --git a/packages/sol-cov/src/utils.ts b/packages/sol-cov/src/utils.ts index d970c42ee..7333b2f4d 100644 --- a/packages/sol-cov/src/utils.ts +++ b/packages/sol-cov/src/utils.ts @@ -1,4 +1,6 @@ -import { LineColumn, SingleFileSourceRange } from './types'; +import * as _ from 'lodash'; + +import { ContractData, LineColumn, SingleFileSourceRange } from './types'; export const utils = { compareLineColumn(lhs: LineColumn, rhs: LineColumn): number { @@ -14,4 +16,30 @@ export const utils = { utils.compareLineColumn(childRange.end, parentRange.end) <= 0 ); }, + bytecodeToBytecodeRegex(bytecode: string): string { + const bytecodeRegex = bytecode + // Library linking placeholder: __ConvertLib____________________________ + .replace(/_.*_/, '.*') + // Last 86 characters is solidity compiler metadata that's different between compilations + .replace(/.{86}$/, '') + // Libraries contain their own address at the beginning of the code and it's impossible to know it in advance + .replace(/^0x730000000000000000000000000000000000000000/, '0x73........................................'); + // HACK: Node regexes can't be longer that 32767 characters. Contracts bytecode can. We jsut truncate the regexes. It's safe in practice. + const MAX_REGEX_LENGTH = 32767; + const truncatedBytecodeRegex = bytecodeRegex.slice(0, MAX_REGEX_LENGTH); + return truncatedBytecodeRegex; + }, + getContractDataIfExists(contractsData: ContractData[], bytecode: string): ContractData | undefined { + if (!bytecode.startsWith('0x')) { + throw new Error(`0x hex prefix missing: ${bytecode}`); + } + const contractData = _.find(contractsData, contractDataCandidate => { + const bytecodeRegex = utils.bytecodeToBytecodeRegex(contractDataCandidate.bytecode); + const runtimeBytecodeRegex = utils.bytecodeToBytecodeRegex(contractDataCandidate.runtimeBytecode); + // We use that function to find by bytecode or runtimeBytecode. Those are quasi-random strings so + // collisions are practically impossible and it allows us to reuse that code + return !_.isNull(bytecode.match(bytecodeRegex)) || !_.isNull(bytecode.match(runtimeBytecodeRegex)); + }); + return contractData; + }, }; -- cgit v1.2.3