From 98c1952956d87adf5e73a42a1b78b8bf8b4119d2 Mon Sep 17 00:00:00 2001 From: "F. Eugene Aumson" Date: Tue, 2 Oct 2018 15:00:35 -0400 Subject: fix: persist artifacts with only relevant sources https://github.com/0xProject/0x-monorepo/pull/1108 https://app.asana.com/0/684263176955174/842516551768097/f --- packages/sol-compiler/src/compiler.ts | 57 ++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 5 deletions(-) (limited to 'packages/sol-compiler/src') diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts index a29367485..350a5a125 100644 --- a/packages/sol-compiler/src/compiler.ts +++ b/packages/sol-compiler/src/compiler.ts @@ -296,13 +296,12 @@ export class Compiler { compilerOutput: solc.StandardOutput, ): Promise { const compiledContract = compilerOutput.contracts[contractPath][contractName]; - const sourceCodes = _.mapValues( - compilerOutput.sources, - (_1, sourceFilePath) => this._resolver.resolve(sourceFilePath).source, - ); + + const { sourceCodes, sources } = this._getSourcesWithDependencies(contractPath); + const contractVersion: ContractVersionData = { compilerOutput: compiledContract, - sources: compilerOutput.sources, + sources, sourceCodes, sourceTreeHashHex, compiler: { @@ -333,6 +332,54 @@ export class Compiler { await fsWrapper.writeFileAsync(currentArtifactPath, artifactString); logUtils.warn(`${contractName} artifact saved!`); } + private _getSourcesWithDependencies( + contractPath: string, + ): { sourceCodes: { [sourceName: string]: string }; sources: { [sourceName: string]: { id: number } } } { + const sources = { [contractPath]: { id: 0 } }; + const sourceCodes = { [contractPath]: this._resolver.resolve(contractPath).source }; + this._recursivelyGatherDependencySources(contractPath, sourceCodes[contractPath], sources, sourceCodes, 1); + return { sourceCodes, sources }; + } + private _recursivelyGatherDependencySources( + contractPath: string, + contractSource: string, + sourcesToAppendTo: { [sourceName: string]: { id: number } }, + sourceCodesToAppendTo: { [sourceName: string]: string }, + nextId: number, + ): number { + let nextId_ = nextId; + + const importStatementMatches = contractSource.match(/import[^;]*;/g); + if (importStatementMatches === null) { + return nextId_; + } + for (const importStatementMatch of importStatementMatches) { + const importPathMatches = importStatementMatch.match(/\"([^\"]*)\"/); + if (importPathMatches === null || importPathMatches.length === 0) { + continue; + } + + let importPath = importPathMatches[1]; + const lastPathSeparatorPos = contractPath.lastIndexOf('/'); + const importFolder = lastPathSeparatorPos === -1 ? '' : contractPath.slice(0, lastPathSeparatorPos + 1); + importPath = importPath.slice(0, 2) === './' ? importPath.replace(/^.\//, importFolder) : importPath; + + if (_.isUndefined(sourcesToAppendTo[importPath])) { + sourcesToAppendTo[importPath] = { id: nextId_ }; + sourceCodesToAppendTo[importPath] = this._resolver.resolve(importPath).source; + nextId_ += 1; + + nextId_ = this._recursivelyGatherDependencySources( + importPath, + this._resolver.resolve(importPath).source, + sourcesToAppendTo, + sourceCodesToAppendTo, + nextId_, + ); + } + } + return nextId_; + } private _compile(solcInstance: solc.SolcInstance, standardInput: solc.StandardInput): solc.StandardOutput { const compiled: solc.StandardOutput = JSON.parse( solcInstance.compileStandardWrapper(JSON.stringify(standardInput), importPath => { -- cgit v1.2.3 From 37c55302e74d6a6706a390fe03432c0d3f119510 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Tue, 2 Oct 2018 17:29:47 -0700 Subject: Fix some small bugs in compiler.ts --- packages/sol-compiler/src/compiler.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'packages/sol-compiler/src') diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts index 350a5a125..ea080c312 100644 --- a/packages/sol-compiler/src/compiler.ts +++ b/packages/sol-compiler/src/compiler.ts @@ -349,7 +349,7 @@ export class Compiler { ): number { let nextId_ = nextId; - const importStatementMatches = contractSource.match(/import[^;]*;/g); + const importStatementMatches = contractSource.match(/\nimport[^;]*;/g); if (importStatementMatches === null) { return nextId_; } @@ -360,9 +360,24 @@ export class Compiler { } let importPath = importPathMatches[1]; + // HACK(ablrow): We have, e.g.: + // + // importPath = "../../utils/LibBytes/LibBytes.sol" + // contractPath = "2.0.0/protocol/AssetProxyOwner/AssetProxyOwner.sol" + // + // Resolver doesn't understand "../" so we want to pass + // "2.0.0/utils/LibBytes/LibBytes.sol" to resolver. + // + // This hack involves using path.resolve. But path.resolve returns + // absolute directories by default. We trick it into thinking that + // contractPath is a root directory by prepending a '/' and then + // removing the '/' the end. + // + // path.resolve("/a/b/c", ""../../d/e") === "/a/d/e" + // const lastPathSeparatorPos = contractPath.lastIndexOf('/'); - const importFolder = lastPathSeparatorPos === -1 ? '' : contractPath.slice(0, lastPathSeparatorPos + 1); - importPath = importPath.slice(0, 2) === './' ? importPath.replace(/^.\//, importFolder) : importPath; + const contractFolder = lastPathSeparatorPos === -1 ? '' : contractPath.slice(0, lastPathSeparatorPos + 1); + importPath = path.resolve('/' + contractFolder, importPath).replace('/', ''); if (_.isUndefined(sourcesToAppendTo[importPath])) { sourcesToAppendTo[importPath] = { id: nextId_ }; -- cgit v1.2.3 From 39a336ca6d6879ad6413cee714d929cb99bcd968 Mon Sep 17 00:00:00 2001 From: "F. Eugene Aumson" Date: Wed, 3 Oct 2018 10:44:19 -0400 Subject: fix: use original source ID's from compiler output Previously, was generating fresh source ID's but per @LogvinovLeon 's comment (cited below) that will likely break existing source code mappings. Changed to use the original source code mapping ID's that were generated by the compiler https://app.asana.com/0/684263176955174/842516551768097/f https://github.com/0xProject/0x-monorepo/pull/1108 https://github.com/0xProject/0x-monorepo/pull/1108#pullrequestreview-161059063 --- packages/sol-compiler/src/compiler.ts | 37 +++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 13 deletions(-) (limited to 'packages/sol-compiler/src') diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts index ea080c312..620728ed2 100644 --- a/packages/sol-compiler/src/compiler.ts +++ b/packages/sol-compiler/src/compiler.ts @@ -297,7 +297,7 @@ export class Compiler { ): Promise { const compiledContract = compilerOutput.contracts[contractPath][contractName]; - const { sourceCodes, sources } = this._getSourcesWithDependencies(contractPath); + const { sourceCodes, sources } = this._getSourcesWithDependencies(contractPath, compilerOutput.sources); const contractVersion: ContractVersionData = { compilerOutput: compiledContract, @@ -332,26 +332,39 @@ export class Compiler { await fsWrapper.writeFileAsync(currentArtifactPath, artifactString); logUtils.warn(`${contractName} artifact saved!`); } + /** + * For the given @param contractPath, populates JSON objects to be used in the ContractArtifact interface's + * properties `sources` (source code file names mapped to ID numbers) and `sourceCodes` (source code content of + * contracts) for that contract. The source code pointed to by contractPath is read and parsed directly (via + * `this._resolver.resolve().source`), as are its imports, recursively. The ID numbers for @return `sources` are + * taken from the corresponding ID's in @param fullSources, and the content for @return sourceCodes is read from + * disk (via the aforementioned `resolver.source`). + */ private _getSourcesWithDependencies( contractPath: string, + fullSources: { [sourceName: string]: { id: number } }, ): { sourceCodes: { [sourceName: string]: string }; sources: { [sourceName: string]: { id: number } } } { - const sources = { [contractPath]: { id: 0 } }; + const sources = { [contractPath]: { id: fullSources[contractPath].id } }; const sourceCodes = { [contractPath]: this._resolver.resolve(contractPath).source }; - this._recursivelyGatherDependencySources(contractPath, sourceCodes[contractPath], sources, sourceCodes, 1); + this._recursivelyGatherDependencySources( + contractPath, + sourceCodes[contractPath], + fullSources, + sources, + sourceCodes, + ); return { sourceCodes, sources }; } private _recursivelyGatherDependencySources( contractPath: string, contractSource: string, + fullSources: { [sourceName: string]: { id: number } }, sourcesToAppendTo: { [sourceName: string]: { id: number } }, sourceCodesToAppendTo: { [sourceName: string]: string }, - nextId: number, - ): number { - let nextId_ = nextId; - + ): void { const importStatementMatches = contractSource.match(/\nimport[^;]*;/g); if (importStatementMatches === null) { - return nextId_; + return; } for (const importStatementMatch of importStatementMatches) { const importPathMatches = importStatementMatch.match(/\"([^\"]*)\"/); @@ -380,20 +393,18 @@ export class Compiler { importPath = path.resolve('/' + contractFolder, importPath).replace('/', ''); if (_.isUndefined(sourcesToAppendTo[importPath])) { - sourcesToAppendTo[importPath] = { id: nextId_ }; + sourcesToAppendTo[importPath] = { id: fullSources[importPath].id }; sourceCodesToAppendTo[importPath] = this._resolver.resolve(importPath).source; - nextId_ += 1; - nextId_ = this._recursivelyGatherDependencySources( + this._recursivelyGatherDependencySources( importPath, this._resolver.resolve(importPath).source, + fullSources, sourcesToAppendTo, sourceCodesToAppendTo, - nextId_, ); } } - return nextId_; } private _compile(solcInstance: solc.SolcInstance, standardInput: solc.StandardInput): solc.StandardOutput { const compiled: solc.StandardOutput = JSON.parse( -- cgit v1.2.3 From f614a2425f79c0ba4982b0868ed6edd9c491ef1b Mon Sep 17 00:00:00 2001 From: "F. Eugene Aumson" Date: Wed, 3 Oct 2018 10:57:37 -0400 Subject: fix: comment need for sourceCodes pruning --- packages/sol-compiler/src/compiler.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'packages/sol-compiler/src') diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts index 620728ed2..7eefc1474 100644 --- a/packages/sol-compiler/src/compiler.ts +++ b/packages/sol-compiler/src/compiler.ts @@ -297,6 +297,10 @@ export class Compiler { ): Promise { const compiledContract = compilerOutput.contracts[contractPath][contractName]; + // need to gather sourceCodes for this artifact, but compilerOutput.sources (the list of contract modules) + // contains listings for for every contract compiled during the compiler invocation that compiled the contract + // to be persisted, which could include many that are irrelevant to the contract at hand. So, gather up only + // the relevant sources: const { sourceCodes, sources } = this._getSourcesWithDependencies(contractPath, compilerOutput.sources); const contractVersion: ContractVersionData = { @@ -333,7 +337,7 @@ export class Compiler { logUtils.warn(`${contractName} artifact saved!`); } /** - * For the given @param contractPath, populates JSON objects to be used in the ContractArtifact interface's + * For the given @param contractPath, populates JSON objects to be used in the ContractVersionData interface's * properties `sources` (source code file names mapped to ID numbers) and `sourceCodes` (source code content of * contracts) for that contract. The source code pointed to by contractPath is read and parsed directly (via * `this._resolver.resolve().source`), as are its imports, recursively. The ID numbers for @return `sources` are -- cgit v1.2.3