From 0c12128f64f7d9a8de6088e98c2e638533d6f5bf Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Mon, 28 Jan 2019 16:24:54 +0100 Subject: Temp CI --- contracts/examples/compiler.json | 1 - contracts/extensions/compiler.json | 1 - contracts/interfaces/compiler.json | 1 - contracts/libs/compiler.json | 1 - contracts/multisig/compiler.json | 1 - contracts/protocol/compiler.json | 1 - contracts/tokens/compiler.json | 1 - contracts/utils/compiler.json | 1 - packages/ethereum-types/src/index.ts | 4 +- packages/sol-compiler/src/compiler.ts | 82 +++++++--------- .../src/schemas/compiler_options_schema.ts | 1 - packages/sol-compiler/src/utils/compiler.ts | 109 ++++++++++----------- .../sol-resolver/src/resolvers/npm_resolver.ts | 13 +-- packages/typescript-typings/types/solc/index.d.ts | 2 +- 14 files changed, 93 insertions(+), 126 deletions(-) diff --git a/contracts/examples/compiler.json b/contracts/examples/compiler.json index 868a11905..7c78fba3b 100644 --- a/contracts/examples/compiler.json +++ b/contracts/examples/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/extensions/compiler.json b/contracts/extensions/compiler.json index a5c7bcc21..e95478e34 100644 --- a/contracts/extensions/compiler.json +++ b/contracts/extensions/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/interfaces/compiler.json b/contracts/interfaces/compiler.json index 496bc2327..ab3bc1078 100644 --- a/contracts/interfaces/compiler.json +++ b/contracts/interfaces/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/libs/compiler.json b/contracts/libs/compiler.json index 09ce91442..c4f0c6bd9 100644 --- a/contracts/libs/compiler.json +++ b/contracts/libs/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/multisig/compiler.json b/contracts/multisig/compiler.json index 9555fbbfd..5c4d262dc 100644 --- a/contracts/multisig/compiler.json +++ b/contracts/multisig/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/protocol/compiler.json b/contracts/protocol/compiler.json index 4fc3712b0..f4d2a5f45 100644 --- a/contracts/protocol/compiler.json +++ b/contracts/protocol/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/tokens/compiler.json b/contracts/tokens/compiler.json index 12e603cab..eec24a769 100644 --- a/contracts/tokens/compiler.json +++ b/contracts/tokens/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/contracts/utils/compiler.json b/contracts/utils/compiler.json index df4ef1e9f..7473ea6d1 100644 --- a/contracts/utils/compiler.json +++ b/contracts/utils/compiler.json @@ -1,7 +1,6 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", - "workspaceDir": "../..", "useDockerisedSolc": true, "compilerSettings": { "optimizer": { diff --git a/packages/ethereum-types/src/index.ts b/packages/ethereum-types/src/index.ts index a1f6919a8..bbc3682c7 100644 --- a/packages/ethereum-types/src/index.ts +++ b/packages/ethereum-types/src/index.ts @@ -503,8 +503,7 @@ export interface Source { /** * Options you can specify (as flags or in a compiler.json file) when invoking sol-compiler - * contractsDir: Directory containing your package's Solidity contracts. Can contain nested directories. - * workspaceDir: Directory containing your project's Solidity contracts. All the contracts used in compilation must be withing it. Similar to --allow-paths in Solidity. + * contractsDir: Directory containing your project's Solidity contracts. Can contain nested directories. * artifactsDir: Directory where you want the generated artifacts.json written to * compilerSettings: Desired settings to pass to the Solidity compiler during compilation. * (http://solidity.readthedocs.io/en/v0.4.24/using-the-compiler.html#compiler-input-and-output-json-description) @@ -516,7 +515,6 @@ export interface Source { */ export interface CompilerOptions { contractsDir?: string; - workspaceDir?: string; artifactsDir?: string; compilerSettings?: CompilerSettings; contracts?: string[] | '*'; diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts index 856bcbd48..b5cdf9865 100644 --- a/packages/sol-compiler/src/compiler.ts +++ b/packages/sol-compiler/src/compiler.ts @@ -28,10 +28,13 @@ import { compileSolcJS, createDirIfDoesNotExistAsync, getContractArtifactIfExistsAsync, + getDependencyNameToPackagePath, getSolcJSAsync, getSourcesWithDependencies, getSourceTreeHash, + makeContractPathsRelative, parseSolidityVersionRange, + printCompilationErrorsAndWarnings, } from './utils/compiler'; import { constants } from './utils/constants'; import { fsWrapper } from './utils/fs_wrapper'; @@ -83,7 +86,6 @@ export class Compiler { private readonly _resolver: Resolver; private readonly _nameResolver: NameResolver; private readonly _contractsDir: string; - private readonly _workspaceDir: string; private readonly _compilerSettings: solc.CompilerSettings; private readonly _artifactsDir: string; private readonly _solcVersionIfExists: string | undefined; @@ -103,12 +105,6 @@ export class Compiler { const passedOpts = opts || {}; assert.doesConformToSchema('compiler.json', config, compilerOptionsSchema); this._contractsDir = path.resolve(passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR); - this._workspaceDir = path.resolve(passedOpts.workspaceDir || config.workspaceDir || this._contractsDir); - if (!this._contractsDir.includes(this._workspaceDir)) { - throw new Error( - `Contracts dir ${this._contractsDir} is outside of the workspace dir ${this._workspaceDir}`, - ); - } this._solcVersionIfExists = passedOpts.solcVersion || config.solcVersion; this._compilerSettings = passedOpts.compilerSettings || config.compilerSettings || DEFAULT_COMPILER_SETTINGS; this._artifactsDir = passedOpts.artifactsDir || config.artifactsDir || DEFAULT_ARTIFACTS_DIR; @@ -118,7 +114,7 @@ export class Compiler { this._nameResolver = new NameResolver(this._contractsDir); const resolver = new FallthroughResolver(); resolver.appendResolver(new URLResolver()); - resolver.appendResolver(new NPMResolver(this._contractsDir, this._workspaceDir)); + resolver.appendResolver(new NPMResolver(this._contractsDir)); resolver.appendResolver(new RelativeFSResolver(this._contractsDir)); resolver.appendResolver(new FSResolver()); resolver.appendResolver(this._nameResolver); @@ -217,9 +213,10 @@ export class Compiler { // map contract paths to data about them for later verification and persistence const contractPathToData: ContractPathToData = {}; - const spyResolver = new SpyResolver(this._resolver); + const resolvedContractSources = []; for (const contractName of contractNames) { + const spyResolver = new SpyResolver(this._resolver); const contractSource = spyResolver.resolve(contractName); const sourceTreeHashHex = getSourceTreeHash( spyResolver, @@ -249,40 +246,19 @@ export class Compiler { }; } // add input to the right version batch - versionToInputs[solcVersion].standardInput.sources[contractSource.path] = { - content: contractSource.source, - }; + for (const resolvedContractSource of spyResolver.resolvedContractSources) { + versionToInputs[solcVersion].standardInput.sources[resolvedContractSource.absolutePath] = { + content: resolvedContractSource.source, + }; + } + resolvedContractSources.push(...spyResolver.resolvedContractSources); versionToInputs[solcVersion].contractsToCompile.push(contractSource.path); } - const allTouchedFiles = spyResolver.resolvedContractSources.map( - contractSource => `${contractSource.absolutePath}`, - ); - const NODE_MODULES = 'node_modules'; - const allTouchedDependencies = _.filter(allTouchedFiles, filePath => filePath.includes(NODE_MODULES)); - const dependencyNameToPackagePath: { [dependencyName: string]: string } = {}; - _.map(allTouchedDependencies, dependencyFilePath => { - const lastNodeModulesStart = dependencyFilePath.lastIndexOf(NODE_MODULES); - const lastNodeModulesEnd = lastNodeModulesStart + NODE_MODULES.length; - const importPath = dependencyFilePath.substr(lastNodeModulesEnd + 1); - let packageName; - let packageScopeIfExists; - let dependencyName; - if (_.startsWith(importPath, '@')) { - [packageScopeIfExists, packageName] = importPath.split('/'); - dependencyName = `${packageScopeIfExists}/${packageName}`; - } else { - [packageName] = importPath.split('/'); - dependencyName = `${packageName}`; - } - const dependencyPackagePath = path.join(dependencyFilePath.substr(0, lastNodeModulesEnd), dependencyName); - dependencyNameToPackagePath[dependencyName] = dependencyPackagePath; - }); + const dependencyNameToPackagePath = getDependencyNameToPackagePath(resolvedContractSources); const compilerOutputs: StandardOutput[] = []; - - const solcVersions = _.keys(versionToInputs); - for (const solcVersion of solcVersions) { + for (const solcVersion of _.keys(versionToInputs)) { const input = versionToInputs[solcVersion]; logUtils.warn( `Compiling ${input.contractsToCompile.length} contracts (${ @@ -291,29 +267,37 @@ export class Compiler { ); let compilerOutput; let fullSolcVersion; + input.standardInput.settings.remappings = _.map( + dependencyNameToPackagePath, + (dependencyPackagePath: string, dependencyName: string) => `${dependencyName}=${dependencyPackagePath}`, + ); if (this._useDockerisedSolc) { const dockerCommand = `docker run ethereum/solc:${solcVersion} --version`; const versionCommandOutput = execSync(dockerCommand).toString(); const versionCommandOutputParts = versionCommandOutput.split(' '); fullSolcVersion = versionCommandOutputParts[versionCommandOutputParts.length - 1].trim(); - compilerOutput = compileDocker( - this._resolver, - this._contractsDir, - this._workspaceDir, - solcVersion, - dependencyNameToPackagePath, - input.standardInput, - ); + compilerOutput = compileDocker(solcVersion, input.standardInput); } else { fullSolcVersion = binPaths[solcVersion]; - const solcInstance = await getSolcJSAsync(solcVersion); - compilerOutput = compileSolcJS(this._resolver, solcInstance, input.standardInput); + compilerOutput = compileSolcJS(solcVersion, input.standardInput); + } + compilerOutput.sources = makeContractPathsRelative( + compilerOutput.sources, + this._contractsDir, + dependencyNameToPackagePath, + ); + compilerOutput.contracts = makeContractPathsRelative( + compilerOutput.contracts, + this._contractsDir, + dependencyNameToPackagePath, + ); + if (!_.isUndefined(compilerOutput.errors)) { + printCompilationErrorsAndWarnings(compilerOutput.errors); } compilerOutputs.push(compilerOutput); for (const contractPath of input.contractsToCompile) { - // console.log('contractsPath', contractPath); const contractName = contractPathToData[contractPath].contractName; const compiledContract = compilerOutput.contracts[contractPath][contractName]; diff --git a/packages/sol-compiler/src/schemas/compiler_options_schema.ts b/packages/sol-compiler/src/schemas/compiler_options_schema.ts index 657b801ad..c0766b625 100644 --- a/packages/sol-compiler/src/schemas/compiler_options_schema.ts +++ b/packages/sol-compiler/src/schemas/compiler_options_schema.ts @@ -2,7 +2,6 @@ export const compilerOptionsSchema = { id: '/CompilerOptions', properties: { contractsDir: { type: 'string' }, - workspaceDir: { type: 'string' }, artifactsDir: { type: 'string' }, solcVersion: { type: 'string', pattern: '^\\d+.\\d+.\\d+$' }, compilerSettings: { type: 'object' }, diff --git a/packages/sol-compiler/src/utils/compiler.ts b/packages/sol-compiler/src/utils/compiler.ts index 669bdd7a4..6381e525f 100644 --- a/packages/sol-compiler/src/utils/compiler.ts +++ b/packages/sol-compiler/src/utils/compiler.ts @@ -118,76 +118,35 @@ export function parseDependencies(contractSource: ContractSource): string[] { /** * Compiles the contracts and prints errors/warnings - * @param resolver Resolver - * @param solcInstance Instance of a solc compiler + * @param solcVersion Version of a solc compiler * @param standardInput Solidity standard JSON input */ -export function compileSolcJS( - resolver: Resolver, - solcInstance: solc.SolcInstance, +export async function compileSolcJSAsync( + solcVersion: string, standardInput: solc.StandardInput, -): solc.StandardOutput { +): Promise { + const solcInstance = await getSolcJSAsync(solcVersion); const standardInputStr = JSON.stringify(standardInput); - const standardOutputStr = solcInstance.compileStandardWrapper(standardInputStr, importPath => { - const sourceCodeIfExists = resolver.resolve(importPath); - return { contents: sourceCodeIfExists.source }; - }); + const standardOutputStr = solcInstance.compileStandardWrapper(standardInputStr); const compiled: solc.StandardOutput = JSON.parse(standardOutputStr); - if (!_.isUndefined(compiled.errors)) { - printCompilationErrorsAndWarnings(compiled.errors); - } return compiled; } /** * Compiles the contracts and prints errors/warnings - * @param resolver Resolver - * @param contractsDir Contracts directory - * @param workspaceDir Workspace directory * @param solcVersion Version of a solc compiler - * @param dependencyNameToPackagePath Mapping of dependency name to it's package path * @param standardInput Solidity standard JSON input */ -export function compileDocker( - resolver: Resolver, - contractsDir: string, - workspaceDir: string, +export async function compileDockerAsync( solcVersion: string, - dependencyNameToPackagePath: { [dependencyName: string]: string }, standardInput: solc.StandardInput, -): solc.StandardOutput { - const standardInputDocker = _.cloneDeep(standardInput); - standardInputDocker.settings.remappings = _.map( - dependencyNameToPackagePath, - (dependencyPackagePath: string, dependencyName: string) => `${dependencyName}=${dependencyPackagePath}`, - ); - standardInputDocker.sources = _.mapKeys( - standardInputDocker.sources, - (_source: solc.Source, sourcePath: string) => resolver.resolve(sourcePath).absolutePath, - ); - - const standardInputStrDocker = JSON.stringify(standardInputDocker, null, 2); +): Promise { + const standardInputStr = JSON.stringify(standardInput, null, 2); const dockerCommand = - `docker run -i -a stdin -a stdout -a stderr -v ${workspaceDir}:${workspaceDir} ethereum/solc:${solcVersion} ` + - `solc --standard-json --allow-paths ${workspaceDir}`; - const standardOutputStrDocker = execSync(dockerCommand, { input: standardInputStrDocker }).toString(); - const compiledDocker: solc.StandardOutput = JSON.parse(standardOutputStrDocker); - - if (!_.isUndefined(compiledDocker.errors)) { - printCompilationErrorsAndWarnings(compiledDocker.errors); - } - - compiledDocker.sources = makeContractPathsRelative( - compiledDocker.sources, - contractsDir, - dependencyNameToPackagePath, - ); - compiledDocker.contracts = makeContractPathsRelative( - compiledDocker.contracts, - contractsDir, - dependencyNameToPackagePath, - ); - return compiledDocker; + `docker run -i -a stdin -a stdout -a stderr ethereum/solc:${solcVersion} ` + `solc --standard-json`; + const standardOutputStr = execSync(dockerCommand, { input: standardInputStr }).toString(); + const compiled: solc.StandardOutput = JSON.parse(standardOutputStr); + return compiled; } function makeContractPathRelative( @@ -202,7 +161,13 @@ function makeContractPathRelative( return contractPath; } -function makeContractPathsRelative( +/** + * Makes the path relative removing all system-dependent data. Converts absolute paths to a format suitable for artifacts. + * @param absolutePathToSmth Absolute path to contract or source + * @param contractsDir Current package contracts directory location + * @param dependencyNameToPackagePath Mapping of dependency name to package path + */ +export function makeContractPathsRelative( absolutePathToSmth: { [absoluteContractPath: string]: any }, contractsDir: string, dependencyNameToPackagePath: { [dependencyName: string]: string }, @@ -216,7 +181,7 @@ function makeContractPathsRelative( * Separates errors from warnings, formats the messages and prints them. Throws if there is any compilation error (not warning). * @param solcErrors The errors field of standard JSON output that contains errors and warnings. */ -function printCompilationErrorsAndWarnings(solcErrors: solc.SolcError[]): void { +export function printCompilationErrorsAndWarnings(solcErrors: solc.SolcError[]): void { const SOLIDITY_WARNING = 'warning'; const errors = _.filter(solcErrors, entry => entry.severity !== SOLIDITY_WARNING); const warnings = _.filter(solcErrors, entry => entry.severity === SOLIDITY_WARNING); @@ -392,3 +357,35 @@ export function addHexPrefixToContractBytecode(compiledContract: solc.StandardCo } } } + +/** + * Takes the list of resolved contract sources from `SpyResolver` and produces a mapping from dependency name + * to package path used in `remappings` later. + * @param contractSources The list of resolved contract sources + */ +export function getDependencyNameToPackagePath( + contractSources: ContractSource[], +): { [dependencyName: string]: string } { + const allTouchedFiles = contractSources.map(contractSource => `${contractSource.absolutePath}`); + const NODE_MODULES = 'node_modules'; + const allTouchedDependencies = _.filter(allTouchedFiles, filePath => filePath.includes(NODE_MODULES)); + const dependencyNameToPackagePath: { [dependencyName: string]: string } = {}; + _.map(allTouchedDependencies, dependencyFilePath => { + const lastNodeModulesStart = dependencyFilePath.lastIndexOf(NODE_MODULES); + const lastNodeModulesEnd = lastNodeModulesStart + NODE_MODULES.length; + const importPath = dependencyFilePath.substr(lastNodeModulesEnd + 1); + let packageName; + let packageScopeIfExists; + let dependencyName; + if (_.startsWith(importPath, '@')) { + [packageScopeIfExists, packageName] = importPath.split('/'); + dependencyName = `${packageScopeIfExists}/${packageName}`; + } else { + [packageName] = importPath.split('/'); + dependencyName = `${packageName}`; + } + const dependencyPackagePath = path.join(dependencyFilePath.substr(0, lastNodeModulesEnd), dependencyName); + dependencyNameToPackagePath[dependencyName] = dependencyPackagePath; + }); + return dependencyNameToPackagePath; +} diff --git a/packages/sol-resolver/src/resolvers/npm_resolver.ts b/packages/sol-resolver/src/resolvers/npm_resolver.ts index d9377989f..3c1d09557 100644 --- a/packages/sol-resolver/src/resolvers/npm_resolver.ts +++ b/packages/sol-resolver/src/resolvers/npm_resolver.ts @@ -8,11 +8,9 @@ import { Resolver } from './resolver'; export class NPMResolver extends Resolver { private readonly _packagePath: string; - private readonly _workspacePath: string; - constructor(packagePath: string, workspacePath: string = '/') { + constructor(packagePath: string) { super(); this._packagePath = packagePath; - this._workspacePath = workspacePath; } public resolveIfExists(importPath: string): ContractSource | undefined { if (!importPath.startsWith('/')) { @@ -25,11 +23,9 @@ export class NPMResolver extends Resolver { [packageName, ...other] = importPath.split('/'); } const pathWithinPackage = path.join(...other); - for ( - let currentPath = this._packagePath; - currentPath.includes(this._workspacePath); - currentPath = path.dirname(currentPath) - ) { + let currentPath = this._packagePath; + const ROOT_PATH = '/'; + while (currentPath !== ROOT_PATH) { const packagePath = _.isUndefined(packageScopeIfExists) ? packageName : path.join(packageScopeIfExists, packageName); @@ -38,6 +34,7 @@ export class NPMResolver extends Resolver { const fileContent = fs.readFileSync(lookupPath).toString(); return { source: fileContent, path: importPath, absolutePath: lookupPath }; } + currentPath = path.dirname(currentPath); } } return undefined; diff --git a/packages/typescript-typings/types/solc/index.d.ts b/packages/typescript-typings/types/solc/index.d.ts index f4c05cd7c..fefad9f6a 100644 --- a/packages/typescript-typings/types/solc/index.d.ts +++ b/packages/typescript-typings/types/solc/index.d.ts @@ -95,7 +95,7 @@ declare module 'solc' { optimizerEnabled: number, findImports: (importPath: string) => ImportContents, ): CompilationResult; - compileStandardWrapper(input: string, findImports: (importPath: string) => ImportContents): string; + compileStandardWrapper(input: string, findImports?: (importPath: string) => ImportContents): string; } export function loadRemoteVersion( versionName: string, -- cgit v1.2.3