From e8d68dc07faa1c8daa59b2a3f1980328b2b49017 Mon Sep 17 00:00:00 2001 From: Leonid Logvinov Date: Wed, 23 Jan 2019 16:54:27 +0100 Subject: Implement docker as another backend for sol-compiler --- contracts/examples/compiler.json | 2 + contracts/extensions/compiler.json | 2 + contracts/interfaces/compiler.json | 2 + contracts/libs/compiler.json | 2 + contracts/multisig/compiler.json | 2 + contracts/protocol/compiler.json | 2 + contracts/tokens/compiler.json | 2 + contracts/utils/compiler.json | 2 + packages/ethereum-types/src/index.ts | 6 +- packages/sol-compiler/src/compiler.ts | 76 ++++++++++++++++--- .../src/schemas/compiler_options_schema.ts | 2 + packages/sol-compiler/src/utils/compiler.ts | 85 ++++++++++++++++++++-- packages/sol-resolver/src/resolvers/fs_resolver.ts | 4 +- .../sol-resolver/src/resolvers/npm_resolver.ts | 13 ++-- 14 files changed, 179 insertions(+), 23 deletions(-) diff --git a/contracts/examples/compiler.json b/contracts/examples/compiler.json index 375fa0c55..868a11905 100644 --- a/contracts/examples/compiler.json +++ b/contracts/examples/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/extensions/compiler.json b/contracts/extensions/compiler.json index 2bb468724..a5c7bcc21 100644 --- a/contracts/extensions/compiler.json +++ b/contracts/extensions/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/interfaces/compiler.json b/contracts/interfaces/compiler.json index 38a232541..496bc2327 100644 --- a/contracts/interfaces/compiler.json +++ b/contracts/interfaces/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/libs/compiler.json b/contracts/libs/compiler.json index 349d3063b..09ce91442 100644 --- a/contracts/libs/compiler.json +++ b/contracts/libs/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/multisig/compiler.json b/contracts/multisig/compiler.json index 5a1f689e2..9555fbbfd 100644 --- a/contracts/multisig/compiler.json +++ b/contracts/multisig/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/protocol/compiler.json b/contracts/protocol/compiler.json index 10e5bb0a1..4fc3712b0 100644 --- a/contracts/protocol/compiler.json +++ b/contracts/protocol/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/tokens/compiler.json b/contracts/tokens/compiler.json index 498c5d826..12e603cab 100644 --- a/contracts/tokens/compiler.json +++ b/contracts/tokens/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/contracts/utils/compiler.json b/contracts/utils/compiler.json index 1524c1eaa..df4ef1e9f 100644 --- a/contracts/utils/compiler.json +++ b/contracts/utils/compiler.json @@ -1,6 +1,8 @@ { "artifactsDir": "./generated-artifacts", "contractsDir": "./contracts", + "workspaceDir": "../..", + "useDockerisedSolc": true, "compilerSettings": { "optimizer": { "enabled": true, diff --git a/packages/ethereum-types/src/index.ts b/packages/ethereum-types/src/index.ts index a8dcfd68a..a1f6919a8 100644 --- a/packages/ethereum-types/src/index.ts +++ b/packages/ethereum-types/src/index.ts @@ -503,19 +503,23 @@ export interface Source { /** * Options you can specify (as flags or in a compiler.json file) when invoking sol-compiler - * contractsDir: Directory containing your project's Solidity contracts. Can contain nested directories. + * contractsDir: Directory containing your package's Solidity contracts. Can contain nested directories. + * workspaceDir: Directory containing your project's Solidity contracts. All the contracts used in compilation must be withing it. Similar to --allow-paths in Solidity. * artifactsDir: Directory where you want the generated artifacts.json written to * compilerSettings: Desired settings to pass to the Solidity compiler during compilation. * (http://solidity.readthedocs.io/en/v0.4.24/using-the-compiler.html#compiler-input-and-output-json-description) * contracts: List of contract names you wish to compile, or alternatively ['*'] to compile all contracts in the * specified directory. + * useDockerisedSolc: If set to true - sol-compiler will try using docker to achieve faster compilation times. Otherwise and by default - solcjs will be used. * solcVersion: If you don't want to compile each contract with the Solidity version specified in-file, you can force all * contracts to compile with the the version specified here. */ export interface CompilerOptions { contractsDir?: string; + workspaceDir?: string; artifactsDir?: string; compilerSettings?: CompilerSettings; contracts?: string[] | '*'; + useDockerisedSolc?: boolean; solcVersion?: string; } // tslint:disable-line:max-file-line-count diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts index d38ccbf39..856bcbd48 100644 --- a/packages/sol-compiler/src/compiler.ts +++ b/packages/sol-compiler/src/compiler.ts @@ -10,6 +10,7 @@ import { URLResolver, } from '@0x/sol-resolver'; import { logUtils } from '@0x/utils'; +import { execSync } from 'child_process'; import * as chokidar from 'chokidar'; import { CompilerOptions, ContractArtifact, ContractVersionData, StandardOutput } from 'ethereum-types'; import * as fs from 'fs'; @@ -23,10 +24,11 @@ import { compilerOptionsSchema } from './schemas/compiler_options_schema'; import { binPaths } from './solc/bin_paths'; import { addHexPrefixToContractBytecode, - compile, + compileDocker, + compileSolcJS, createDirIfDoesNotExistAsync, getContractArtifactIfExistsAsync, - getSolcAsync, + getSolcJSAsync, getSourcesWithDependencies, getSourceTreeHash, parseSolidityVersionRange, @@ -40,6 +42,7 @@ const ALL_CONTRACTS_IDENTIFIER = '*'; const ALL_FILES_IDENTIFIER = '*'; const DEFAULT_CONTRACTS_DIR = path.resolve('contracts'); const DEFAULT_ARTIFACTS_DIR = path.resolve('artifacts'); +const DEFAULT_USE_DOCKERISED_SOLC = false; // Solc compiler settings cannot be configured from the commandline. // If you need this configured, please create a `compiler.json` config file // with your desired configurations. @@ -80,10 +83,12 @@ export class Compiler { private readonly _resolver: Resolver; private readonly _nameResolver: NameResolver; private readonly _contractsDir: string; + private readonly _workspaceDir: string; private readonly _compilerSettings: solc.CompilerSettings; private readonly _artifactsDir: string; private readonly _solcVersionIfExists: string | undefined; private readonly _specifiedContracts: string[] | TYPE_ALL_FILES_IDENTIFIER; + private readonly _useDockerisedSolc: boolean; /** * Instantiates a new instance of the Compiler class. * @param opts Optional compiler options @@ -97,16 +102,23 @@ export class Compiler { : {}; const passedOpts = opts || {}; assert.doesConformToSchema('compiler.json', config, compilerOptionsSchema); - this._contractsDir = passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR; + this._contractsDir = path.resolve(passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR); + this._workspaceDir = path.resolve(passedOpts.workspaceDir || config.workspaceDir || this._contractsDir); + if (!this._contractsDir.includes(this._workspaceDir)) { + throw new Error( + `Contracts dir ${this._contractsDir} is outside of the workspace dir ${this._workspaceDir}`, + ); + } this._solcVersionIfExists = passedOpts.solcVersion || config.solcVersion; this._compilerSettings = passedOpts.compilerSettings || config.compilerSettings || DEFAULT_COMPILER_SETTINGS; this._artifactsDir = passedOpts.artifactsDir || config.artifactsDir || DEFAULT_ARTIFACTS_DIR; this._specifiedContracts = passedOpts.contracts || config.contracts || ALL_CONTRACTS_IDENTIFIER; - this._nameResolver = new NameResolver(path.resolve(this._contractsDir)); + this._useDockerisedSolc = + passedOpts.useDockerisedSolc || config.useDockerisedSolc || DEFAULT_USE_DOCKERISED_SOLC; + this._nameResolver = new NameResolver(this._contractsDir); const resolver = new FallthroughResolver(); resolver.appendResolver(new URLResolver()); - const packagePath = path.resolve(''); - resolver.appendResolver(new NPMResolver(packagePath)); + resolver.appendResolver(new NPMResolver(this._contractsDir, this._workspaceDir)); resolver.appendResolver(new RelativeFSResolver(this._contractsDir)); resolver.appendResolver(new FSResolver()); resolver.appendResolver(this._nameResolver); @@ -205,11 +217,12 @@ export class Compiler { // map contract paths to data about them for later verification and persistence const contractPathToData: ContractPathToData = {}; + const spyResolver = new SpyResolver(this._resolver); for (const contractName of contractNames) { - const contractSource = this._resolver.resolve(contractName); + const contractSource = spyResolver.resolve(contractName); const sourceTreeHashHex = getSourceTreeHash( - this._resolver, + spyResolver, path.join(this._contractsDir, contractSource.path), ).toString('hex'); const contractData = { @@ -242,6 +255,30 @@ export class Compiler { versionToInputs[solcVersion].contractsToCompile.push(contractSource.path); } + const allTouchedFiles = spyResolver.resolvedContractSources.map( + contractSource => `${contractSource.absolutePath}`, + ); + const NODE_MODULES = 'node_modules'; + const allTouchedDependencies = _.filter(allTouchedFiles, filePath => filePath.includes(NODE_MODULES)); + const dependencyNameToPackagePath: { [dependencyName: string]: string } = {}; + _.map(allTouchedDependencies, dependencyFilePath => { + const lastNodeModulesStart = dependencyFilePath.lastIndexOf(NODE_MODULES); + const lastNodeModulesEnd = lastNodeModulesStart + NODE_MODULES.length; + const importPath = dependencyFilePath.substr(lastNodeModulesEnd + 1); + let packageName; + let packageScopeIfExists; + let dependencyName; + if (_.startsWith(importPath, '@')) { + [packageScopeIfExists, packageName] = importPath.split('/'); + dependencyName = `${packageScopeIfExists}/${packageName}`; + } else { + [packageName] = importPath.split('/'); + dependencyName = `${packageName}`; + } + const dependencyPackagePath = path.join(dependencyFilePath.substr(0, lastNodeModulesEnd), dependencyName); + dependencyNameToPackagePath[dependencyName] = dependencyPackagePath; + }); + const compilerOutputs: StandardOutput[] = []; const solcVersions = _.keys(versionToInputs); @@ -252,12 +289,31 @@ export class Compiler { input.contractsToCompile }) with Solidity v${solcVersion}...`, ); + let compilerOutput; + let fullSolcVersion; + if (this._useDockerisedSolc) { + const dockerCommand = `docker run ethereum/solc:${solcVersion} --version`; + const versionCommandOutput = execSync(dockerCommand).toString(); + const versionCommandOutputParts = versionCommandOutput.split(' '); + fullSolcVersion = versionCommandOutputParts[versionCommandOutputParts.length - 1].trim(); + compilerOutput = compileDocker( + this._resolver, + this._contractsDir, + this._workspaceDir, + solcVersion, + dependencyNameToPackagePath, + input.standardInput, + ); + } else { + fullSolcVersion = binPaths[solcVersion]; + const solcInstance = await getSolcJSAsync(solcVersion); + compilerOutput = compileSolcJS(this._resolver, solcInstance, input.standardInput); + } - const { solcInstance, fullSolcVersion } = await getSolcAsync(solcVersion); - const compilerOutput = compile(this._resolver, solcInstance, input.standardInput); compilerOutputs.push(compilerOutput); for (const contractPath of input.contractsToCompile) { + // console.log('contractsPath', contractPath); const contractName = contractPathToData[contractPath].contractName; const compiledContract = compilerOutput.contracts[contractPath][contractName]; diff --git a/packages/sol-compiler/src/schemas/compiler_options_schema.ts b/packages/sol-compiler/src/schemas/compiler_options_schema.ts index d4d1b6017..657b801ad 100644 --- a/packages/sol-compiler/src/schemas/compiler_options_schema.ts +++ b/packages/sol-compiler/src/schemas/compiler_options_schema.ts @@ -2,6 +2,7 @@ export const compilerOptionsSchema = { id: '/CompilerOptions', properties: { contractsDir: { type: 'string' }, + workspaceDir: { type: 'string' }, artifactsDir: { type: 'string' }, solcVersion: { type: 'string', pattern: '^\\d+.\\d+.\\d+$' }, compilerSettings: { type: 'object' }, @@ -19,6 +20,7 @@ export const compilerOptionsSchema = { }, ], }, + useDockerisedSolc: { type: 'boolean' }, }, type: 'object', required: [], diff --git a/packages/sol-compiler/src/utils/compiler.ts b/packages/sol-compiler/src/utils/compiler.ts index db308f2b5..669bdd7a4 100644 --- a/packages/sol-compiler/src/utils/compiler.ts +++ b/packages/sol-compiler/src/utils/compiler.ts @@ -1,6 +1,7 @@ import { ContractSource, Resolver } from '@0x/sol-resolver'; import { fetchAsync, logUtils } from '@0x/utils'; import chalk from 'chalk'; +import { execSync } from 'child_process'; import { ContractArtifact } from 'ethereum-types'; import * as ethUtil from 'ethereumjs-util'; import * as _ from 'lodash'; @@ -121,7 +122,7 @@ export function parseDependencies(contractSource: ContractSource): string[] { * @param solcInstance Instance of a solc compiler * @param standardInput Solidity standard JSON input */ -export function compile( +export function compileSolcJS( resolver: Resolver, solcInstance: solc.SolcInstance, standardInput: solc.StandardInput, @@ -137,6 +138,80 @@ export function compile( } return compiled; } + +/** + * Compiles the contracts and prints errors/warnings + * @param resolver Resolver + * @param contractsDir Contracts directory + * @param workspaceDir Workspace directory + * @param solcVersion Version of a solc compiler + * @param dependencyNameToPackagePath Mapping of dependency name to it's package path + * @param standardInput Solidity standard JSON input + */ +export function compileDocker( + resolver: Resolver, + contractsDir: string, + workspaceDir: string, + solcVersion: string, + dependencyNameToPackagePath: { [dependencyName: string]: string }, + standardInput: solc.StandardInput, +): solc.StandardOutput { + const standardInputDocker = _.cloneDeep(standardInput); + standardInputDocker.settings.remappings = _.map( + dependencyNameToPackagePath, + (dependencyPackagePath: string, dependencyName: string) => `${dependencyName}=${dependencyPackagePath}`, + ); + standardInputDocker.sources = _.mapKeys( + standardInputDocker.sources, + (_source: solc.Source, sourcePath: string) => resolver.resolve(sourcePath).absolutePath, + ); + + const standardInputStrDocker = JSON.stringify(standardInputDocker, null, 2); + const dockerCommand = + `docker run -i -a stdin -a stdout -a stderr -v ${workspaceDir}:${workspaceDir} ethereum/solc:${solcVersion} ` + + `solc --standard-json --allow-paths ${workspaceDir}`; + const standardOutputStrDocker = execSync(dockerCommand, { input: standardInputStrDocker }).toString(); + const compiledDocker: solc.StandardOutput = JSON.parse(standardOutputStrDocker); + + if (!_.isUndefined(compiledDocker.errors)) { + printCompilationErrorsAndWarnings(compiledDocker.errors); + } + + compiledDocker.sources = makeContractPathsRelative( + compiledDocker.sources, + contractsDir, + dependencyNameToPackagePath, + ); + compiledDocker.contracts = makeContractPathsRelative( + compiledDocker.contracts, + contractsDir, + dependencyNameToPackagePath, + ); + return compiledDocker; +} + +function makeContractPathRelative( + absolutePath: string, + contractsDir: string, + dependencyNameToPackagePath: { [dependencyName: string]: string }, +): string { + let contractPath = absolutePath.replace(`${contractsDir}/`, ''); + _.map(dependencyNameToPackagePath, (packagePath: string, dependencyName: string) => { + contractPath = contractPath.replace(packagePath, dependencyName); + }); + return contractPath; +} + +function makeContractPathsRelative( + absolutePathToSmth: { [absoluteContractPath: string]: any }, + contractsDir: string, + dependencyNameToPackagePath: { [dependencyName: string]: string }, +): { [contractPath: string]: any } { + return _.mapKeys(absolutePathToSmth, (_val: any, absoluteContractPath: string) => + makeContractPathRelative(absoluteContractPath, contractsDir, dependencyNameToPackagePath), + ); +} + /** * Separates errors from warnings, formats the messages and prints them. Throws if there is any compilation error (not warning). * @param solcErrors The errors field of standard JSON output that contains errors and warnings. @@ -267,13 +342,11 @@ function recursivelyGatherDependencySources( } /** - * Gets the solidity compiler instance and full version name. If the compiler is already cached - gets it from FS, + * Gets the solidity compiler instance. If the compiler is already cached - gets it from FS, * otherwise - fetches it and caches it. * @param solcVersion The compiler version. e.g. 0.5.0 */ -export async function getSolcAsync( - solcVersion: string, -): Promise<{ solcInstance: solc.SolcInstance; fullSolcVersion: string }> { +export async function getSolcJSAsync(solcVersion: string): Promise { const fullSolcVersion = binPaths[solcVersion]; if (_.isUndefined(fullSolcVersion)) { throw new Error(`${solcVersion} is not a known compiler version`); @@ -297,7 +370,7 @@ export async function getSolcAsync( throw new Error('No compiler available'); } const solcInstance = solc.setupMethods(requireFromString(solcjs, compilerBinFilename)); - return { solcInstance, fullSolcVersion }; + return solcInstance; } /** diff --git a/packages/sol-resolver/src/resolvers/fs_resolver.ts b/packages/sol-resolver/src/resolvers/fs_resolver.ts index 86128023d..248fa405e 100644 --- a/packages/sol-resolver/src/resolvers/fs_resolver.ts +++ b/packages/sol-resolver/src/resolvers/fs_resolver.ts @@ -1,4 +1,5 @@ import * as fs from 'fs'; +import * as path from 'path'; import { ContractSource } from '../types'; @@ -9,7 +10,8 @@ export class FSResolver extends Resolver { public resolveIfExists(importPath: string): ContractSource | undefined { if (fs.existsSync(importPath) && fs.lstatSync(importPath).isFile()) { const fileContent = fs.readFileSync(importPath).toString(); - return { source: fileContent, path: importPath, absolutePath: importPath }; + const absolutePath = path.resolve(importPath); + return { source: fileContent, path: importPath, absolutePath } as any; } return undefined; } diff --git a/packages/sol-resolver/src/resolvers/npm_resolver.ts b/packages/sol-resolver/src/resolvers/npm_resolver.ts index 3c1d09557..1c9e56957 100644 --- a/packages/sol-resolver/src/resolvers/npm_resolver.ts +++ b/packages/sol-resolver/src/resolvers/npm_resolver.ts @@ -8,9 +8,11 @@ import { Resolver } from './resolver'; export class NPMResolver extends Resolver { private readonly _packagePath: string; - constructor(packagePath: string) { + private readonly _workspacePath: string; + constructor(packagePath: string, workspacePath: string) { super(); this._packagePath = packagePath; + this._workspacePath = workspacePath; } public resolveIfExists(importPath: string): ContractSource | undefined { if (!importPath.startsWith('/')) { @@ -23,9 +25,11 @@ export class NPMResolver extends Resolver { [packageName, ...other] = importPath.split('/'); } const pathWithinPackage = path.join(...other); - let currentPath = this._packagePath; - const ROOT_PATH = '/'; - while (currentPath !== ROOT_PATH) { + for ( + let currentPath = this._packagePath; + currentPath.includes(this._workspacePath); + currentPath = path.dirname(currentPath) + ) { const packagePath = _.isUndefined(packageScopeIfExists) ? packageName : path.join(packageScopeIfExists, packageName); @@ -34,7 +38,6 @@ export class NPMResolver extends Resolver { const fileContent = fs.readFileSync(lookupPath).toString(); return { source: fileContent, path: importPath, absolutePath: lookupPath }; } - currentPath = path.dirname(currentPath); } } return undefined; -- cgit v1.2.3