aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLeonid Logvinov <logvinov.leon@gmail.com>2019-01-23 23:54:27 +0800
committerLeonid Logvinov <logvinov.leon@gmail.com>2019-01-23 23:54:27 +0800
commite8d68dc07faa1c8daa59b2a3f1980328b2b49017 (patch)
treedbbcae9fbaeddb329556b78bba6ac2f9f1bbcc74
parentef928aff31f09b368307548bd5a512b4212a0c95 (diff)
downloaddexon-0x-contracts-e8d68dc07faa1c8daa59b2a3f1980328b2b49017.tar
dexon-0x-contracts-e8d68dc07faa1c8daa59b2a3f1980328b2b49017.tar.gz
dexon-0x-contracts-e8d68dc07faa1c8daa59b2a3f1980328b2b49017.tar.bz2
dexon-0x-contracts-e8d68dc07faa1c8daa59b2a3f1980328b2b49017.tar.lz
dexon-0x-contracts-e8d68dc07faa1c8daa59b2a3f1980328b2b49017.tar.xz
dexon-0x-contracts-e8d68dc07faa1c8daa59b2a3f1980328b2b49017.tar.zst
dexon-0x-contracts-e8d68dc07faa1c8daa59b2a3f1980328b2b49017.zip
Implement docker as another backend for sol-compiler
-rw-r--r--contracts/examples/compiler.json2
-rw-r--r--contracts/extensions/compiler.json2
-rw-r--r--contracts/interfaces/compiler.json2
-rw-r--r--contracts/libs/compiler.json2
-rw-r--r--contracts/multisig/compiler.json2
-rw-r--r--contracts/protocol/compiler.json2
-rw-r--r--contracts/tokens/compiler.json2
-rw-r--r--contracts/utils/compiler.json2
-rw-r--r--packages/ethereum-types/src/index.ts6
-rw-r--r--packages/sol-compiler/src/compiler.ts76
-rw-r--r--packages/sol-compiler/src/schemas/compiler_options_schema.ts2
-rw-r--r--packages/sol-compiler/src/utils/compiler.ts85
-rw-r--r--packages/sol-resolver/src/resolvers/fs_resolver.ts4
-rw-r--r--packages/sol-resolver/src/resolvers/npm_resolver.ts13
14 files changed, 179 insertions, 23 deletions
diff --git a/contracts/examples/compiler.json b/contracts/examples/compiler.json
index 375fa0c55..868a11905 100644
--- a/contracts/examples/compiler.json
+++ b/contracts/examples/compiler.json
@@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
+ "workspaceDir": "../..",
+ "useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,
diff --git a/contracts/extensions/compiler.json b/contracts/extensions/compiler.json
index 2bb468724..a5c7bcc21 100644
--- a/contracts/extensions/compiler.json
+++ b/contracts/extensions/compiler.json
@@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
+ "workspaceDir": "../..",
+ "useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,
diff --git a/contracts/interfaces/compiler.json b/contracts/interfaces/compiler.json
index 38a232541..496bc2327 100644
--- a/contracts/interfaces/compiler.json
+++ b/contracts/interfaces/compiler.json
@@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
+ "workspaceDir": "../..",
+ "useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,
diff --git a/contracts/libs/compiler.json b/contracts/libs/compiler.json
index 349d3063b..09ce91442 100644
--- a/contracts/libs/compiler.json
+++ b/contracts/libs/compiler.json
@@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
+ "workspaceDir": "../..",
+ "useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,
diff --git a/contracts/multisig/compiler.json b/contracts/multisig/compiler.json
index 5a1f689e2..9555fbbfd 100644
--- a/contracts/multisig/compiler.json
+++ b/contracts/multisig/compiler.json
@@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
+ "workspaceDir": "../..",
+ "useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,
diff --git a/contracts/protocol/compiler.json b/contracts/protocol/compiler.json
index 10e5bb0a1..4fc3712b0 100644
--- a/contracts/protocol/compiler.json
+++ b/contracts/protocol/compiler.json
@@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
+ "workspaceDir": "../..",
+ "useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,
diff --git a/contracts/tokens/compiler.json b/contracts/tokens/compiler.json
index 498c5d826..12e603cab 100644
--- a/contracts/tokens/compiler.json
+++ b/contracts/tokens/compiler.json
@@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
+ "workspaceDir": "../..",
+ "useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,
diff --git a/contracts/utils/compiler.json b/contracts/utils/compiler.json
index 1524c1eaa..df4ef1e9f 100644
--- a/contracts/utils/compiler.json
+++ b/contracts/utils/compiler.json
@@ -1,6 +1,8 @@
{
"artifactsDir": "./generated-artifacts",
"contractsDir": "./contracts",
+ "workspaceDir": "../..",
+ "useDockerisedSolc": true,
"compilerSettings": {
"optimizer": {
"enabled": true,
diff --git a/packages/ethereum-types/src/index.ts b/packages/ethereum-types/src/index.ts
index a8dcfd68a..a1f6919a8 100644
--- a/packages/ethereum-types/src/index.ts
+++ b/packages/ethereum-types/src/index.ts
@@ -503,19 +503,23 @@ export interface Source {
/**
* Options you can specify (as flags or in a compiler.json file) when invoking sol-compiler
- * contractsDir: Directory containing your project's Solidity contracts. Can contain nested directories.
+ * contractsDir: Directory containing your package's Solidity contracts. Can contain nested directories.
+ * workspaceDir: Directory containing your project's Solidity contracts. All the contracts used in compilation must be withing it. Similar to --allow-paths in Solidity.
* artifactsDir: Directory where you want the generated artifacts.json written to
* compilerSettings: Desired settings to pass to the Solidity compiler during compilation.
* (http://solidity.readthedocs.io/en/v0.4.24/using-the-compiler.html#compiler-input-and-output-json-description)
* contracts: List of contract names you wish to compile, or alternatively ['*'] to compile all contracts in the
* specified directory.
+ * useDockerisedSolc: If set to true - sol-compiler will try using docker to achieve faster compilation times. Otherwise and by default - solcjs will be used.
* solcVersion: If you don't want to compile each contract with the Solidity version specified in-file, you can force all
* contracts to compile with the the version specified here.
*/
export interface CompilerOptions {
contractsDir?: string;
+ workspaceDir?: string;
artifactsDir?: string;
compilerSettings?: CompilerSettings;
contracts?: string[] | '*';
+ useDockerisedSolc?: boolean;
solcVersion?: string;
} // tslint:disable-line:max-file-line-count
diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts
index d38ccbf39..856bcbd48 100644
--- a/packages/sol-compiler/src/compiler.ts
+++ b/packages/sol-compiler/src/compiler.ts
@@ -10,6 +10,7 @@ import {
URLResolver,
} from '@0x/sol-resolver';
import { logUtils } from '@0x/utils';
+import { execSync } from 'child_process';
import * as chokidar from 'chokidar';
import { CompilerOptions, ContractArtifact, ContractVersionData, StandardOutput } from 'ethereum-types';
import * as fs from 'fs';
@@ -23,10 +24,11 @@ import { compilerOptionsSchema } from './schemas/compiler_options_schema';
import { binPaths } from './solc/bin_paths';
import {
addHexPrefixToContractBytecode,
- compile,
+ compileDocker,
+ compileSolcJS,
createDirIfDoesNotExistAsync,
getContractArtifactIfExistsAsync,
- getSolcAsync,
+ getSolcJSAsync,
getSourcesWithDependencies,
getSourceTreeHash,
parseSolidityVersionRange,
@@ -40,6 +42,7 @@ const ALL_CONTRACTS_IDENTIFIER = '*';
const ALL_FILES_IDENTIFIER = '*';
const DEFAULT_CONTRACTS_DIR = path.resolve('contracts');
const DEFAULT_ARTIFACTS_DIR = path.resolve('artifacts');
+const DEFAULT_USE_DOCKERISED_SOLC = false;
// Solc compiler settings cannot be configured from the commandline.
// If you need this configured, please create a `compiler.json` config file
// with your desired configurations.
@@ -80,10 +83,12 @@ export class Compiler {
private readonly _resolver: Resolver;
private readonly _nameResolver: NameResolver;
private readonly _contractsDir: string;
+ private readonly _workspaceDir: string;
private readonly _compilerSettings: solc.CompilerSettings;
private readonly _artifactsDir: string;
private readonly _solcVersionIfExists: string | undefined;
private readonly _specifiedContracts: string[] | TYPE_ALL_FILES_IDENTIFIER;
+ private readonly _useDockerisedSolc: boolean;
/**
* Instantiates a new instance of the Compiler class.
* @param opts Optional compiler options
@@ -97,16 +102,23 @@ export class Compiler {
: {};
const passedOpts = opts || {};
assert.doesConformToSchema('compiler.json', config, compilerOptionsSchema);
- this._contractsDir = passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR;
+ this._contractsDir = path.resolve(passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR);
+ this._workspaceDir = path.resolve(passedOpts.workspaceDir || config.workspaceDir || this._contractsDir);
+ if (!this._contractsDir.includes(this._workspaceDir)) {
+ throw new Error(
+ `Contracts dir ${this._contractsDir} is outside of the workspace dir ${this._workspaceDir}`,
+ );
+ }
this._solcVersionIfExists = passedOpts.solcVersion || config.solcVersion;
this._compilerSettings = passedOpts.compilerSettings || config.compilerSettings || DEFAULT_COMPILER_SETTINGS;
this._artifactsDir = passedOpts.artifactsDir || config.artifactsDir || DEFAULT_ARTIFACTS_DIR;
this._specifiedContracts = passedOpts.contracts || config.contracts || ALL_CONTRACTS_IDENTIFIER;
- this._nameResolver = new NameResolver(path.resolve(this._contractsDir));
+ this._useDockerisedSolc =
+ passedOpts.useDockerisedSolc || config.useDockerisedSolc || DEFAULT_USE_DOCKERISED_SOLC;
+ this._nameResolver = new NameResolver(this._contractsDir);
const resolver = new FallthroughResolver();
resolver.appendResolver(new URLResolver());
- const packagePath = path.resolve('');
- resolver.appendResolver(new NPMResolver(packagePath));
+ resolver.appendResolver(new NPMResolver(this._contractsDir, this._workspaceDir));
resolver.appendResolver(new RelativeFSResolver(this._contractsDir));
resolver.appendResolver(new FSResolver());
resolver.appendResolver(this._nameResolver);
@@ -205,11 +217,12 @@ export class Compiler {
// map contract paths to data about them for later verification and persistence
const contractPathToData: ContractPathToData = {};
+ const spyResolver = new SpyResolver(this._resolver);
for (const contractName of contractNames) {
- const contractSource = this._resolver.resolve(contractName);
+ const contractSource = spyResolver.resolve(contractName);
const sourceTreeHashHex = getSourceTreeHash(
- this._resolver,
+ spyResolver,
path.join(this._contractsDir, contractSource.path),
).toString('hex');
const contractData = {
@@ -242,6 +255,30 @@ export class Compiler {
versionToInputs[solcVersion].contractsToCompile.push(contractSource.path);
}
+ const allTouchedFiles = spyResolver.resolvedContractSources.map(
+ contractSource => `${contractSource.absolutePath}`,
+ );
+ const NODE_MODULES = 'node_modules';
+ const allTouchedDependencies = _.filter(allTouchedFiles, filePath => filePath.includes(NODE_MODULES));
+ const dependencyNameToPackagePath: { [dependencyName: string]: string } = {};
+ _.map(allTouchedDependencies, dependencyFilePath => {
+ const lastNodeModulesStart = dependencyFilePath.lastIndexOf(NODE_MODULES);
+ const lastNodeModulesEnd = lastNodeModulesStart + NODE_MODULES.length;
+ const importPath = dependencyFilePath.substr(lastNodeModulesEnd + 1);
+ let packageName;
+ let packageScopeIfExists;
+ let dependencyName;
+ if (_.startsWith(importPath, '@')) {
+ [packageScopeIfExists, packageName] = importPath.split('/');
+ dependencyName = `${packageScopeIfExists}/${packageName}`;
+ } else {
+ [packageName] = importPath.split('/');
+ dependencyName = `${packageName}`;
+ }
+ const dependencyPackagePath = path.join(dependencyFilePath.substr(0, lastNodeModulesEnd), dependencyName);
+ dependencyNameToPackagePath[dependencyName] = dependencyPackagePath;
+ });
+
const compilerOutputs: StandardOutput[] = [];
const solcVersions = _.keys(versionToInputs);
@@ -252,12 +289,31 @@ export class Compiler {
input.contractsToCompile
}) with Solidity v${solcVersion}...`,
);
+ let compilerOutput;
+ let fullSolcVersion;
+ if (this._useDockerisedSolc) {
+ const dockerCommand = `docker run ethereum/solc:${solcVersion} --version`;
+ const versionCommandOutput = execSync(dockerCommand).toString();
+ const versionCommandOutputParts = versionCommandOutput.split(' ');
+ fullSolcVersion = versionCommandOutputParts[versionCommandOutputParts.length - 1].trim();
+ compilerOutput = compileDocker(
+ this._resolver,
+ this._contractsDir,
+ this._workspaceDir,
+ solcVersion,
+ dependencyNameToPackagePath,
+ input.standardInput,
+ );
+ } else {
+ fullSolcVersion = binPaths[solcVersion];
+ const solcInstance = await getSolcJSAsync(solcVersion);
+ compilerOutput = compileSolcJS(this._resolver, solcInstance, input.standardInput);
+ }
- const { solcInstance, fullSolcVersion } = await getSolcAsync(solcVersion);
- const compilerOutput = compile(this._resolver, solcInstance, input.standardInput);
compilerOutputs.push(compilerOutput);
for (const contractPath of input.contractsToCompile) {
+ // console.log('contractsPath', contractPath);
const contractName = contractPathToData[contractPath].contractName;
const compiledContract = compilerOutput.contracts[contractPath][contractName];
diff --git a/packages/sol-compiler/src/schemas/compiler_options_schema.ts b/packages/sol-compiler/src/schemas/compiler_options_schema.ts
index d4d1b6017..657b801ad 100644
--- a/packages/sol-compiler/src/schemas/compiler_options_schema.ts
+++ b/packages/sol-compiler/src/schemas/compiler_options_schema.ts
@@ -2,6 +2,7 @@ export const compilerOptionsSchema = {
id: '/CompilerOptions',
properties: {
contractsDir: { type: 'string' },
+ workspaceDir: { type: 'string' },
artifactsDir: { type: 'string' },
solcVersion: { type: 'string', pattern: '^\\d+.\\d+.\\d+$' },
compilerSettings: { type: 'object' },
@@ -19,6 +20,7 @@ export const compilerOptionsSchema = {
},
],
},
+ useDockerisedSolc: { type: 'boolean' },
},
type: 'object',
required: [],
diff --git a/packages/sol-compiler/src/utils/compiler.ts b/packages/sol-compiler/src/utils/compiler.ts
index db308f2b5..669bdd7a4 100644
--- a/packages/sol-compiler/src/utils/compiler.ts
+++ b/packages/sol-compiler/src/utils/compiler.ts
@@ -1,6 +1,7 @@
import { ContractSource, Resolver } from '@0x/sol-resolver';
import { fetchAsync, logUtils } from '@0x/utils';
import chalk from 'chalk';
+import { execSync } from 'child_process';
import { ContractArtifact } from 'ethereum-types';
import * as ethUtil from 'ethereumjs-util';
import * as _ from 'lodash';
@@ -121,7 +122,7 @@ export function parseDependencies(contractSource: ContractSource): string[] {
* @param solcInstance Instance of a solc compiler
* @param standardInput Solidity standard JSON input
*/
-export function compile(
+export function compileSolcJS(
resolver: Resolver,
solcInstance: solc.SolcInstance,
standardInput: solc.StandardInput,
@@ -137,6 +138,80 @@ export function compile(
}
return compiled;
}
+
+/**
+ * Compiles the contracts and prints errors/warnings
+ * @param resolver Resolver
+ * @param contractsDir Contracts directory
+ * @param workspaceDir Workspace directory
+ * @param solcVersion Version of a solc compiler
+ * @param dependencyNameToPackagePath Mapping of dependency name to it's package path
+ * @param standardInput Solidity standard JSON input
+ */
+export function compileDocker(
+ resolver: Resolver,
+ contractsDir: string,
+ workspaceDir: string,
+ solcVersion: string,
+ dependencyNameToPackagePath: { [dependencyName: string]: string },
+ standardInput: solc.StandardInput,
+): solc.StandardOutput {
+ const standardInputDocker = _.cloneDeep(standardInput);
+ standardInputDocker.settings.remappings = _.map(
+ dependencyNameToPackagePath,
+ (dependencyPackagePath: string, dependencyName: string) => `${dependencyName}=${dependencyPackagePath}`,
+ );
+ standardInputDocker.sources = _.mapKeys(
+ standardInputDocker.sources,
+ (_source: solc.Source, sourcePath: string) => resolver.resolve(sourcePath).absolutePath,
+ );
+
+ const standardInputStrDocker = JSON.stringify(standardInputDocker, null, 2);
+ const dockerCommand =
+ `docker run -i -a stdin -a stdout -a stderr -v ${workspaceDir}:${workspaceDir} ethereum/solc:${solcVersion} ` +
+ `solc --standard-json --allow-paths ${workspaceDir}`;
+ const standardOutputStrDocker = execSync(dockerCommand, { input: standardInputStrDocker }).toString();
+ const compiledDocker: solc.StandardOutput = JSON.parse(standardOutputStrDocker);
+
+ if (!_.isUndefined(compiledDocker.errors)) {
+ printCompilationErrorsAndWarnings(compiledDocker.errors);
+ }
+
+ compiledDocker.sources = makeContractPathsRelative(
+ compiledDocker.sources,
+ contractsDir,
+ dependencyNameToPackagePath,
+ );
+ compiledDocker.contracts = makeContractPathsRelative(
+ compiledDocker.contracts,
+ contractsDir,
+ dependencyNameToPackagePath,
+ );
+ return compiledDocker;
+}
+
+function makeContractPathRelative(
+ absolutePath: string,
+ contractsDir: string,
+ dependencyNameToPackagePath: { [dependencyName: string]: string },
+): string {
+ let contractPath = absolutePath.replace(`${contractsDir}/`, '');
+ _.map(dependencyNameToPackagePath, (packagePath: string, dependencyName: string) => {
+ contractPath = contractPath.replace(packagePath, dependencyName);
+ });
+ return contractPath;
+}
+
+function makeContractPathsRelative(
+ absolutePathToSmth: { [absoluteContractPath: string]: any },
+ contractsDir: string,
+ dependencyNameToPackagePath: { [dependencyName: string]: string },
+): { [contractPath: string]: any } {
+ return _.mapKeys(absolutePathToSmth, (_val: any, absoluteContractPath: string) =>
+ makeContractPathRelative(absoluteContractPath, contractsDir, dependencyNameToPackagePath),
+ );
+}
+
/**
* Separates errors from warnings, formats the messages and prints them. Throws if there is any compilation error (not warning).
* @param solcErrors The errors field of standard JSON output that contains errors and warnings.
@@ -267,13 +342,11 @@ function recursivelyGatherDependencySources(
}
/**
- * Gets the solidity compiler instance and full version name. If the compiler is already cached - gets it from FS,
+ * Gets the solidity compiler instance. If the compiler is already cached - gets it from FS,
* otherwise - fetches it and caches it.
* @param solcVersion The compiler version. e.g. 0.5.0
*/
-export async function getSolcAsync(
- solcVersion: string,
-): Promise<{ solcInstance: solc.SolcInstance; fullSolcVersion: string }> {
+export async function getSolcJSAsync(solcVersion: string): Promise<solc.SolcInstance> {
const fullSolcVersion = binPaths[solcVersion];
if (_.isUndefined(fullSolcVersion)) {
throw new Error(`${solcVersion} is not a known compiler version`);
@@ -297,7 +370,7 @@ export async function getSolcAsync(
throw new Error('No compiler available');
}
const solcInstance = solc.setupMethods(requireFromString(solcjs, compilerBinFilename));
- return { solcInstance, fullSolcVersion };
+ return solcInstance;
}
/**
diff --git a/packages/sol-resolver/src/resolvers/fs_resolver.ts b/packages/sol-resolver/src/resolvers/fs_resolver.ts
index 86128023d..248fa405e 100644
--- a/packages/sol-resolver/src/resolvers/fs_resolver.ts
+++ b/packages/sol-resolver/src/resolvers/fs_resolver.ts
@@ -1,4 +1,5 @@
import * as fs from 'fs';
+import * as path from 'path';
import { ContractSource } from '../types';
@@ -9,7 +10,8 @@ export class FSResolver extends Resolver {
public resolveIfExists(importPath: string): ContractSource | undefined {
if (fs.existsSync(importPath) && fs.lstatSync(importPath).isFile()) {
const fileContent = fs.readFileSync(importPath).toString();
- return { source: fileContent, path: importPath, absolutePath: importPath };
+ const absolutePath = path.resolve(importPath);
+ return { source: fileContent, path: importPath, absolutePath } as any;
}
return undefined;
}
diff --git a/packages/sol-resolver/src/resolvers/npm_resolver.ts b/packages/sol-resolver/src/resolvers/npm_resolver.ts
index 3c1d09557..1c9e56957 100644
--- a/packages/sol-resolver/src/resolvers/npm_resolver.ts
+++ b/packages/sol-resolver/src/resolvers/npm_resolver.ts
@@ -8,9 +8,11 @@ import { Resolver } from './resolver';
export class NPMResolver extends Resolver {
private readonly _packagePath: string;
- constructor(packagePath: string) {
+ private readonly _workspacePath: string;
+ constructor(packagePath: string, workspacePath: string) {
super();
this._packagePath = packagePath;
+ this._workspacePath = workspacePath;
}
public resolveIfExists(importPath: string): ContractSource | undefined {
if (!importPath.startsWith('/')) {
@@ -23,9 +25,11 @@ export class NPMResolver extends Resolver {
[packageName, ...other] = importPath.split('/');
}
const pathWithinPackage = path.join(...other);
- let currentPath = this._packagePath;
- const ROOT_PATH = '/';
- while (currentPath !== ROOT_PATH) {
+ for (
+ let currentPath = this._packagePath;
+ currentPath.includes(this._workspacePath);
+ currentPath = path.dirname(currentPath)
+ ) {
const packagePath = _.isUndefined(packageScopeIfExists)
? packageName
: path.join(packageScopeIfExists, packageName);
@@ -34,7 +38,6 @@ export class NPMResolver extends Resolver {
const fileContent = fs.readFileSync(lookupPath).toString();
return { source: fileContent, path: importPath, absolutePath: lookupPath };
}
- currentPath = path.dirname(currentPath);
}
}
return undefined;