From 3150ab2bcfe6ab66d426916fc1a003ae52799b72 Mon Sep 17 00:00:00 2001 From: chriseth Date: Tue, 7 Jun 2016 19:44:32 +0200 Subject: Allow remappings to change depending on the context. --- docs/layout-of-source-files.rst | 28 ++++++++-- docs/miscellaneous.rst | 6 +- libsolidity/interface/CompilerStack.cpp | 79 +++++++++++++++++++++----- libsolidity/interface/CompilerStack.h | 25 ++++++++- solc/CommandLineInterface.cpp | 98 ++++++++++++--------------------- solc/CommandLineInterface.h | 2 - solc/jsonCompiler.cpp | 19 ++++--- test/libsolidity/Imports.cpp | 22 ++++++++ 8 files changed, 185 insertions(+), 94 deletions(-) diff --git a/docs/layout-of-source-files.rst b/docs/layout-of-source-files.rst index c21e7280..a0170c5a 100644 --- a/docs/layout-of-source-files.rst +++ b/docs/layout-of-source-files.rst @@ -6,6 +6,8 @@ Source files can contain an arbitrary number of contract definitions and include .. index:: source file, ! import +.. _import: + Importing other Source Files ============================ @@ -68,15 +70,20 @@ remappings so that e.g. ``github.com/ethereum/dapp-bin/library`` is remapped to ``/usr/local/dapp-bin/library`` and the compiler will read the files from there. If remapping keys are prefixes of each other, the longest is tried first. This allows for a "fallback-remapping" with e.g. ``""`` maps to -``"/usr/local/include/solidity"``. +``"/usr/local/include/solidity"``. Furthermore, these remappings can +depend on the context, which allows you to configure packages to +import e.g. different versions of a library of the same name. **solc**: -For solc (the commandline compiler), these remappings are provided as ``key=value`` -arguments, where the ``=value`` part is optional (and defaults to key in that +For solc (the commandline compiler), these remappings are provided as +``context:prefix=target`` arguments, where both the ``context:`` and the +``=target`` parts are optional (where target defaults to prefix in that case). All remapping values that are regular files are compiled (including their dependencies). This mechanism is completely backwards-compatible (as long -as no filename contains a =) and thus not a breaking change. +as no filename contains = or :) and thus not a breaking change. All imports +in files in or below the directory ``context`` that import a file that +starts with ``prefix`` are redirected by replacing ``prefix`` by ``target``. So as an example, if you clone ``github.com/ethereum/dapp-bin/`` locally to ``/usr/local/dapp-bin``, you can use @@ -92,6 +99,19 @@ and then run the compiler as solc github.com/ethereum/dapp-bin/=/usr/local/dapp-bin/ source.sol +As a more complex example, suppose you rely on some module that uses a +very old version of dapp-bin. That old version of dapp-bin is checked +out at ``/usr/local/dapp-bin_old``, then you can use + +.. code-block:: bash + + solc module1:github.com/ethereum/dapp-bin/=/usr/local/dapp-bin/ \ + module2:github.com/ethereum/dapp-bin/=/usr/local/dapp-bin_old/ \ + source.sol + +so that all imports in ``module2`` point to the old version but imports +in ``module1`` get the new version. + Note that solc only allows you to include files from certain directories: They have to be in the directory (or subdirectory) of one of the explicitly specified source files or in the directory (or subdirectory) of a remapping diff --git a/docs/miscellaneous.rst b/docs/miscellaneous.rst index 4e61b283..c9a8890f 100644 --- a/docs/miscellaneous.rst +++ b/docs/miscellaneous.rst @@ -108,7 +108,7 @@ Using ``solc --help`` provides you with an explanation of all options. The compi If you only want to compile a single file, you run it as ``solc --bin sourceFile.sol`` and it will print the binary. Before you deploy your contract, activate the optimizer while compiling using ``solc --optimize --bin sourceFile.sol``. If you want to get some of the more advanced output variants of ``solc``, it is probably better to tell it to output everything to separate files using ``solc -o outputDirectory --bin --ast --asm sourceFile.sol``. The commandline compiler will automatically read imported files from the filesystem, but -it is also possible to provide path redirects using ``prefix=path`` in the following way: +it is also possible to provide path redirects using ``context:prefix=path`` in the following way: :: @@ -121,6 +121,10 @@ always matches). ``solc`` will not read files from the filesystem that lie outsi the remapping targets and outside of the directories where explicitly specified source files reside, so things like ``import "/etc/passwd";`` only work if you add ``=/`` as a remapping. +You can restrict remappings to only certain source files by prefixing a context. + +The section on :ref:`import` provides more details on remappings. + If there are multiple matches due to remappings, the one with the longest common prefix is selected. If your contracts use :ref:`libraries `, you will notice that the bytecode contains substrings of the form ``__LibraryName______``. You can use ``solc`` as a linker meaning that it will insert the library addresses for you at those points: diff --git a/libsolidity/interface/CompilerStack.cpp b/libsolidity/interface/CompilerStack.cpp index 83459183..c28e926b 100644 --- a/libsolidity/interface/CompilerStack.cpp +++ b/libsolidity/interface/CompilerStack.cpp @@ -63,6 +63,24 @@ CompilerStack::CompilerStack(bool _addStandardSources, ReadFileCallback const& _ addSources(StandardSources, true); // add them as libraries } +void CompilerStack::setRemappings(vector const& _remappings) +{ + vector remappings; + for (auto const& remapping: _remappings) + { + auto eq = find(remapping.begin(), remapping.end(), '='); + if (eq == remapping.end()) + continue; // ignore + auto colon = find(remapping.begin(), eq, ':'); + Remapping r; + r.context = colon == eq ? string() : string(remapping.begin(), colon); + r.prefix = colon == eq ? string(remapping.begin(), eq) : string(colon + 1, eq); + r.target = string(eq + 1, remapping.end()); + remappings.push_back(r); + } + swap(m_remappings, remappings); +} + void CompilerStack::reset(bool _keepSources, bool _addStandardSources) { m_parseSuccessful = false; @@ -384,37 +402,72 @@ tuple CompilerStack::positionFromSourceLocation(SourceLocati return make_tuple(++startLine, ++startColumn, ++endLine, ++endColumn); } -StringMap CompilerStack::loadMissingSources(SourceUnit const& _ast, std::string const& _path) +StringMap CompilerStack::loadMissingSources(SourceUnit const& _ast, std::string const& _sourcePath) { StringMap newSources; for (auto const& node: _ast.nodes()) if (ImportDirective const* import = dynamic_cast(node.get())) { - string path = absolutePath(import->path(), _path); - import->annotation().absolutePath = path; - if (m_sources.count(path) || newSources.count(path)) + string importPath = absolutePath(import->path(), _sourcePath); + // The current value of `path` is the absolute path as seen from this source file. + // We first have to apply remappings before we can store the actual absolute path + // as seen globally. + importPath = applyRemapping(importPath, _sourcePath); + import->annotation().absolutePath = importPath; + if (m_sources.count(importPath) || newSources.count(importPath)) continue; - string contents; - string errorMessage; - if (!m_readFile) - errorMessage = "File not supplied initially."; + + ReadFileResult result{false, string("File not supplied initially.")}; + if (m_readFile) + result = m_readFile(importPath); + + if (result.success) + newSources[importPath] = result.contentsOrErrorMesage; else - tie(contents, errorMessage) = m_readFile(path); - if (!errorMessage.empty()) { auto err = make_shared(Error::Type::ParserError); *err << errinfo_sourceLocation(import->location()) << - errinfo_comment("Source not found: " + errorMessage); + errinfo_comment("Source \"" + importPath + "\" not found: " + result.contentsOrErrorMesage); m_errors.push_back(std::move(err)); continue; } - else - newSources[path] = contents; } return newSources; } +string CompilerStack::applyRemapping(string const& _path, string const& _context) +{ + // Try to find the longest prefix match in all remappings that are active in the current context. + auto isPrefixOf = [](string const& _a, string const& _b) + { + if (_a.length() > _b.length()) + return false; + return std::equal(_a.begin(), _a.end(), _b.begin()); + }; + + size_t longestPrefix = 0; + string longestPrefixTarget; + for (auto const& redir: m_remappings) + { + // Skip if we already have a closer match. + if (longestPrefix > 0 && redir.prefix.length() <= longestPrefix) + continue; + // Skip if redir.context is not a prefix of _context + if (!isPrefixOf(redir.context, _context)) + continue; + // Skip if the prefix does not match. + if (!isPrefixOf(redir.prefix, _path)) + continue; + + longestPrefix = redir.prefix.length(); + longestPrefixTarget = redir.target; + } + string path = longestPrefixTarget; + path.append(_path.begin() + longestPrefix, _path.end()); + return path; +} + void CompilerStack::resolveImports() { // topological sorting (depth first search) of the import graph, cutting potential cycles diff --git a/libsolidity/interface/CompilerStack.h b/libsolidity/interface/CompilerStack.h index c7f98184..e111c982 100644 --- a/libsolidity/interface/CompilerStack.h +++ b/libsolidity/interface/CompilerStack.h @@ -75,15 +75,23 @@ enum class DocumentationType: uint8_t class CompilerStack: boost::noncopyable { public: - /// File reading callback, should return a pair of content and error message (exactly one nonempty) - /// for a given path. - using ReadFileCallback = std::function(std::string const&)>; + struct ReadFileResult + { + bool success; + std::string contentsOrErrorMesage; + }; + + /// File reading callback. + using ReadFileCallback = std::function; /// Creates a new compiler stack. /// @param _readFile callback to used to read files for import statements. Should return /// @param _addStandardSources Adds standard sources if @a _addStandardSources. explicit CompilerStack(bool _addStandardSources = true, ReadFileCallback const& _readFile = ReadFileCallback()); + /// Sets path remappings in the format "context:prefix=target" + void setRemappings(std::vector const& _remappings); + /// Resets the compiler to a state where the sources are not parsed or even removed. void reset(bool _keepSources = false, bool _addStandardSources = true); @@ -209,6 +217,7 @@ private: /// @a m_readFile and stores the absolute paths of all imports in the AST annotations. /// @returns the newly loaded sources. StringMap loadMissingSources(SourceUnit const& _ast, std::string const& _path); + std::string applyRemapping(std::string const& _path, std::string const& _context); void resolveImports(); /// Checks whether there are libraries with the same name, reports that as an error and /// @returns false in this case. @@ -226,7 +235,17 @@ private: Contract const& contract(std::string const& _contractName = "") const; Source const& source(std::string const& _sourceName = "") const; + struct Remapping + { + std::string context; + std::string prefix; + std::string target; + }; + ReadFileCallback m_readFile; + /// list of path prefix remappings, e.g. mylibrary: github.com/ethereum = /usr/local/ethereum + /// "context:prefix=target" + std::vector m_remappings; bool m_parseSuccessful; std::map m_sources; std::shared_ptr m_globalContext; diff --git a/solc/CommandLineInterface.cpp b/solc/CommandLineInterface.cpp index 21624b3b..571ea2da 100644 --- a/solc/CommandLineInterface.cpp +++ b/solc/CommandLineInterface.cpp @@ -318,36 +318,31 @@ void CommandLineInterface::readInputFilesAndConfigureRemappings() } } else - for (string const& infile: m_args["input-file"].as>()) + for (string path: m_args["input-file"].as>()) { - auto eq = find(infile.begin(), infile.end(), '='); - if (eq != infile.end()) - { - string target(eq + 1, infile.end()); - m_remappings.push_back(make_pair(string(infile.begin(), eq), target)); - m_allowedDirectories.push_back(boost::filesystem::path(target).remove_filename()); - } + auto eq = find(path.begin(), path.end(), '='); + if (eq != path.end()) + path = string(eq + 1, path.end()); else { - auto path = boost::filesystem::path(infile); - if (!boost::filesystem::exists(path)) + auto infile = boost::filesystem::path(path); + if (!boost::filesystem::exists(infile)) { cerr << "Skipping non existant input file \"" << infile << "\"" << endl; continue; } - if (!boost::filesystem::is_regular_file(path)) + if (!boost::filesystem::is_regular_file(infile)) { cerr << "\"" << infile << "\" is not a valid file. Skipping" << endl; continue; } - m_sourceCodes[path.string()] = dev::contentsString(path.string()); - m_allowedDirectories.push_back(boost::filesystem::canonical(path).remove_filename()); + m_sourceCodes[infile.string()] = dev::contentsString(infile.string()); + path = boost::filesystem::canonical(infile).string(); } + m_allowedDirectories.push_back(boost::filesystem::path(path).remove_filename()); } - // Add empty remapping to try the path itself. - m_remappings.push_back(make_pair(string(), string())); } bool CommandLineInterface::parseLibraryOption(string const& _input) @@ -534,67 +529,42 @@ bool CommandLineInterface::processInput() return link(); } - function(string const&)> fileReader = [this](string const& _path) + CompilerStack::ReadFileCallback fileReader = [this](string const& _path) { - // Try to find the longest prefix match in all remappings. At the end, there will bean - // empty remapping so that we also try the path itself, but any file should be either - // in (a subdirectory of) the directory of an explicit source or a remapping target. - int errorLevel = 0; - size_t longestPrefix = 0; - string bestMatchPath; - for (auto const& redir: m_remappings) + auto boostPath = boost::filesystem::path(_path); + if (!boost::filesystem::exists(boostPath)) + return CompilerStack::ReadFileResult{false, "File not found."}; + boostPath = boost::filesystem::canonical(boostPath); + bool isAllowed = false; + for (auto const& allowedDir: m_allowedDirectories) { - auto const& virt = redir.first; - if (longestPrefix > 0 && virt.length() <= longestPrefix) - continue; - if (virt.length() > _path.length() || !std::equal(virt.begin(), virt.end(), _path.begin())) - continue; - string path = redir.second; - path.append(_path.begin() + virt.length(), _path.end()); - auto boostPath = boost::filesystem::path(path); - if (!boost::filesystem::exists(boostPath)) + // If dir is a prefix of boostPath, we are fine. + if ( + std::distance(allowedDir.begin(), allowedDir.end()) <= std::distance(boostPath.begin(), boostPath.end()) && + std::equal(allowedDir.begin(), allowedDir.end(), boostPath.begin()) + ) { - errorLevel = max(errorLevel, 0); - continue; - } - boostPath = boost::filesystem::canonical(boostPath); - bool isAllowed = false; - for (auto const& dir: m_allowedDirectories) - { - // If dir is a prefix of boostPath, we are fine. - if ( - std::distance(dir.begin(), dir.end()) <= std::distance(boostPath.begin(), boostPath.end()) && - std::equal(dir.begin(), dir.end(), boostPath.begin()) - ) - { - isAllowed = true; - break; - } - } - if (!isAllowed) - errorLevel = max(errorLevel, 2); - else if (!boost::filesystem::is_regular_file(boostPath)) - errorLevel = max(errorLevel, 1); - else - { - longestPrefix = virt.length(); - bestMatchPath = path; + isAllowed = true; + break; } } - if (!bestMatchPath.empty()) - return make_pair(m_sourceCodes[bestMatchPath] = dev::contentsString(bestMatchPath), string()); - if (errorLevel == 0) - return make_pair(string(), string("File not found.")); - else if (errorLevel == 1) - return make_pair(string(), string("Not a valid file.")); + if (!isAllowed) + return CompilerStack::ReadFileResult{false, "File outside of allowed directories."}; + else if (!boost::filesystem::is_regular_file(boostPath)) + return CompilerStack::ReadFileResult{false, "Not a valid file."}; else - return make_pair(string(), string("File outside of allowed directories.")); + { + auto contents = dev::contentsString(boostPath.string()); + m_sourceCodes[boostPath.string()] = contents; + return CompilerStack::ReadFileResult{true, contents}; + } }; m_compiler.reset(new CompilerStack(m_args.count(g_argAddStandard) > 0, fileReader)); auto scannerFromSourceName = [&](string const& _sourceName) -> solidity::Scanner const& { return m_compiler->scanner(_sourceName); }; try { + m_compiler->setRemappings(m_args["input-file"].as>()); for (auto const& sourceCode: m_sourceCodes) m_compiler->addSource(sourceCode.first, sourceCode.second); // TODO: Perhaps we should not compile unless requested diff --git a/solc/CommandLineInterface.h b/solc/CommandLineInterface.h index 52854bac..e240fe96 100644 --- a/solc/CommandLineInterface.h +++ b/solc/CommandLineInterface.h @@ -85,8 +85,6 @@ private: boost::program_options::variables_map m_args; /// map of input files to source code strings std::map m_sourceCodes; - /// list of path prefix remappings, e.g. github.com/ethereum -> /usr/local/ethereum - std::vector> m_remappings; /// list of allowed directories to read files from std::vector m_allowedDirectories; /// map of library names to addresses diff --git a/solc/jsonCompiler.cpp b/solc/jsonCompiler.cpp index eaf83705..e8f674a0 100644 --- a/solc/jsonCompiler.cpp +++ b/solc/jsonCompiler.cpp @@ -132,26 +132,31 @@ string compile(StringMap const& _sources, bool _optimize, CStyleReadFileCallback CompilerStack::ReadFileCallback readCallback; if (_readCallback) { - readCallback = [=](string const& _path) -> pair + readCallback = [=](string const& _path) { char* contents_c = nullptr; char* error_c = nullptr; _readCallback(_path.c_str(), &contents_c, &error_c); - string contents; - string error; + CompilerStack::ReadFileResult result; + result.success = true; if (!contents_c && !error_c) - error = "File not found."; + { + result.success = false; + result.contentsOrErrorMesage = "File not found."; + } if (contents_c) { - contents = string(contents_c); + result.success = true; + result.contentsOrErrorMesage = string(contents_c); free(contents_c); } if (error_c) { - error = string(error_c); + result.success = false; + result.contentsOrErrorMesage = string(error_c); free(error_c); } - return make_pair(move(contents), move(error)); + return result; }; } CompilerStack compiler(true, readCallback); diff --git a/test/libsolidity/Imports.cpp b/test/libsolidity/Imports.cpp index 94d3e423..0736a853 100644 --- a/test/libsolidity/Imports.cpp +++ b/test/libsolidity/Imports.cpp @@ -142,6 +142,28 @@ BOOST_AUTO_TEST_CASE(name_clash_in_import) BOOST_CHECK(c.compile()); } +BOOST_AUTO_TEST_CASE(remappings) +{ + CompilerStack c; + c.setRemappings(vector{"s=s_1.4.6", "t=Tee"}); + c.addSource("a", "import \"s/s.sol\"; contract A is S {}"); + c.addSource("b", "import \"t/tee.sol\"; contract A is Tee {} "); + c.addSource("s_1.4.6/s.sol", "contract S {}"); + c.addSource("Tee/tee.sol", "contract Tee {}"); + BOOST_CHECK(c.compile()); +} + +BOOST_AUTO_TEST_CASE(context_dependent_remappings) +{ + CompilerStack c; + c.setRemappings(vector{"a:s=s_1.4.6", "b:s=s_1.4.7"}); + c.addSource("a/a.sol", "import \"s/s.sol\"; contract A is SSix {}"); + c.addSource("b/b.sol", "import \"s/s.sol\"; contract B is SSeven {}"); + c.addSource("s_1.4.6/s.sol", "contract SSix {} "); + c.addSource("s_1.4.7/s.sol", "contract SSeven {} "); + BOOST_CHECK(c.compile()); +} + BOOST_AUTO_TEST_SUITE_END() } -- cgit v1.2.3