diff options
155 files changed, 7571 insertions, 10106 deletions
diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index bc5e3144a..576efab00 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -11,11 +11,6 @@ "Rev": "7dda39b2e7d5e265014674c5af696ba4186679e9" }, { - "ImportPath": "code.google.com/p/snappy-go/snappy", - "Comment": "null-15", - "Rev": "12e4b4183793ac4b061921e7980845e750679fd0" - }, - { "ImportPath": "github.com/codegangsta/cli", "Comment": "1.2.0-95-g9b2bd2b", "Rev": "9b2bd2b3489748d4d0a204fa4eb2ee9e89e0ebc6" @@ -26,10 +21,6 @@ "Rev": "908aad345c9fbf3ab9bbb94031dc02d0d90df1b8" }, { - "ImportPath": "github.com/ethereum/serpent-go", - "Rev": "5767a0dbd759d313df3f404dadb7f98d7ab51443" - }, - { "ImportPath": "github.com/howeyc/fsnotify", "Comment": "v0.9.0-11-g6b1ef89", "Rev": "6b1ef893dc11e0447abda6da20a5203481878dda" @@ -47,10 +38,6 @@ "Rev": "ccfcd0245381f0c94c68f50626665eed3c6b726a" }, { - "ImportPath": "github.com/robertkrimen/otto", - "Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba" - }, - { "ImportPath": "github.com/obscuren/qml", "Rev": "c288002b52e905973b131089a8a7c761d4a2c36a" }, @@ -67,27 +54,7 @@ "Rev": "907cca0f578a5316fb864ec6992dc3d9730ec58c" }, { - "ImportPath": "github.com/robertkrimen/otto/ast", - "Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba" - }, - { - "ImportPath": "github.com/robertkrimen/otto/dbg", - "Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba" - }, - { - "ImportPath": "github.com/robertkrimen/otto/file", - "Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba" - }, - { - "ImportPath": "github.com/robertkrimen/otto/parser", - "Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba" - }, - { - "ImportPath": "github.com/robertkrimen/otto/registry", - "Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba" - }, - { - "ImportPath": "github.com/robertkrimen/otto/token", + "ImportPath": "github.com/robertkrimen/otto", "Rev": "dea31a3d392779af358ec41f77a07fcc7e9d04ba" }, { @@ -96,7 +63,11 @@ }, { "ImportPath": "github.com/syndtr/goleveldb/leveldb", - "Rev": "832fa7ed4d28545eab80f19e1831fc004305cade" + "Rev": "4875955338b0a434238a31165cb87255ab6e9e4a" + }, + { + "ImportPath": "github.com/syndtr/gosnappy/snappy", + "Rev": "156a073208e131d7d2e212cb749feae7c339e846" }, { "ImportPath": "golang.org/x/crypto/pbkdf2", diff --git a/Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/decode.go b/Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/decode.go deleted file mode 100644 index d93c1b9db..000000000 --- a/Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/decode.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snappy - -import ( - "encoding/binary" - "errors" -) - -// ErrCorrupt reports that the input is invalid. -var ErrCorrupt = errors.New("snappy: corrupt input") - -// DecodedLen returns the length of the decoded block. -func DecodedLen(src []byte) (int, error) { - v, _, err := decodedLen(src) - return v, err -} - -// decodedLen returns the length of the decoded block and the number of bytes -// that the length header occupied. -func decodedLen(src []byte) (blockLen, headerLen int, err error) { - v, n := binary.Uvarint(src) - if n == 0 { - return 0, 0, ErrCorrupt - } - if uint64(int(v)) != v { - return 0, 0, errors.New("snappy: decoded block is too large") - } - return int(v), n, nil -} - -// Decode returns the decoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire decoded block. -// Otherwise, a newly allocated slice will be returned. -// It is valid to pass a nil dst. -func Decode(dst, src []byte) ([]byte, error) { - dLen, s, err := decodedLen(src) - if err != nil { - return nil, err - } - if len(dst) < dLen { - dst = make([]byte, dLen) - } - - var d, offset, length int - for s < len(src) { - switch src[s] & 0x03 { - case tagLiteral: - x := uint(src[s] >> 2) - switch { - case x < 60: - s += 1 - case x == 60: - s += 2 - if s > len(src) { - return nil, ErrCorrupt - } - x = uint(src[s-1]) - case x == 61: - s += 3 - if s > len(src) { - return nil, ErrCorrupt - } - x = uint(src[s-2]) | uint(src[s-1])<<8 - case x == 62: - s += 4 - if s > len(src) { - return nil, ErrCorrupt - } - x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16 - case x == 63: - s += 5 - if s > len(src) { - return nil, ErrCorrupt - } - x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24 - } - length = int(x + 1) - if length <= 0 { - return nil, errors.New("snappy: unsupported literal length") - } - if length > len(dst)-d || length > len(src)-s { - return nil, ErrCorrupt - } - copy(dst[d:], src[s:s+length]) - d += length - s += length - continue - - case tagCopy1: - s += 2 - if s > len(src) { - return nil, ErrCorrupt - } - length = 4 + int(src[s-2])>>2&0x7 - offset = int(src[s-2])&0xe0<<3 | int(src[s-1]) - - case tagCopy2: - s += 3 - if s > len(src) { - return nil, ErrCorrupt - } - length = 1 + int(src[s-3])>>2 - offset = int(src[s-2]) | int(src[s-1])<<8 - - case tagCopy4: - return nil, errors.New("snappy: unsupported COPY_4 tag") - } - - end := d + length - if offset > d || end > len(dst) { - return nil, ErrCorrupt - } - for ; d < end; d++ { - dst[d] = dst[d-offset] - } - } - if d != dLen { - return nil, ErrCorrupt - } - return dst[:d], nil -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/.gitignore b/Godeps/_workspace/src/github.com/ethereum/serpent-go/.gitignore deleted file mode 100644 index 5d02b54e5..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/tmp -*/**/*un~ -*un~ -.DS_Store -*/**/.DS_Store diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/.gitmodules b/Godeps/_workspace/src/github.com/ethereum/serpent-go/.gitmodules deleted file mode 100644 index 054c7d628..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "serp"] - path = serpent - url = https://github.com/ethereum/serpent.git diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/README.md b/Godeps/_workspace/src/github.com/ethereum/serpent-go/README.md deleted file mode 100644 index 404f1b380..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/README.md +++ /dev/null @@ -1,12 +0,0 @@ -[serpent](https://github.com/ethereum/serpent) go bindings. - -## Build instructions - -``` -go get -d github.com/ethereum/serpent-go -cd $GOPATH/src/github.com/ethereum/serpent-go -git submodule init -git submodule update -``` - -You're now ready to go :-) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/all.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/all.cpp deleted file mode 100644 index 80032f900..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/all.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "serpent/bignum.cpp" -#include "serpent/util.cpp" -#include "serpent/tokenize.cpp" -#include "serpent/parser.cpp" -#include "serpent/compiler.cpp" -#include "serpent/funcs.cpp" -#include "serpent/lllparser.cpp" -#include "serpent/rewriter.cpp" - -#include "serpent/opcodes.cpp" -#include "serpent/optimize.cpp" -#include "serpent/functions.cpp" -#include "serpent/preprocess.cpp" -#include "serpent/rewriteutils.cpp" - -#include "cpp/api.cpp" diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/cpp/api.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/cpp/api.cpp deleted file mode 100644 index bd2c85c7d..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/cpp/api.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include <string> - -#include "serpent/lllparser.h" -#include "serpent/bignum.h" -#include "serpent/util.h" -#include "serpent/tokenize.h" -#include "serpent/parser.h" -#include "serpent/compiler.h" - -#include "cpp/api.h" - -const char *compileGo(char *code, int *err) -{ - try { - std::string c = binToHex(compile(std::string(code))); - - return c.c_str(); - } - catch(std::string &error) { - *err = 1; - return error.c_str(); - } - catch(...) { - return "Unknown error"; - } -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/cpp/api.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/cpp/api.h deleted file mode 100644 index 235b5eb4a..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/cpp/api.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef CPP_API_H -#define CPP_API_H - -#ifdef __cplusplus -extern "C" { -#endif - -const char *compileGo(char *code, int *err); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent.go b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent.go deleted file mode 100644 index 39b60eed7..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent.go +++ /dev/null @@ -1,27 +0,0 @@ -package serpent - -// #cgo CXXFLAGS: -I. -Ilangs/ -std=c++0x -Wall -fno-strict-aliasing -// #cgo LDFLAGS: -lstdc++ -// -// #include "cpp/api.h" -// -import "C" - -import ( - "encoding/hex" - "errors" - "unsafe" -) - -func Compile(str string) ([]byte, error) { - var err C.int - out := C.GoString(C.compileGo(C.CString(str), (*C.int)(unsafe.Pointer(&err)))) - - if err == C.int(1) { - return nil, errors.New(out) - } - - bytes, _ := hex.DecodeString(out) - - return bytes, nil -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/.gitignore b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/.gitignore deleted file mode 100644 index 72b65e446..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/.gitignore +++ /dev/null @@ -1,12 +0,0 @@ -[._]*.s[a-w][a-z] -[._]s[a-w][a-z] -*.un~ -Session.vim -.netrwhist -*~ -*.o -serpent -libserpent.a -pyserpent.so -dist -*.egg-info diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/MANIFEST.in b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/MANIFEST.in deleted file mode 100644 index 5f5766ced..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include *.cpp -include *.h -include *py -include README.md -include Makefile diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/Makefile b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/Makefile deleted file mode 100644 index 28c38728e..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/Makefile +++ /dev/null @@ -1,55 +0,0 @@ -PLATFORM_OPTS = -PYTHON = /usr/include/python2.7 -CXXFLAGS = -fPIC -# -g3 -O0 -BOOST_INC = /usr/include -BOOST_LIB = /usr/lib -TARGET = pyserpent -COMMON_OBJS = bignum.o util.o tokenize.o lllparser.o parser.o opcodes.o optimize.o functions.o rewriteutils.o preprocess.o rewriter.o compiler.o funcs.o -HEADERS = bignum.h util.h tokenize.h lllparser.h parser.h opcodes.h functions.h optimize.h rewriteutils.h preprocess.h rewriter.h compiler.h funcs.h -PYTHON_VERSION = 2.7 - -serpent : serpentc lib - -lib: - ar rvs libserpent.a $(COMMON_OBJS) - g++ $(CXXFLAGS) -shared $(COMMON_OBJS) -o libserpent.so - -serpentc: $(COMMON_OBJS) cmdline.o - rm -rf serpent - g++ -Wall $(COMMON_OBJS) cmdline.o -o serpent - -bignum.o : bignum.cpp bignum.h - -opcodes.o : opcodes.cpp opcodes.h - -util.o : util.cpp util.h bignum.o - -tokenize.o : tokenize.cpp tokenize.h util.o - -lllparser.o : lllparser.cpp lllparser.h tokenize.o util.o - -parser.o : parser.cpp parser.h tokenize.o util.o - -rewriter.o : rewriter.cpp rewriter.h lllparser.o util.o rewriteutils.o preprocess.o opcodes.o functions.o - -preprocessor.o: rewriteutils.o functions.o - -compiler.o : compiler.cpp compiler.h util.o - -funcs.o : funcs.cpp funcs.h - -cmdline.o: cmdline.cpp - -pyext.o: pyext.cpp - -clean: - rm -f serpent *\.o libserpent.a libserpent.so - -install: - cp serpent /usr/local/bin - cp libserpent.a /usr/local/lib - cp libserpent.so /usr/local/lib - rm -rf /usr/local/include/libserpent - mkdir -p /usr/local/include/libserpent - cp $(HEADERS) /usr/local/include/libserpent diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/README.md b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/README.md deleted file mode 100644 index 03dfcc15f..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/README.md +++ /dev/null @@ -1,3 +0,0 @@ -Installation: - -```make && sudo make install``` diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/bignum.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/bignum.cpp deleted file mode 100644 index 108b1eb04..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/bignum.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "bignum.h" - -//Integer to string conversion -std::string unsignedToDecimal(unsigned branch) { - if (branch < 10) return nums.substr(branch, 1); - else return unsignedToDecimal(branch / 10) + nums.substr(branch % 10,1); -} - -//Add two strings representing decimal values -std::string decimalAdd(std::string a, std::string b) { - std::string o = a; - while (b.length() < a.length()) b = "0" + b; - while (o.length() < b.length()) o = "0" + o; - bool carry = false; - for (int i = o.length() - 1; i >= 0; i--) { - o[i] = o[i] + b[i] - '0'; - if (carry) o[i]++; - if (o[i] > '9') { - o[i] -= 10; - carry = true; - } - else carry = false; - } - if (carry) o = "1" + o; - return o; -} - -//Helper function for decimalMul -std::string decimalDigitMul(std::string a, int dig) { - if (dig == 0) return "0"; - else return decimalAdd(a, decimalDigitMul(a, dig - 1)); -} - -//Multiply two strings representing decimal values -std::string decimalMul(std::string a, std::string b) { - std::string o = "0"; - for (unsigned i = 0; i < b.length(); i++) { - std::string n = decimalDigitMul(a, b[i] - '0'); - if (n != "0") { - for (unsigned j = i + 1; j < b.length(); j++) n += "0"; - } - o = decimalAdd(o, n); - } - return o; -} - -//Modexp -std::string decimalModExp(std::string b, std::string e, std::string m) { - if (e == "0") return "1"; - else if (e == "1") return b; - else if (decimalMod(e, "2") == "0") { - std::string o = decimalModExp(b, decimalDiv(e, "2"), m); - return decimalMod(decimalMul(o, o), m); - } - else { - std::string o = decimalModExp(b, decimalDiv(e, "2"), m); - return decimalMod(decimalMul(decimalMul(o, o), b), m); - } -} - -//Is a greater than b? Flag allows equality -bool decimalGt(std::string a, std::string b, bool eqAllowed) { - if (a == b) return eqAllowed; - return (a.length() > b.length()) || (a.length() >= b.length() && a > b); -} - -//Subtract the two strings representing decimal values -std::string decimalSub(std::string a, std::string b) { - if (b == "0") return a; - if (b == a) return "0"; - while (b.length() < a.length()) b = "0" + b; - std::string c = b; - for (unsigned i = 0; i < c.length(); i++) c[i] = '0' + ('9' - c[i]); - std::string o = decimalAdd(decimalAdd(a, c).substr(1), "1"); - while (o.size() > 1 && o[0] == '0') o = o.substr(1); - return o; -} - -//Divide the two strings representing decimal values -std::string decimalDiv(std::string a, std::string b) { - std::string c = b; - if (decimalGt(c, a)) return "0"; - int zeroes = -1; - while (decimalGt(a, c, true)) { - zeroes += 1; - c = c + "0"; - } - c = c.substr(0, c.size() - 1); - std::string quot = "0"; - while (decimalGt(a, c, true)) { - a = decimalSub(a, c); - quot = decimalAdd(quot, "1"); - } - for (int i = 0; i < zeroes; i++) quot += "0"; - return decimalAdd(quot, decimalDiv(a, b)); -} - -//Modulo the two strings representing decimal values -std::string decimalMod(std::string a, std::string b) { - return decimalSub(a, decimalMul(decimalDiv(a, b), b)); -} - -//String to int conversion -unsigned decimalToUnsigned(std::string a) { - if (a.size() == 0) return 0; - else return (a[a.size() - 1] - '0') - + decimalToUnsigned(a.substr(0,a.size()-1)) * 10; -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/bignum.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/bignum.h deleted file mode 100644 index 99571acd2..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/bignum.h +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef ETHSERP_BIGNUM -#define ETHSERP_BIGNUM - -const std::string nums = "0123456789"; - -const std::string tt256 = -"115792089237316195423570985008687907853269984665640564039457584007913129639936" -; - -const std::string tt256m1 = -"115792089237316195423570985008687907853269984665640564039457584007913129639935" -; - -const std::string tt255 = -"57896044618658097711785492504343953926634992332820282019728792003956564819968"; - -const std::string tt176 = -"95780971304118053647396689196894323976171195136475136"; - -std::string unsignedToDecimal(unsigned branch); - -std::string decimalAdd(std::string a, std::string b); - -std::string decimalMul(std::string a, std::string b); - -std::string decimalSub(std::string a, std::string b); - -std::string decimalDiv(std::string a, std::string b); - -std::string decimalMod(std::string a, std::string b); - -std::string decimalModExp(std::string b, std::string e, std::string m); - -bool decimalGt(std::string a, std::string b, bool eqAllowed=false); - -unsigned decimalToUnsigned(std::string a); - -#define utd unsignedToDecimal -#define dtu decimalToUnsigned - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/cmdline.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/cmdline.cpp deleted file mode 100644 index fe2560830..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/cmdline.cpp +++ /dev/null @@ -1,132 +0,0 @@ -#include <stdio.h> -#include <string> -#include <iostream> -#include <vector> -#include <map> -#include "funcs.h" - -int main(int argv, char** argc) { - if (argv == 1) { - std::cerr << "Must provide a command and arguments! Try parse, rewrite, compile, assemble\n"; - return 0; - } - if (argv == 2 && std::string(argc[1]) == "--help" || std::string(argc[1]) == "-h" ) { - std::cout << argc[1] << "\n"; - - std::cout << "serpent command input\n"; - std::cout << "where input -s for from stdin, a file, or interpreted as serpent code if does not exist as file."; - std::cout << "where command: \n"; - std::cout << " parse: Just parses and returns s-expression code.\n"; - std::cout << " rewrite: Parse, use rewrite rules print s-expressions of result.\n"; - std::cout << " compile: Return resulting compiled EVM code in hex.\n"; - std::cout << " assemble: Return result from step before compilation.\n"; - return 0; - } - - std::string flag = ""; - std::string command = argc[1]; - std::string input; - std::string secondInput; - if (std::string(argc[1]) == "-s") { - flag = command.substr(1); - command = argc[2]; - input = ""; - std::string line; - while (std::getline(std::cin, line)) { - input += line + "\n"; - } - secondInput = argv == 3 ? "" : argc[3]; - } - else { - if (argv == 2) { - std::cerr << "Not enough arguments for serpent cmdline\n"; - throw(0); - } - input = argc[2]; - secondInput = argv == 3 ? "" : argc[3]; - } - bool haveSec = secondInput.length() > 0; - if (command == "parse" || command == "parse_serpent") { - std::cout << printAST(parseSerpent(input), haveSec) << "\n"; - } - else if (command == "rewrite") { - std::cout << printAST(rewrite(parseLLL(input, true)), haveSec) << "\n"; - } - else if (command == "compile_to_lll") { - std::cout << printAST(compileToLLL(input), haveSec) << "\n"; - } - else if (command == "rewrite_chunk") { - std::cout << printAST(rewriteChunk(parseLLL(input, true)), haveSec) << "\n"; - } - else if (command == "compile_chunk_to_lll") { - std::cout << printAST(compileChunkToLLL(input), haveSec) << "\n"; - } - else if (command == "build_fragtree") { - std::cout << printAST(buildFragmentTree(parseLLL(input, true))) << "\n"; - } - else if (command == "compile_lll") { - std::cout << binToHex(compileLLL(parseLLL(input, true))) << "\n"; - } - else if (command == "dereference") { - std::cout << printAST(dereference(parseLLL(input, true)), haveSec) <<"\n"; - } - else if (command == "pretty_assemble") { - std::cout << printTokens(prettyAssemble(parseLLL(input, true))) <<"\n"; - } - else if (command == "pretty_compile_lll") { - std::cout << printTokens(prettyCompileLLL(parseLLL(input, true))) << "\n"; - } - else if (command == "pretty_compile") { - std::cout << printTokens(prettyCompile(input)) << "\n"; - } - else if (command == "pretty_compile_chunk") { - std::cout << printTokens(prettyCompileChunk(input)) << "\n"; - } - else if (command == "assemble") { - std::cout << assemble(parseLLL(input, true)) << "\n"; - } - else if (command == "serialize") { - std::cout << binToHex(serialize(tokenize(input, Metadata(), false))) << "\n"; - } - else if (command == "flatten") { - std::cout << printTokens(flatten(parseLLL(input, true))) << "\n"; - } - else if (command == "deserialize") { - std::cout << printTokens(deserialize(hexToBin(input))) << "\n"; - } - else if (command == "compile") { - std::cout << binToHex(compile(input)) << "\n"; - } - else if (command == "compile_chunk") { - std::cout << binToHex(compileChunk(input)) << "\n"; - } - else if (command == "encode_datalist") { - std::vector<Node> tokens = tokenize(input); - std::vector<std::string> o; - for (int i = 0; i < (int)tokens.size(); i++) { - o.push_back(tokens[i].val); - } - std::cout << binToHex(encodeDatalist(o)) << "\n"; - } - else if (command == "decode_datalist") { - std::vector<std::string> o = decodeDatalist(hexToBin(input)); - std::vector<Node> tokens; - for (int i = 0; i < (int)o.size(); i++) - tokens.push_back(token(o[i])); - std::cout << printTokens(tokens) << "\n"; - } - else if (command == "tokenize") { - std::cout << printTokens(tokenize(input)); - } - else if (command == "biject") { - if (argv == 3) - std::cerr << "Not enough arguments for biject\n"; - int pos = decimalToUnsigned(secondInput); - std::vector<Node> n = prettyCompile(input); - if (pos >= (int)n.size()) - std::cerr << "Code position too high\n"; - Metadata m = n[pos].metadata; - std::cout << "Opcode: " << n[pos].val << ", file: " << m.file << - ", line: " << m.ln << ", char: " << m.ch << "\n"; - } -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/compiler.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/compiler.cpp deleted file mode 100644 index b9281dcbc..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/compiler.cpp +++ /dev/null @@ -1,554 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" -#include "bignum.h" -#include "opcodes.h" - -struct programAux { - std::map<std::string, std::string> vars; - int nextVarMem; - bool allocUsed; - bool calldataUsed; - int step; - int labelLength; -}; - -struct programVerticalAux { - int height; - std::string innerScopeName; - std::map<std::string, int> dupvars; - std::map<std::string, int> funvars; - std::vector<mss> scopes; -}; - -struct programData { - programAux aux; - Node code; - int outs; -}; - -programAux Aux() { - programAux o; - o.allocUsed = false; - o.calldataUsed = false; - o.step = 0; - o.nextVarMem = 32; - return o; -} - -programVerticalAux verticalAux() { - programVerticalAux o; - o.height = 0; - o.dupvars = std::map<std::string, int>(); - o.funvars = std::map<std::string, int>(); - o.scopes = std::vector<mss>(); - return o; -} - -programData pd(programAux aux = Aux(), Node code=token("_"), int outs=0) { - programData o; - o.aux = aux; - o.code = code; - o.outs = outs; - return o; -} - -Node multiToken(Node nodes[], int len, Metadata met) { - std::vector<Node> out; - for (int i = 0; i < len; i++) { - out.push_back(nodes[i]); - } - return astnode("_", out, met); -} - -Node finalize(programData c); - -Node popwrap(Node node) { - Node nodelist[] = { - node, - token("POP", node.metadata) - }; - return multiToken(nodelist, 2, node.metadata); -} - -// Grabs variables -mss getVariables(Node node, mss cur=mss()) { - Metadata m = node.metadata; - // Tokens don't contain any variables - if (node.type == TOKEN) - return cur; - // Don't descend into call fragments - else if (node.val == "lll") - return getVariables(node.args[1], cur); - // At global scope get/set/ref also declare - else if (node.val == "get" || node.val == "set" || node.val == "ref") { - if (node.args[0].type != TOKEN) - err("Variable name must be simple token," - " not complex expression!", m); - if (!cur.count(node.args[0].val)) { - cur[node.args[0].val] = utd(cur.size() * 32 + 32); - //std::cerr << node.args[0].val << " " << cur[node.args[0].val] << "\n"; - } - } - // Recursively process children - for (unsigned i = 0; i < node.args.size(); i++) { - cur = getVariables(node.args[i], cur); - } - return cur; -} - -// Turns LLL tree into tree of code fragments -programData opcodeify(Node node, - programAux aux=Aux(), - programVerticalAux vaux=verticalAux()) { - std::string symb = "_"+mkUniqueToken(); - Metadata m = node.metadata; - // Get variables - if (!aux.vars.size()) { - aux.vars = getVariables(node); - aux.nextVarMem = aux.vars.size() * 32 + 32; - } - // Numbers - if (node.type == TOKEN) { - return pd(aux, nodeToNumeric(node), 1); - } - else if (node.val == "ref" || node.val == "get" || node.val == "set") { - std::string varname = node.args[0].val; - // Determine reference to variable - Node varNode = tkn(aux.vars[varname], m); - //std::cerr << varname << " " << printSimple(varNode) << "\n"; - // Set variable - if (node.val == "set") { - programData sub = opcodeify(node.args[1], aux, vaux); - if (!sub.outs) - err("Value to set variable must have nonzero arity!", m); - // What if we are setting a stack variable? - if (vaux.dupvars.count(node.args[0].val)) { - int h = vaux.height - vaux.dupvars[node.args[0].val]; - if (h > 16) err("Too deep for stack variable (max 16)", m); - Node nodelist[] = { - sub.code, - token("SWAP"+unsignedToDecimal(h), m), - token("POP", m) - }; - return pd(sub.aux, multiToken(nodelist, 3, m), 0); - } - // Setting a memory variable - else { - Node nodelist[] = { - sub.code, - varNode, - token("MSTORE", m), - }; - return pd(sub.aux, multiToken(nodelist, 3, m), 0); - } - } - // Get variable - else if (node.val == "get") { - // Getting a stack variable - if (vaux.dupvars.count(node.args[0].val)) { - int h = vaux.height - vaux.dupvars[node.args[0].val]; - if (h > 16) err("Too deep for stack variable (max 16)", m); - return pd(aux, token("DUP"+unsignedToDecimal(h)), 1); - } - // Getting a memory variable - else { - Node nodelist[] = - { varNode, token("MLOAD", m) }; - return pd(aux, multiToken(nodelist, 2, m), 1); - } - } - // Refer variable - else if (node.val == "ref") { - if (vaux.dupvars.count(node.args[0].val)) - err("Cannot ref stack variable!", m); - return pd(aux, varNode, 1); - } - } - // Comments do nothing - else if (node.val == "comment") { - Node nodelist[] = { }; - return pd(aux, multiToken(nodelist, 0, m), 0); - } - // Custom operation sequence - // eg. (ops bytez id msize swap1 msize add 0 swap1 mstore) == alloc - if (node.val == "ops") { - std::vector<Node> subs2; - int depth = 0; - for (unsigned i = 0; i < node.args.size(); i++) { - std::string op = upperCase(node.args[i].val); - if (node.args[i].type == ASTNODE || opinputs(op) == -1) { - programVerticalAux vaux2 = vaux; - vaux2.height = vaux.height - i - 1 + node.args.size(); - programData sub = opcodeify(node.args[i], aux, vaux2); - aux = sub.aux; - depth += sub.outs; - subs2.push_back(sub.code); - } - else { - subs2.push_back(token(op, m)); - depth += opoutputs(op) - opinputs(op); - } - } - if (depth < 0 || depth > 1) err("Stack depth mismatch", m); - return pd(aux, astnode("_", subs2, m), 0); - } - // Code blocks - if (node.val == "lll" && node.args.size() == 2) { - if (node.args[1].val != "0") aux.allocUsed = true; - std::vector<Node> o; - o.push_back(finalize(opcodeify(node.args[0]))); - programData sub = opcodeify(node.args[1], aux, vaux); - Node code = astnode("____CODE", o, m); - Node nodelist[] = { - token("$begincode"+symb+".endcode"+symb, m), token("DUP1", m), - token("$begincode"+symb, m), sub.code, token("CODECOPY", m), - token("$endcode"+symb, m), token("JUMP", m), - token("~begincode"+symb, m), code, - token("~endcode"+symb, m), token("JUMPDEST", m) - }; - return pd(sub.aux, multiToken(nodelist, 11, m), 1); - } - // Stack variables - if (node.val == "with") { - programData initial = opcodeify(node.args[1], aux, vaux); - programVerticalAux vaux2 = vaux; - vaux2.dupvars[node.args[0].val] = vaux.height; - vaux2.height += 1; - if (!initial.outs) - err("Initial variable value must have nonzero arity!", m); - programData sub = opcodeify(node.args[2], initial.aux, vaux2); - Node nodelist[] = { - initial.code, - sub.code - }; - programData o = pd(sub.aux, multiToken(nodelist, 2, m), sub.outs); - if (sub.outs) - o.code.args.push_back(token("SWAP1", m)); - o.code.args.push_back(token("POP", m)); - return o; - } - // Seq of multiple statements - if (node.val == "seq") { - std::vector<Node> children; - int lastOut = 0; - for (unsigned i = 0; i < node.args.size(); i++) { - programData sub = opcodeify(node.args[i], aux, vaux); - aux = sub.aux; - if (sub.outs == 1) { - if (i < node.args.size() - 1) sub.code = popwrap(sub.code); - else lastOut = 1; - } - children.push_back(sub.code); - } - return pd(aux, astnode("_", children, m), lastOut); - } - // 2-part conditional (if gets rewritten to unless in rewrites) - else if (node.val == "unless" && node.args.size() == 2) { - programData cond = opcodeify(node.args[0], aux, vaux); - programData action = opcodeify(node.args[1], cond.aux, vaux); - aux = action.aux; - if (!cond.outs) err("Condition of if/unless statement has arity 0", m); - if (action.outs) action.code = popwrap(action.code); - Node nodelist[] = { - cond.code, - token("$endif"+symb, m), token("JUMPI", m), - action.code, - token("~endif"+symb, m), token("JUMPDEST", m) - }; - return pd(aux, multiToken(nodelist, 6, m), 0); - } - // 3-part conditional - else if (node.val == "if" && node.args.size() == 3) { - programData ifd = opcodeify(node.args[0], aux, vaux); - programData thend = opcodeify(node.args[1], ifd.aux, vaux); - programData elsed = opcodeify(node.args[2], thend.aux, vaux); - aux = elsed.aux; - if (!ifd.outs) - err("Condition of if/unless statement has arity 0", m); - // Handle cases where one conditional outputs something - // and the other does not - int outs = (thend.outs && elsed.outs) ? 1 : 0; - if (thend.outs > outs) thend.code = popwrap(thend.code); - if (elsed.outs > outs) elsed.code = popwrap(elsed.code); - Node nodelist[] = { - ifd.code, - token("ISZERO", m), - token("$else"+symb, m), token("JUMPI", m), - thend.code, - token("$endif"+symb, m), token("JUMP", m), - token("~else"+symb, m), token("JUMPDEST", m), - elsed.code, - token("~endif"+symb, m), token("JUMPDEST", m) - }; - return pd(aux, multiToken(nodelist, 12, m), outs); - } - // While (rewritten to this in rewrites) - else if (node.val == "until") { - programData cond = opcodeify(node.args[0], aux, vaux); - programData action = opcodeify(node.args[1], cond.aux, vaux); - aux = action.aux; - if (!cond.outs) - err("Condition of while/until loop has arity 0", m); - if (action.outs) action.code = popwrap(action.code); - Node nodelist[] = { - token("~beg"+symb, m), token("JUMPDEST", m), - cond.code, - token("$end"+symb, m), token("JUMPI", m), - action.code, - token("$beg"+symb, m), token("JUMP", m), - token("~end"+symb, m), token("JUMPDEST", m), - }; - return pd(aux, multiToken(nodelist, 10, m)); - } - // Memory allocations - else if (node.val == "alloc") { - programData bytez = opcodeify(node.args[0], aux, vaux); - aux = bytez.aux; - if (!bytez.outs) - err("Alloc input has arity 0", m); - aux.allocUsed = true; - Node nodelist[] = { - bytez.code, - token("MSIZE", m), token("SWAP1", m), token("MSIZE", m), - token("ADD", m), - token("0", m), token("SWAP1", m), token("MSTORE", m) - }; - return pd(aux, multiToken(nodelist, 8, m), 1); - } - // All other functions/operators - else { - std::vector<Node> subs2; - int depth = opinputs(upperCase(node.val)); - if (depth == -1) - err("Not a function or opcode: "+node.val, m); - if ((int)node.args.size() != depth) - err("Invalid arity for "+node.val, m); - for (int i = node.args.size() - 1; i >= 0; i--) { - programVerticalAux vaux2 = vaux; - vaux2.height = vaux.height - i - 1 + node.args.size(); - programData sub = opcodeify(node.args[i], aux, vaux2); - aux = sub.aux; - if (!sub.outs) - err("Input "+unsignedToDecimal(i)+" has arity 0", sub.code.metadata); - subs2.push_back(sub.code); - } - subs2.push_back(token(upperCase(node.val), m)); - int outdepth = opoutputs(upperCase(node.val)); - return pd(aux, astnode("_", subs2, m), outdepth); - } -} - -// Adds necessary wrappers to a program -Node finalize(programData c) { - std::vector<Node> bottom; - Metadata m = c.code.metadata; - // If we are using both alloc and variables, we need to pre-zfill - // some memory - if ((c.aux.allocUsed || c.aux.calldataUsed) && c.aux.vars.size() > 0) { - Node nodelist[] = { - token("0", m), - token(unsignedToDecimal(c.aux.nextVarMem - 1)), - token("MSTORE8", m) - }; - bottom.push_back(multiToken(nodelist, 3, m)); - } - // The actual code - bottom.push_back(c.code); - return astnode("_", bottom, m); -} - -//LLL -> code fragment tree -Node buildFragmentTree(Node node) { - return finalize(opcodeify(node)); -} - - -// Builds a dictionary mapping labels to variable names -programAux buildDict(Node program, programAux aux, int labelLength) { - Metadata m = program.metadata; - // Token - if (program.type == TOKEN) { - if (isNumberLike(program)) { - aux.step += 1 + toByteArr(program.val, m).size(); - } - else if (program.val[0] == '~') { - aux.vars[program.val.substr(1)] = unsignedToDecimal(aux.step); - } - else if (program.val[0] == '$') { - aux.step += labelLength + 1; - } - else aux.step += 1; - } - // A sub-program (ie. LLL) - else if (program.val == "____CODE") { - programAux auks = Aux(); - for (unsigned i = 0; i < program.args.size(); i++) { - auks = buildDict(program.args[i], auks, labelLength); - } - for (std::map<std::string,std::string>::iterator it=auks.vars.begin(); - it != auks.vars.end(); - it++) { - aux.vars[(*it).first] = (*it).second; - } - aux.step += auks.step; - } - // Normal sub-block - else { - for (unsigned i = 0; i < program.args.size(); i++) { - aux = buildDict(program.args[i], aux, labelLength); - } - } - return aux; -} - -// Applies that dictionary -Node substDict(Node program, programAux aux, int labelLength) { - Metadata m = program.metadata; - std::vector<Node> out; - std::vector<Node> inner; - if (program.type == TOKEN) { - if (program.val[0] == '$') { - std::string tokStr = "PUSH"+unsignedToDecimal(labelLength); - out.push_back(token(tokStr, m)); - int dotLoc = program.val.find('.'); - if (dotLoc == -1) { - std::string val = aux.vars[program.val.substr(1)]; - inner = toByteArr(val, m, labelLength); - } - else { - std::string start = aux.vars[program.val.substr(1, dotLoc-1)], - end = aux.vars[program.val.substr(dotLoc + 1)], - dist = decimalSub(end, start); - inner = toByteArr(dist, m, labelLength); - } - out.push_back(astnode("_", inner, m)); - } - else if (program.val[0] == '~') { } - else if (isNumberLike(program)) { - inner = toByteArr(program.val, m); - out.push_back(token("PUSH"+unsignedToDecimal(inner.size()))); - out.push_back(astnode("_", inner, m)); - } - else return program; - } - else { - for (unsigned i = 0; i < program.args.size(); i++) { - Node n = substDict(program.args[i], aux, labelLength); - if (n.type == TOKEN || n.args.size()) out.push_back(n); - } - } - return astnode("_", out, m); -} - -// Compiled fragtree -> compiled fragtree without labels -Node dereference(Node program) { - int sz = treeSize(program) * 4; - int labelLength = 1; - while (sz >= 256) { labelLength += 1; sz /= 256; } - programAux aux = buildDict(program, Aux(), labelLength); - return substDict(program, aux, labelLength); -} - -// Dereferenced fragtree -> opcodes -std::vector<Node> flatten(Node derefed) { - std::vector<Node> o; - if (derefed.type == TOKEN) { - o.push_back(derefed); - } - else { - for (unsigned i = 0; i < derefed.args.size(); i++) { - std::vector<Node> oprime = flatten(derefed.args[i]); - for (unsigned j = 0; j < oprime.size(); j++) o.push_back(oprime[j]); - } - } - return o; -} - -// Opcodes -> bin -std::string serialize(std::vector<Node> codons) { - std::string o; - for (unsigned i = 0; i < codons.size(); i++) { - int v; - if (isNumberLike(codons[i])) { - v = decimalToUnsigned(codons[i].val); - } - else if (codons[i].val.substr(0,4) == "PUSH") { - v = 95 + decimalToUnsigned(codons[i].val.substr(4)); - } - else { - v = opcode(codons[i].val); - } - o += (char)v; - } - return o; -} - -// Bin -> opcodes -std::vector<Node> deserialize(std::string ser) { - std::vector<Node> o; - int backCount = 0; - for (unsigned i = 0; i < ser.length(); i++) { - unsigned char v = (unsigned char)ser[i]; - std::string oper = op((int)v); - if (oper != "" && backCount <= 0) o.push_back(token(oper)); - else if (v >= 96 && v < 128 && backCount <= 0) { - o.push_back(token("PUSH"+unsignedToDecimal(v - 95))); - } - else o.push_back(token(unsignedToDecimal(v))); - if (v >= 96 && v < 128 && backCount <= 0) { - backCount = v - 95; - } - else backCount--; - } - return o; -} - -// Fragtree -> bin -std::string assemble(Node fragTree) { - return serialize(flatten(dereference(fragTree))); -} - -// Fragtree -> tokens -std::vector<Node> prettyAssemble(Node fragTree) { - return flatten(dereference(fragTree)); -} - -// LLL -> bin -std::string compileLLL(Node program) { - return assemble(buildFragmentTree(program)); -} - -// LLL -> tokens -std::vector<Node> prettyCompileLLL(Node program) { - return prettyAssemble(buildFragmentTree(program)); -} - -// Converts a list of integer values to binary transaction data -std::string encodeDatalist(std::vector<std::string> vals) { - std::string o; - for (unsigned i = 0; i < vals.size(); i++) { - std::vector<Node> n = toByteArr(strToNumeric(vals[i]), Metadata(), 32); - for (unsigned j = 0; j < n.size(); j++) { - int v = decimalToUnsigned(n[j].val); - o += (char)v; - } - } - return o; -} - -// Converts binary transaction data into a list of integer values -std::vector<std::string> decodeDatalist(std::string ser) { - std::vector<std::string> out; - for (unsigned i = 0; i < ser.length(); i+= 32) { - std::string o = "0"; - for (unsigned j = i; j < i + 32; j++) { - int vj = (int)(unsigned char)ser[j]; - o = decimalAdd(decimalMul(o, "256"), unsignedToDecimal(vj)); - } - out.push_back(o); - } - return out; -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/compiler.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/compiler.h deleted file mode 100644 index aecaa3718..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/compiler.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef ETHSERP_COMPILER -#define ETHSERP_COMPILER - -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" - -// Compiled fragtree -> compiled fragtree without labels -Node dereference(Node program); - -// LLL -> fragtree -Node buildFragmentTree(Node program); - -// Dereferenced fragtree -> opcodes -std::vector<Node> flatten(Node derefed); - -// opcodes -> bin -std::string serialize(std::vector<Node> codons); - -// Fragtree -> bin -std::string assemble(Node fragTree); - -// Fragtree -> opcodes -std::vector<Node> prettyAssemble(Node fragTree); - -// LLL -> bin -std::string compileLLL(Node program); - -// LLL -> opcodes -std::vector<Node> prettyCompileLLL(Node program); - -// bin -> opcodes -std::vector<Node> deserialize(std::string ser); - -// Converts a list of integer values to binary transaction data -std::string encodeDatalist(std::vector<std::string> vals); - -// Converts binary transaction data into a list of integer values -std::vector<std::string> decodeDatalist(std::string ser); - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/example.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/example.cpp deleted file mode 100644 index 1ce2590d0..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/example.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include <libserpent/funcs.h> -#include <libserpent/bignum.h> -#include <iostream> - -using namespace std; - -int main() { - cout << printAST(compileToLLL(get_file_contents("examples/namecoin.se"))) << "\n"; - cout << decimalSub("10234", "10234") << "\n"; - cout << decimalSub("10234", "10233") << "\n"; -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/collatz.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/collatz.se deleted file mode 100644 index 148b47b59..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/collatz.se +++ /dev/null @@ -1,11 +0,0 @@ -x = msg.data[0] -steps = 0 - -while x > 1: - steps += 1 - if (x % 2) == 0: - x /= 2 - else: - x = 3 * x + 1 - -return(steps) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/counterparty/counterparty.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/counterparty/counterparty.se deleted file mode 100644 index abec0d102..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/counterparty/counterparty.se +++ /dev/null @@ -1,274 +0,0 @@ -# Ethereum forks Counterparty in 340 lines of serpent -# Not yet tested - -# assets[i] = a registered asset, assets[i].holders[j] = former or current i-holder -data assets[2^50](creator, name, calldate, callprice, dividend_paid, holders[2^50], holdersCount) -data nextAssetId - -# holdersMap: holdersMap[addr][asset] = 1 if addr holds asset -data holdersMap[2^160][2^50] - -# balances[x][y] = how much of y x holds -data balances[2^160][2^50] - -# orders[a][b] = heap of indices to (c, d, e) -# = c offers to sell d units of a at a price of e units of b per 10^18 units -# of a -data orderbooks[2^50][2^50] - -# store of general order data -data orders[2^50](seller, asset_sold, quantity, price) -data ordersCount - -# data feeds -data feeds[2^50](owner, value) -data feedCount - -# heap -data heap -extern heap: [register, push, pop, top, size] - -data cfds[2^50](maker, acceptor, feed, asset, strike, leverage, min, max, maturity) -data cfdCount - -data bets[2^50](maker, acceptor, feed, asset, makerstake, acceptorstake, eqtest, maturity) -data betCount - -def init(): - heap = create('heap.se') - -# Add units (internal method) -def add(to, asset, value): - assert msg.sender == self - self.balances[to][asset] += value - # Add the holder to the holders list - if not self.holdersMap[to][asset]: - self.holdersMap[to][asset] = 1 - c = self.assets[asset].holdersCount - self.assets[asset].holders[c] = to - self.assets[asset].holdersCount = c + 1 - -# Register a new asset -def register_asset(q, name, calldate, callprice): - newid = self.nextAssetId - self.assets[newid].creator = msg.sender - self.assets[newid].name = name - self.assets[newid].calldate = calldate - self.assets[newid].callprice = callprice - self.assets[newid].holders[0] = msg.sender - self.assets[newid].holdersCount = 1 - self.balances[msg.sender][newid] = q - self.holdersMap[msg.sender][newid] = 1 - -# Send -def send(to, asset, value): - fromval = self.balances[msg.sender][asset] - if fromval >= value: - self.balances[msg.sender][asset] -= value - self.add(to, asset, value) - -# Order -def mkorder(selling, buying, quantity, price): - # Make sure you have enough to pay for the order - assert self.balances[msg.sender][selling] >= quantity: - # Try to match existing orders - o = orderbooks[buying][selling] - if not o: - o = self.heap.register() - orderbooks[selling][buying] = o - sz = self.heap.size(o) - invprice = 10^36 / price - while quantity > 0 and sz > 0: - orderid = self.heap.pop() - p = self.orders[orderid].price - if p > invprice: - sz = 0 - else: - q = self.orders[orderid].quantity - oq = min(q, quantity) - b = self.orders[orderid].seller - self.balances[msg.sender][selling] -= oq * p / 10^18 - self.add(msg.sender, buying, oq) - self.add(b, selling, oq * p / 10^18) - self.orders[orderid].quantity = q - oq - if oq == q: - self.orders[orderid].seller = 0 - self.orders[orderid].price = 0 - self.orders[orderid].asset_sold = 0 - quantity -= oq - sz -= 1 - assert quantity > 0 - # Make the order - c = self.ordersCount - self.orders[c].seller = msg.sender - self.orders[c].asset_sold = selling - self.orders[c].quantity = quantity - self.orders[c].price = price - self.ordersCount += 1 - # Add it to the heap - o = orderbooks[selling][buying] - if not o: - o = self.heap.register() - orderbooks[selling][buying] = o - self.balances[msg.sender][selling] -= quantity - self.heap.push(o, price, c) - return(c) - -def cancel_order(id): - if self.orders[id].seller == msg.sender: - self.orders[id].seller = 0 - self.orders[id].price = 0 - self.balances[msg.sender][self.orders[id].asset_sold] += self.orders[id].quantity - self.orders[id].quantity = 0 - self.orders[id].asset_sold = 0 - -def register_feed(): - c = self.feedCount - self.feeds[c].owner = msg.sender - self.feedCount = c + 1 - return(c) - -def set_feed(id, v): - if self.feeds[id].owner == msg.sender: - self.feeds[id].value = v - -def mk_cfd_offer(feed, asset, strike, leverage, min, max, maturity): - b = self.balances[msg.sender][asset] - req = max((strike - min) * leverage, (strike - max) * leverage) - assert b >= req - self.balances[msg.sender][asset] = b - req - c = self.cfdCount - self.cfds[c].maker = msg.sender - self.cfds[c].feed = feed - self.cfds[c].asset = asset - self.cfds[c].strike = strike - self.cfds[c].leverage = leverage - self.cfds[c].min = min - self.cfds[c].max = max - self.cfds[c].maturity = maturity - self.cfdCount = c + 1 - return(c) - -def accept_cfd_offer(c): - assert not self.cfds[c].acceptor and self.cfds[c].maker - asset = self.cfds[c].asset - strike = self.cfds[c].strike - min = self.cfds[c].min - max = self.cfds[c].max - leverage = self.cfds[c].leverage - b = self.balances[msg.sender][asset] - req = max((min - strike) * leverage, (max - strike) * leverage) - assert b >= req - self.balances[msg.sender][asset] = b - req - self.cfds[c].acceptor = msg.sender - self.cfds[c].maturity += block.timestamp - -def claim_cfd_offer(c): - asset = self.cfds[c].asset - strike = self.cfds[c].strike - min = self.cfds[c].min - max = self.cfds[c].max - leverage = self.cfds[c].leverage - v = self.feeds[self.cfds[c].feed].value - assert v <= min or v >= max or block.timestamp >= self.cfds[c].maturity - maker_req = max((strike - min) * leverage, (strike - max) * leverage) - acceptor_req = max((min - strike) * leverage, (max - strike) * leverage) - paydelta = (strike - v) * leverage - self.add(self.cfds[c].maker, asset, maker_req + paydelta) - self.add(self.cfds[c].acceptor, asset, acceptor_req - paydelta) - self.cfds[c].maker = 0 - self.cfds[c].acceptor = 0 - self.cfds[c].feed = 0 - self.cfds[c].asset = 0 - self.cfds[c].strike = 0 - self.cfds[c].leverage = 0 - self.cfds[c].min = 0 - self.cfds[c].max = 0 - self.cfds[c].maturity = 0 - -def withdraw_cfd_offer(c): - if self.cfds[c].maker == msg.sender and not self.cfds[c].acceptor: - asset = self.cfds[c].asset - strike = self.cfds[c].strike - min = self.cfds[c].min - max = self.cfds[c].max - leverage = self.cfds[c].leverage - maker_req = max((strike - min) * leverage, (strike - max) * leverage) - self.balances[self.cfds[c].maker][asset] += maker_req - self.cfds[c].maker = 0 - self.cfds[c].acceptor = 0 - self.cfds[c].feed = 0 - self.cfds[c].asset = 0 - self.cfds[c].strike = 0 - self.cfds[c].leverage = 0 - self.cfds[c].min = 0 - self.cfds[c].max = 0 - self.cfds[c].maturity = 0 - - -def mk_bet_offer(feed, asset, makerstake, acceptorstake, eqtest, maturity): - assert self.balances[msg.sender][asset] >= makerstake - c = self.betCount - self.bets[c].maker = msg.sender - self.bets[c].feed = feed - self.bets[c].asset = asset - self.bets[c].makerstake = makerstake - self.bets[c].acceptorstake = acceptorstake - self.bets[c].eqtest = eqtest - self.bets[c].maturity = maturity - self.balances[msg.sender][asset] -= makerstake - self.betCount = c + 1 - return(c) - -def accept_bet_offer(c): - assert self.bets[c].maker and not self.bets[c].acceptor - asset = self.bets[c].asset - acceptorstake = self.bets[c].acceptorstake - assert self.balances[msg.sender][asset] >= acceptorstake - self.balances[msg.sender][asset] -= acceptorstake - self.bets[c].acceptor = msg.sender - -def claim_bet_offer(c): - assert block.timestamp >= self.bets[c].maturity - v = self.feeds[self.bets[c].feed].value - totalstake = self.bets[c].makerstake + self.bets[c].acceptorstake - if v == self.bets[c].eqtest: - self.add(self.bets[c].maker, self.bets[c].asset, totalstake) - else: - self.add(self.bets[c].acceptor, self.bets[c].asset, totalstake) - self.bets[c].maker = 0 - self.bets[c].feed = 0 - self.bets[c].asset = 0 - self.bets[c].makerstake = 0 - self.bets[c].acceptorstake = 0 - self.bets[c].eqtest = 0 - self.bets[c].maturity = 0 - -def cancel_bet(c): - assert not self.bets[c].acceptor and msg.sender == self.bets[c].maker - self.balances[msg.sender][self.bets[c].asset] += self.bets[c].makerstake - self.bets[c].maker = 0 - self.bets[c].feed = 0 - self.bets[c].asset = 0 - self.bets[c].makerstake = 0 - self.bets[c].acceptorstake = 0 - self.bets[c].eqtest = 0 - self.bets[c].maturity = 0 - -def dividend(holder_asset, divvying_asset, ratio): - i = 0 - sz = self.assets[holder_asset].holdersCount - t = 0 - holders = array(sz) - payments = array(sz) - while i < sz: - holders[i] = self.assets[holder_asset].holders[i] - payments[i] = self.balances[holders[i]][holder_asset] * ratio / 10^18 - t += payments[i] - i += 1 - if self.balances[msg.sender][divvying_asset] >= t: - i = 0 - while i < sz: - self.add(holders[i], divvying_asset, payments[i]) - i += 1 - self.balances[msg.sender][divvying_asset] -= t diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/counterparty/heap.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/counterparty/heap.se deleted file mode 100644 index 4a43a3974..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/counterparty/heap.se +++ /dev/null @@ -1,69 +0,0 @@ -data heaps[2^50](owner, size, nodes[2^50](key, value)) -data heapIndex - -def register(): - i = self.heapIndex - self.heaps[i].owner = msg.sender - self.heapIndex = i + 1 - return(i) - -def push(heap, key, value): - assert msg.sender == self.heaps[heap].owner - sz = self.heaps[heap].size - self.heaps[heap].nodes[sz].key = key - self.heaps[heap].nodes[sz].value = value - k = sz + 1 - while k > 1: - bottom = self.heaps[heap].nodes[k].key - top = self.heaps[heap].nodes[k/2].key - if bottom < top: - tvalue = self.heaps[heap].nodes[k/2].value - bvalue = self.heaps[heap].nodes[k].value - self.heaps[heap].nodes[k].key = top - self.heaps[heap].nodes[k].value = tvalue - self.heaps[heap].nodes[k/2].key = bottom - self.heaps[heap].nodes[k/2].value = bvalue - k /= 2 - else: - k = 0 - self.heaps[heap].size = sz + 1 - -def pop(heap): - sz = self.heaps[heap].size - assert sz - prevtop = self.heaps[heap].nodes[1].value - self.heaps[heap].nodes[1].key = self.heaps[heap].nodes[sz].key - self.heaps[heap].nodes[1].value = self.heaps[heap].nodes[sz].value - self.heaps[heap].nodes[sz].key = 0 - self.heaps[heap].nodes[sz].value = 0 - top = self.heaps[heap].nodes[1].key - k = 1 - while k * 2 < sz: - bottom1 = self.heaps[heap].nodes[k * 2].key - bottom2 = self.heaps[heap].nodes[k * 2 + 1].key - if bottom1 < top and (bottom1 < bottom2 or k * 2 + 1 >= sz): - tvalue = self.heaps[heap].nodes[1].value - bvalue = self.heaps[heap].nodes[k * 2].value - self.heaps[heap].nodes[k].key = bottom1 - self.heaps[heap].nodes[k].value = bvalue - self.heaps[heap].nodes[k * 2].key = top - self.heaps[heap].nodes[k * 2].value = tvalue - k = k * 2 - elif bottom2 < top and bottom2 < bottom1 and k * 2 + 1 < sz: - tvalue = self.heaps[heap].nodes[1].value - bvalue = self.heaps[heap].nodes[k * 2 + 1].value - self.heaps[heap].nodes[k].key = bottom2 - self.heaps[heap].nodes[k].value = bvalue - self.heaps[heap].nodes[k * 2 + 1].key = top - self.heaps[heap].nodes[k * 2 + 1].value = tvalue - k = k * 2 + 1 - else: - k = sz - self.heaps[heap].size = sz - 1 - return(prevtop) - -def top(heap): - return(self.heaps[heap].nodes[1].value) - -def size(heap): - return(self.heaps[heap].size) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/crowdfund.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/crowdfund.se deleted file mode 100644 index 9fd1e0643..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/crowdfund.se +++ /dev/null @@ -1,53 +0,0 @@ -data campaigns[2^80](recipient, goal, deadline, contrib_total, contrib_count, contribs[2^50](sender, value)) - -def create_campaign(id, recipient, goal, timelimit): - if self.campaigns[id].recipient: - return(0) - self.campaigns[id].recipient = recipient - self.campaigns[id].goal = goal - self.campaigns[id].deadline = block.timestamp + timelimit - -def contribute(id): - # Update contribution total - total_contributed = self.campaigns[id].contrib_total + msg.value - self.campaigns[id].contrib_total = total_contributed - - # Record new contribution - sub_index = self.campaigns[id].contrib_count - self.campaigns[id].contribs[sub_index].sender = msg.sender - self.campaigns[id].contribs[sub_index].value = msg.value - self.campaigns[id].contrib_count = sub_index + 1 - - # Enough funding? - if total_contributed >= self.campaigns[id].goal: - send(self.campaigns[id].recipient, total_contributed) - self.clear(id) - return(1) - - # Expired? - if block.timestamp > self.campaigns[id].deadline: - i = 0 - c = self.campaigns[id].contrib_count - while i < c: - send(self.campaigns[id].contribs[i].sender, self.campaigns[id].contribs[i].value) - i += 1 - self.clear(id) - return(2) - -def progress_report(id): - return(self.campaigns[id].contrib_total) - -# Clearing function for internal use -def clear(id): - if self == msg.sender: - self.campaigns[id].recipient = 0 - self.campaigns[id].goal = 0 - self.campaigns[id].deadline = 0 - c = self.campaigns[id].contrib_count - self.campaigns[id].contrib_count = 0 - self.campaigns[id].contrib_total = 0 - i = 0 - while i < c: - self.campaigns[id].contribs[i].sender = 0 - self.campaigns[id].contribs[i].value = 0 - i += 1 diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/futarchy.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/futarchy.se deleted file mode 100644 index 0d68622ac..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/futarchy.se +++ /dev/null @@ -1,136 +0,0 @@ -# 0: current epoch -# 1: number of proposals -# 2: master currency -# 3: last winning market -# 4: last txid -# 5: long-term ema currency units purchased -# 6: last block when currency units purchased -# 7: ether allocated to last round -# 8: last block when currency units claimed -# 9: ether allocated to current round -# 1000+: [proposal address, market ID, totprice, totvolume] - -init: - # We technically have two levels of epoch here. We have - # one epoch of 1000, to synchronize with the 1000 epoch - # of the market, and then 100 of those epochs make a - # meta-epoch (I'll nominate the term "seculum") over - # which the futarchy protocol will take place - contract.storage[0] = block.number / 1000 - # The master currency of the futarchy. The futarchy will - # assign currency units to whoever the prediction market - # thinks will best increase the currency's value - master_currency = create('subcurrency.se') - contract.storage[2] = master_currency -code: - curepoch = block.number / 1000 - prevepoch = contract.storage[0] - if curepoch > prevepoch: - if (curepoch % 100) > 50: - # Collect price data - # We take an average over 50 subepochs to determine - # the price of each asset, weighting by volume to - # prevent abuse - contract.storage[0] = curepoch - i = 0 - numprop = contract.storage[1] - while i < numprop: - market = contract.storage[1001 + i * 4] - price = call(market, 2) - volume = call(market, 3) - contract.storage[1002 + i * 4] += price - contract.storage[1003 + i * 4] += volume * price - i += 1 - if (curepoch / 100) > (prevepoch / 100): - # If we are entering a new seculum, we determine the - # market with the highest total average price - best = 0 - bestmarket = 0 - besti = 0 - i = 0 - while i < numprop: - curtotprice = contract.storage[1002 + i * 4] - curvolume = contract.storage[1002 + i * 4] - curavgprice = curtotprice / curvolume - if curavgprice > best: - best = curavgprice - besti = i - bestmarket = contract.storage[1003 + i * 4] - i += 1 - # Reset the number of proposals to 0 - contract.storage[1] = 0 - # Reward the highest proposal - call(contract.storage[2], [best, 10^9, 0], 3) - # Record the winning market so we can later appropriately - # compensate the participants - contract.storage[2] = bestmarket - # The amount of ether allocated to the last round - contract.storage[7] = contract.storage[9] - # The amount of ether allocated to the next round - contract.storage[9] = contract.balance / 2 - # Make a proposal [0, address] - if msg.data[0] == 0 and curepoch % 100 < 50: - pid = contract.storage[1] - market = create('market.se') - c1 = create('subcurrency.se') - c2 = create('subcurrency.se') - call(market, [c1, c2], 2) - contract.storage[1000 + pid * 4] = msg.data[1] - contract.storage[1001 + pid * 4] = market - contract.storage[1] += 1 - # Claim ether [1, address] - # One unit of the first currency in the last round's winning - # market entitles you to a quantity of ether that was decided - # at the start of that epoch - elif msg.data[0] == 1: - first_subcurrency = call(contract.storage[2], 3) - # We ask the first subcurrency contract what the last transaction was. The - # way to make a claim is to send the amount of first currency units that - # you wish to claim with, and then immediately call this contract. For security - # it makes sense to set up a tx which sends both messages in sequence atomically - data = call(first_subcurrency, [], 0, 4) - from = data[0] - to = data[1] - value = data[2] - txid = data[3] - if txid > contract.storage[4] and to == contract.address: - send(to, contract.storage[7] * value / 10^9) - contract.storage[4] = txid - # Claim second currency [2, address] - # One unit of the second currency in the last round's winning - # market entitles you to one unit of the futarchy's master - # currency - elif msg.data[0] == 2: - second_subcurrency = call(contract.storage[2], 3) - data = call(first_subcurrency, [], 0, 4) - from = data[0] - to = data[1] - value = data[2] - txid = data[3] - if txid > contract.storage[4] and to == contract.address: - call(contract.storage[2], [to, value], 2) - contract.storage[4] = txid - # Purchase currency for ether (target releasing 10^9 units per seculum) - # Price starts off 1 eth for 10^9 units but increases hyperbolically to - # limit issuance - elif msg.data[0] == 3: - pre_ema = contract.storage[5] - post_ema = pre_ema + msg.value - pre_reserve = 10^18 / (10^9 + pre_ema / 10^9) - post_reserve = 10^18 / (10^9 + post_ema / 10^9) - call(contract.storage[2], [msg.sender, pre_reserve - post_reserve], 2) - last_sold = contract.storage[6] - contract.storage[5] = pre_ema * (100000 + last_sold - block.number) + msg.value - contract.storage[6] = block.number - # Claim all currencies as the ether miner of the current block - elif msg.data[0] == 2 and msg.sender == block.coinbase and block.number > contract.storage[8]: - i = 0 - numproposals = contract.storage[1] - while i < numproposals: - market = contract.storage[1001 + i * 3] - fc = call(market, 4) - sc = call(market, 5) - call(fc, [msg.sender, 1000], 2) - call(sc, [msg.sender, 1000], 2) - i += 1 - contract.storage[8] = block.number diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/heap.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/heap.se deleted file mode 100644 index 1bc442e6d..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/heap.se +++ /dev/null @@ -1,55 +0,0 @@ -# 0: size -# 1-n: elements - -init: - contract.storage[1000] = msg.sender -code: - # Only owner of the heap is allowed to modify it - if contract.storage[1000] != msg.sender: - stop - # push - if msg.data[0] == 0: - sz = contract.storage[0] - contract.storage[sz + 1] = msg.data[1] - k = sz + 1 - while k > 1: - bottom = contract.storage[k] - top = contract.storage[k/2] - if bottom < top: - contract.storage[k] = top - contract.storage[k/2] = bottom - k /= 2 - else: - k = 0 - contract.storage[0] = sz + 1 - # pop - elif msg.data[0] == 1: - sz = contract.storage[0] - if !sz: - return(0) - prevtop = contract.storage[1] - contract.storage[1] = contract.storage[sz] - contract.storage[sz] = 0 - top = contract.storage[1] - k = 1 - while k * 2 < sz: - bottom1 = contract.storage[k * 2] - bottom2 = contract.storage[k * 2 + 1] - if bottom1 < top and (bottom1 < bottom2 or k * 2 + 1 >= sz): - contract.storage[k] = bottom1 - contract.storage[k * 2] = top - k = k * 2 - elif bottom2 < top and bottom2 < bottom1 and k * 2 + 1 < sz: - contract.storage[k] = bottom2 - contract.storage[k * 2 + 1] = top - k = k * 2 + 1 - else: - k = sz - contract.storage[0] = sz - 1 - return(prevtop) - # top - elif msg.data[0] == 2: - return(contract.storage[1]) - # size - elif msg.data[0] == 3: - return(contract.storage[0]) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/market.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/market.se deleted file mode 100644 index 2303a0b60..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/market.se +++ /dev/null @@ -1,117 +0,0 @@ -# Creates a decentralized market between any two subcurrencies - -# Here, the first subcurrency is the base asset and the second -# subcurrency is the asset priced against the base asset. Hence, -# "buying" refers to trading the first for the second, and -# "selling" refers to trading the second for the first - -# storage 0: buy orders -# storage 1: sell orders -# storage 1000: first subcurrency -# storage 1001: last first subcurrency txid -# storage 2000: second subcurrency -# storage 2001: last second subcurrency txid -# storage 3000: current epoch -# storage 4000: price -# storage 4001: volume - -init: - # Heap for buy orders - contract.storage[0] = create('heap.se') - # Heap for sell orders - contract.storage[1] = create('heap.se') -code: - # Initialize with [ first_subcurrency, second_subcurrency ] - if !contract.storage[1000]: - contract.storage[1000] = msg.data[0] # First subcurrency - contract.storage[1001] = -1 - contract.storage[2000] = msg.data[1] # Second subcurrency - contract.storage[2001] = -1 - contract.storage[3000] = block.number / 1000 - stop - first_subcurrency = contract.storage[1000] - second_subcurrency = contract.storage[2000] - buy_heap = contract.storage[0] - sell_heap = contract.storage[1] - # This contract operates in "epochs" of 100 blocks - # At the end of each epoch, we process all orders - # simultaneously, independent of order. This algorithm - # prevents front-running, and generates a profit from - # the spread. The profit is permanently kept in the - # market (ie. destroyed), making both subcurrencies - # more valuable - - # Epoch transition code - if contract.storage[3000] < block.number / 100: - done = 0 - volume = 0 - while !done: - # Grab the top buy and sell order from each heap - topbuy = call(buy_heap, 1) - topsell = call(sell_heap, 1) - # An order is recorded in the heap as: - # Buys: (2^48 - 1 - price) * 2^208 + units of first currency * 2^160 + from - # Sells: price * 2^208 + units of second currency * 2^160 + from - buyprice = -(topbuy / 2^208) - buyfcvalue = (topbuy / 2^160) % 2^48 - buyer = topbuy % 2^160 - sellprice = topsell / 2^208 - sellscvalue = (topsell / 2^160) % 2^48 - seller = topsell % 2^160 - # Heap empty, or no more matching orders - if not topbuy or not topsell or buyprice < sellprice: - done = 1 - else: - # Add to volume counter - volume += buyfcvalue - # Calculate how much of the second currency the buyer gets, and - # how much of the first currency the seller gets - sellfcvalue = sellscvalue / buyprice - buyscvalue = buyfcvalue * sellprice - # Send the currency units along - call(second_subcurrency, [buyer, buyscvalue], 2) - call(first_subcurrency, [seller, sellfcvalue], 2) - if volume: - contract.storage[4000] = (buyprice + sellprice) / 2 - contract.storage[4001] = volume - contract.storage[3000] = block.number / 100 - # Make buy order [0, price] - if msg.data[0] == 0: - # We ask the first subcurrency contract what the last transaction was. The - # way to make a buy order is to send the amount of first currency units that - # you wish to buy with, and then immediately call this contract. For security - # it makes sense to set up a tx which sends both messages in sequence atomically - data = call(first_subcurrency, [], 0, 4) - from = data[0] - to = data[1] - value = data[2] - txid = data[3] - price = msg.data[1] - if txid > contract.storage[1001] and to == contract.address: - contract.storage[1001] = txid - # Adds the order to the heap - call(buy_heap, [0, -price * 2^208 + (value % 2^48) * 2^160 + from], 2) - # Make sell order [1, price] - elif msg.data[0] == 1: - # Same mechanics as buying - data = call(second_subcurrency, [], 0, 4) - from = data[0] - to = data[1] - value = data[2] - txid = data[3] - price = msg.data[1] - if txid > contract.storage[2001] and to == contract.address: - contract.storage[2001] = txid - call(sell_heap, [0, price * 2^208 + (value % 2^48) * 2^160 + from], 2) - # Ask for price - elif msg.data[0] == 2: - return(contract.storage[4000]) - # Ask for volume - elif msg.data[0] == 3: - return(contract.storage[1000]) - # Ask for first currency - elif msg.data[0] == 4: - return(contract.storage[2000]) - # Ask for second currency - elif msg.data[0] == 5: - return(contract.storage[4001]) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/subcurrency.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/subcurrency.se deleted file mode 100644 index 1501beff7..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/subcurrency.se +++ /dev/null @@ -1,35 +0,0 @@ -# Initialization -# Admin can issue and delete at will -init: - contract.storage[0] = msg.sender -code: - # If a message with one item is sent, that's a balance query - if msg.datasize == 1: - addr = msg.data[0] - return(contract.storage[addr]) - # If a message with two items [to, value] are sent, that's a transfer request - elif msg.datasize == 2: - from = msg.sender - fromvalue = contract.storage[from] - to = msg.data[0] - value = msg.data[1] - if fromvalue >= value and value > 0 and to > 4: - contract.storage[from] = fromvalue - value - contract.storage[to] += value - contract.storage[2] = from - contract.storage[3] = to - contract.storage[4] = value - contract.storage[5] += 1 - return(1) - return(0) - elif msg.datasize == 3 and msg.sender == contract.storage[0]: - # Admin can issue at will by sending a [to, value, 0] message - if msg.data[2] == 0: - contract.storage[msg.data[0]] += msg.data[1] - # Change admin [ newadmin, 0, 1 ] - # Set admin to 0 to disable administration - elif msg.data[2] == 1: - contract.storage[0] = msg.data[0] - # Fetch last transaction - else: - return([contract.storage[2], contract.storage[3], contract.storage[4], contract.storage[5]], 4) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/test.py b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/test.py deleted file mode 100644 index 301a4a845..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/cyberdyne/test.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import print_function -import pyethereum -t = pyethereum.tester -s = t.state() -# Create currencies -c1 = s.contract('subcurrency.se') -print("First currency: %s" % c1) -c2 = s.contract('subcurrency.se') -print("First currency: %s" % c2) -# Allocate units -s.send(t.k0, c1, 0, [t.a0, 1000, 0]) -s.send(t.k0, c1, 0, [t.a1, 1000, 0]) -s.send(t.k0, c2, 0, [t.a2, 1000000, 0]) -s.send(t.k0, c2, 0, [t.a3, 1000000, 0]) -print("Allocated units") -# Market -m = s.contract('market.se') -s.send(t.k0, m, 0, [c1, c2]) -# Place orders -s.send(t.k0, c1, 0, [m, 1000]) -s.send(t.k0, m, 0, [0, 1200]) -s.send(t.k1, c1, 0, [m, 1000]) -s.send(t.k1, m, 0, [0, 1400]) -s.send(t.k2, c2, 0, [m, 1000000]) -s.send(t.k2, m, 0, [1, 800]) -s.send(t.k3, c2, 0, [m, 1000000]) -s.send(t.k3, m, 0, [1, 600]) -print("Orders placed") -# Next epoch and ping -s.mine(100) -print("Mined 100") -s.send(t.k0, m, 0, []) -print("Updating") -# Check -assert s.send(t.k0, c2, 0, [t.a0]) == [800000] -assert s.send(t.k0, c2, 0, [t.a1]) == [600000] -assert s.send(t.k0, c1, 0, [t.a2]) == [833] -assert s.send(t.k0, c1, 0, [t.a3]) == [714] -print("Balance checks passed") diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/datafeed.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/datafeed.se deleted file mode 100644 index 4c4a56de8..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/datafeed.se +++ /dev/null @@ -1,12 +0,0 @@ -# Database updateable only by the original creator -data creator - -def init(): - self.creator = msg.sender - -def update(k, v): - if msg.sender == self.creator: - self.storage[k] = v - -def query(k): - return(self.storage[k]) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/ecrecover.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/ecrecover.se deleted file mode 100644 index ce28f58c2..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/ecrecover.se +++ /dev/null @@ -1,40 +0,0 @@ -# So I looked up on Wikipedia what Jacobian form actually is, and noticed that it's -# actually a rather different and more clever construction than the naive version -# that I created. It may possible to achieve a further 20-50% savings by applying -# that version. - -extern all: [call] - -data JORDANMUL -data JORDANADD -data EXP - -def init(): - self.JORDANMUL = create('jacobian_mul.se') - self.JORDANADD = create('jacobian_add.se') - self.EXP = create('modexp.se') - -def call(h, v, r, s): - N = -432420386565659656852420866394968145599 - P = -4294968273 - h = mod(h, N) - r = mod(r, P) - s = mod(s, N) - Gx = 55066263022277343669578718895168534326250603453777594175500187360389116729240 - Gy = 32670510020758816978083085130507043184471273380659243275938904335757337482424 - x = r - xcubed = mulmod(mulmod(x, x, P), x, P) - beta = self.EXP.call(addmod(xcubed, 7, P), div(P + 1, 4), P) - - # Static-gascost ghetto conditional - y_is_positive = mod(v, 2) xor mod(beta, 2) - y = beta * y_is_positive + (P - beta) * (1 - y_is_positive) - - GZ = self.JORDANMUL.call(Gx, 1, Gy, 1, N - h, outsz=4) - XY = self.JORDANMUL.call(x, 1, y, 1, s, outsz=4) - COMB = self.JORDANADD.call(GZ[0], GZ[1], GZ[2], GZ[3], XY[0], XY[1], XY[2], XY[3], 1, outsz=5) - COMB[4] = self.EXP.call(r, N - 2, N) - Q = self.JORDANMUL.call(data=COMB, datasz=5, outsz=4) - ox = mulmod(Q[0], self.EXP.call(Q[1], P - 2, P), P) - oy = mulmod(Q[2], self.EXP.call(Q[3], P - 2, P), P) - return([ox, oy], 2) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/ecrecover_compiled.evm b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/ecrecover_compiled.evm deleted file mode 100644 index f575fe70f..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/ecrecover_compiled.evm +++ /dev/null @@ -1 +0,0 @@ -6000607f535961071c80610013593961072f566000605f535961013d8061001359396101505661012b8061000e60003961013956600061023f5360003560001a6000141561012a5760806001602037602051151561002c576060511561002f565b60005b156100695760806080599059016000905260a052600060a051526001602060a05101526000604060a05101526001606060a051015260a051f25b6401000003d160000380608051826003846020516020510909098182600260605109836040516040510909828283098382830984858660026020510983098603866040518509088560405183098686888985604051098a038a85602051090809878689846040510909888960605183098a038a6080518509088960805183096080608059905901600090526101e052866101e051528560206101e05101528260406101e05101528160606101e05101526101e051f250505050505050505050505b5b6000f25b816000f090506000555961040680610168593961056e566000603f535961013d8061001359396101505661012b8061000e60003961013956600061023f5360003560001a6000141561012a5760806001602037602051151561002c576060511561002f565b60005b156100695760806080599059016000905260a052600060a051526001602060a05101526000604060a05101526001606060a051015260a051f25b6401000003d160000380608051826003846020516020510909098182600260605109836040516040510909828283098382830984858660026020510983098603866040518509088560405183098686888985604051098a038a85602051090809878689846040510909888960605183098a038a6080518509088960805183096080608059905901600090526101e052866101e051528560206101e05101528260406101e05101528160606101e05101526101e051f250505050505050505050505b5b6000f25b816000f0905060005561029a8061016860003961040256600061043f5360003560001a60001415610299576101006001602037602051151561002d5760605115610030565b60005b1561007657608059905901600090526101405260a051610140515260c051602061014051015260e051604061014051015261010051606061014051015261014051610120525b60a05115156100885760e0511561008b565b60005b156100d0576080599059016000905261016052602051610160515260405160206101605101526060516040610160510152608051606061016051015261016051610120525b61012051156100e157608061012051f25b6401000003d16000036000818260a0516040510983038360c051602051090814156101b1576000818260e051608051098303836101005160605109081415610175576080608080599059016000905260006101c0601f01536020516101e052604051610200526060516102205260805161024052818160816101c0601f01600060005460195a03f1508090509050f26101b0565b608060805990590160009052610280526000610280515260016020610280510152600060406102805101526001606061028051015261028051f25b5b808160405160c051098283610100516060510984038460805160e05109080981828360c0516020510984038460405160a051090883608051610100510909828283098382830984856020518309860386604051850908856040518309868760a051830988038860c0518509088760c051830988898a60405185098b038b8460205109088909898a836040510989098a8b60605183098c038c6080518509088b60805183096080608059905901600090526103e052866103e051528560206103e05101528260406103e05101528160606103e05101526103e051f2505050505050505050505050505b5b6000f25b816000f090506001556101928061058660003961071856600061013f5360003560001a600014156101915760a0600160203770014551231950b75fc4402da1732fc9bebf60000360a0510660a05260a05115606051156020511502011561007e5760806080599059016000905260c052600060c051526001602060c05101526000604060c05101526001606060c051015260c051f25b610120599059016000905260e052600060e051526000602060e05101526001604060e05101526000606060e05101526001608060e0510152600060a060e0510152600060c060e0510152600060e060e0510152600061010060e051015260e0517f80000000000000000000000000000000000000000000000000000000000000005b6000811115610187578060a0511615610165576080602083016081601f85016000600054614e20f15060205160a083015260405160c083015260605160e0830152608051610100830152608060208301610101601f85016000600154614e20f161017b565b6080602083016081601f85016000600054614e20f15b50600281049050610100565b608060208301f250505b5b6000f25b816000f0905060005559610406806107475939610b4d566000603f535961013d8061001359396101505661012b8061000e60003961013956600061023f5360003560001a6000141561012a5760806001602037602051151561002c576060511561002f565b60005b156100695760806080599059016000905260a052600060a051526001602060a05101526000604060a05101526001606060a051015260a051f25b6401000003d160000380608051826003846020516020510909098182600260605109836040516040510909828283098382830984858660026020510983098603866040518509088560405183098686888985604051098a038a85602051090809878689846040510909888960605183098a038a6080518509088960805183096080608059905901600090526101e052866101e051528560206101e05101528260406101e05101528160606101e05101526101e051f250505050505050505050505b5b6000f25b816000f0905060005561029a8061016860003961040256600061043f5360003560001a60001415610299576101006001602037602051151561002d5760605115610030565b60005b1561007657608059905901600090526101405260a051610140515260c051602061014051015260e051604061014051015261010051606061014051015261014051610120525b60a05115156100885760e0511561008b565b60005b156100d0576080599059016000905261016052602051610160515260405160206101605101526060516040610160510152608051606061016051015261016051610120525b61012051156100e157608061012051f25b6401000003d16000036000818260a0516040510983038360c051602051090814156101b1576000818260e051608051098303836101005160605109081415610175576080608080599059016000905260006101c0601f01536020516101e052604051610200526060516102205260805161024052818160816101c0601f01600060005460195a03f1508090509050f26101b0565b608060805990590160009052610280526000610280515260016020610280510152600060406102805101526001606061028051015261028051f25b5b808160405160c051098283610100516060510984038460805160e05109080981828360c0516020510984038460405160a051090883608051610100510909828283098382830984856020518309860386604051850908856040518309868760a051830988038860c0518509088760c051830988898a60405185098b038b8460205109088909898a836040510989098a8b60605183098c038c6080518509088b60805183096080608059905901600090526103e052866103e051528560206103e05101528260406103e05101528160606103e05101526103e051f2505050505050505050505050505b5b6000f25b816000f09050600155596100d080610b655939610c35566100be8061000e6000396100cc5660003560001a600014156100bd576060600160203760017f80000000000000000000000000000000000000000000000000000000000000005b60008111156100b157606051816040511615156020510a606051848509099150606051600282046040511615156020510a606051848509099150606051600482046040511615156020510a606051848509099150606051600882046040511615156020510a606051848509099150601081049050610038565b8160c052602060c0f250505b5b6000f25b816000f090506002556103d280610c4d60003961101f56600061095f5360003560001a600014156103d1576080600160203770014551231950b75fc4402da1732fc9bebf60000360a0526401000003d160000360c05260a0516020510660205260c0516060510660605260a051608051066080527f79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179860e0527f483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8610100526060516101205260c0516101205160c05161012051610120510909610140526000610180601f015360c051600761014051086101a0526004600160c05101046101c05260c0516101e05260206102006061610180601f01600060025460195a03f1506102005161016052600261016051066002604051061861022052610220516001036101605160c05103026102205161016051020161024052608080599059016000905260006102a0601f015360e0516102c05260016102e052610100516103005260016103205260205160a0510361034052818160a16102a0601f01600060005460195a03f150809050905061026052608080599059016000905260006103c0601f0153610120516103e052600161040052610240516104205260016104405260805161046052818160a16103c0601f01600060005460195a03f15080905090506103805260a080599059016000905260006104e0601f015361026051516105005260206102605101516105205260406102605101516105405260606102605101516105605261038051516105805260206103805101516105a05260406103805101516105c05260606103805101516105e05260016106005281816101216104e0601f01600060015460195a03f15080905090506104a0526000610640601f015360605161066052600260a051036106805260a0516106a05260206106c06061610640601f01600060025460195a03f1506106c05160806104a05101526104a05160208103805160018303608080599059016000905260008353818160a185600060005460195a03f150838552809050905090509050905090506106e05260c05160006107e0601f015360206106e051015161080052600260c051036108205260c05161084052602061086060616107e0601f01600060025460195a03f150610860516106e05151096107c05260c05160006108a0601f015360606106e05101516108c052600260c051036108e05260c05161090052602061092060616108a0601f01600060025460195a03f1506109205160406106e05101510961088052604060405990590160009052610940526107c051610940515261088051602061094051015261094051f25b5b6000f2 diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/jacobian_add.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/jacobian_add.se deleted file mode 100644 index 29dc390b2..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/jacobian_add.se +++ /dev/null @@ -1,32 +0,0 @@ -extern all: [call] -data DOUBLE - -def init(): - self.DOUBLE = create('jacobian_double.se') - -def call(axn, axd, ayn, ayd, bxn, bxd, byn, byd): - if !axn and !ayn: - o = [bxn, bxd, byn, byd] - if !bxn and !byn: - o = [axn, axd, ayn, ayd] - if o: - return(o, 4) - with P = -4294968273: - if addmod(mulmod(axn, bxd, P), P - mulmod(axd, bxn, P), P) == 0: - if addmod(mulmod(ayn, byd, P), P - mulmod(ayd, byn, P), P) == 0: - return(self.DOUBLE.call(axn, axd, ayn, ayd, outsz=4), 4) - else: - return([0, 1, 0, 1], 4) - with mn = mulmod(addmod(mulmod(byn, ayd, P), P - mulmod(ayn, byd, P), P), mulmod(bxd, axd, P), P): - with md = mulmod(mulmod(byd, ayd, P), addmod(mulmod(bxn, axd, P), P - mulmod(axn, bxd, P), P), P): - with msqn = mulmod(mn, mn, P): - with msqd = mulmod(md, md, P): - with msqman = addmod(mulmod(msqn, axd, P), P - mulmod(msqd, axn, P), P): - with msqmad = mulmod(msqd, axd, P): - with xn = addmod(mulmod(msqman, bxd, P), P - mulmod(msqmad, bxn, P), P): - with xd = mulmod(msqmad, bxd, P): - with mamxn = mulmod(mn, addmod(mulmod(axn, xd, P), P - mulmod(xn, axd, P), P), P): - with mamxd = mulmod(md, mulmod(axd, xd, P), P): - with yn = addmod(mulmod(mamxn, ayd, P), P - mulmod(mamxd, ayn, P), P): - with yd = mulmod(mamxd, ayd, P): - return([xn, xd, yn, yd], 4) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/jacobian_double.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/jacobian_double.se deleted file mode 100644 index b7d8221a6..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/jacobian_double.se +++ /dev/null @@ -1,16 +0,0 @@ -def call(axn, axd, ayn, ayd): - if !axn and !ayn: - return([0, 1, 0, 1], 4) - with P = -4294968273: - # No need to add (A, 1) because A = 0 for bitcoin - with mn = mulmod(mulmod(mulmod(axn, axn, P), 3, P), ayd, P): - with md = mulmod(mulmod(axd, axd, P), mulmod(ayn, 2, P), P): - with msqn = mulmod(mn, mn, P): - with msqd = mulmod(md, md, P): - with xn = addmod(mulmod(msqn, axd, P), P - mulmod(msqd, mulmod(axn, 2, P), P), P): - with xd = mulmod(msqd, axd, P): - with mamxn = mulmod(addmod(mulmod(axn, xd, P), P - mulmod(axd, xn, P), P), mn, P): - with mamxd = mulmod(mulmod(axd, xd, P), md, P): - with yn = addmod(mulmod(mamxn, ayd, P), P - mulmod(mamxd, ayn, P), P): - with yd = mulmod(mamxd, ayd, P): - return([xn, xd, yn, yd], 4) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/jacobian_mul.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/jacobian_mul.se deleted file mode 100644 index bf5b96bb4..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/jacobian_mul.se +++ /dev/null @@ -1,37 +0,0 @@ -# Expected gas cost -# -# def expect(n, point_at_infinity=False): -# n = n % (2**256 - 432420386565659656852420866394968145599) -# if point_at_infinity: -# return 79 -# if n == 0: -# return 34479 -# L = int(1 + math.log(n) / math.log(2)) -# H = len([x for x in b.encode(n, 2) if x == '1']) -# return 34221 + 94 * L + 343 * H - -data DOUBLE -data ADD - -def init(): - self.DOUBLE = create('jacobian_double.se') - self.ADD = create('jacobian_add.se') - -def call(axn, axd, ayn, ayd, n): - n = mod(n, -432420386565659656852420866394968145599) - if !axn * !ayn + !n: # Constant-gas version of !axn and !ayn or !n - return([0, 1, 0, 1], 4) - with o = [0, 0, 1, 0, 1, 0, 0, 0, 0]: - with b = 2 ^ 255: - while gt(b, 0): - if n & b: - ~call(20000, self.DOUBLE, 0, o + 31, 129, o + 32, 128) - o[5] = axn - o[6] = axd - o[7] = ayn - o[8] = ayd - ~call(20000, self.ADD, 0, o + 31, 257, o + 32, 128) - else: - ~call(20000, self.DOUBLE, 0, o + 31, 129, o + 32, 128) - b = div(b, 2) - return(o + 32, 4) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/modexp.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/modexp.se deleted file mode 100644 index 687b12a04..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/modexp.se +++ /dev/null @@ -1,11 +0,0 @@ -def call(b, e, m): - with o = 1: - with bit = 2 ^ 255: - while gt(bit, 0): - # A touch of loop unrolling for 20% efficiency gain - o = mulmod(mulmod(o, o, m), b ^ !(!(e & bit)), m) - o = mulmod(mulmod(o, o, m), b ^ !(!(e & div(bit, 2))), m) - o = mulmod(mulmod(o, o, m), b ^ !(!(e & div(bit, 4))), m) - o = mulmod(mulmod(o, o, m), b ^ !(!(e & div(bit, 8))), m) - bit = div(bit, 16) - return(o) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/substitutes.py b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/substitutes.py deleted file mode 100644 index 0007da0cf..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/substitutes.py +++ /dev/null @@ -1,78 +0,0 @@ -import bitcoin as b -import math -import sys - - -def signed(o): - return map(lambda x: x - 2**256 if x >= 2**255 else x, o) - - -def hamming_weight(n): - return len([x for x in b.encode(n, 2) if x == '1']) - - -def binary_length(n): - return len(b.encode(n, 2)) - - -def jacobian_mul_substitute(A, B, C, D, N): - if A == 0 and C == 0 or (N % b.N) == 0: - return {"gas": 86, "output": [0, 1, 0, 1]} - else: - output = b.jordan_multiply(((A, B), (C, D)), N) - return { - "gas": 35262 + 95 * binary_length(N % b.N) + 355 * hamming_weight(N % b.N), - "output": signed(list(output[0]) + list(output[1])) - } - - -def jacobian_add_substitute(A, B, C, D, E, F, G, H): - if A == 0 or E == 0: - gas = 149 - elif (A * F - B * E) % b.P == 0: - if (C * H - D * G) % b.P == 0: - gas = 442 - else: - gas = 177 - else: - gas = 301 - output = b.jordan_add(((A, B), (C, D)), ((E, F), (G, H))) - return { - "gas": gas, - "output": signed(list(output[0]) + list(output[1])) - } - - -def modexp_substitute(base, exp, mod): - return { - "gas": 5150, - "output": signed([pow(base, exp, mod) if mod > 0 else 0]) - } - - -def ecrecover_substitute(z, v, r, s): - P, A, B, N, Gx, Gy = b.P, b.A, b.B, b.N, b.Gx, b.Gy - x = r - beta = pow(x*x*x+A*x+B, (P + 1) / 4, P) - BETA_PREMIUM = modexp_substitute(x, (P + 1) / 4, P)["gas"] - y = beta if v % 2 ^ beta % 2 else (P - beta) - Gz = b.jordan_multiply(((Gx, 1), (Gy, 1)), (N - z) % N) - GZ_PREMIUM = jacobian_mul_substitute(Gx, 1, Gy, 1, (N - z) % N)["gas"] - XY = b.jordan_multiply(((x, 1), (y, 1)), s) - XY_PREMIUM = jacobian_mul_substitute(x, 1, y, 1, s % N)["gas"] - Qr = b.jordan_add(Gz, XY) - QR_PREMIUM = jacobian_add_substitute(Gz[0][0], Gz[0][1], Gz[1][0], Gz[1][1], - XY[0][0], XY[0][1], XY[1][0], XY[1][1] - )["gas"] - Q = b.jordan_multiply(Qr, pow(r, N - 2, N)) - Q_PREMIUM = jacobian_mul_substitute(Qr[0][0], Qr[0][1], Qr[1][0], Qr[1][1], - pow(r, N - 2, N))["gas"] - R_PREMIUM = modexp_substitute(r, N - 2, N)["gas"] - OX_PREMIUM = modexp_substitute(Q[0][1], P - 2, P)["gas"] - OY_PREMIUM = modexp_substitute(Q[1][1], P - 2, P)["gas"] - Q = b.from_jordan(Q) - return { - "gas": 991 + BETA_PREMIUM + GZ_PREMIUM + XY_PREMIUM + QR_PREMIUM + - Q_PREMIUM + R_PREMIUM + OX_PREMIUM + OY_PREMIUM, - "output": signed(Q) - } diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/test.py b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/test.py deleted file mode 100644 index 48d21e32f..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/ecc/test.py +++ /dev/null @@ -1,129 +0,0 @@ -import bitcoin as b -import random -import sys -import math -from pyethereum import tester as t -import substitutes -import time - -vals = [random.randrange(2**256) for i in range(12)] - -test_points = [list(p[0]) + list(p[1]) for p in - [b.jordan_multiply(((b.Gx, 1), (b.Gy, 1)), r) for r in vals]] - -G = [b.Gx, 1, b.Gy, 1] -Z = [0, 1, 0, 1] - - -def neg_point(p): - return [p[0], b.P - p[1], p[2], b.P - p[3]] - -s = t.state() -s.block.gas_limit = 10000000 -t.gas_limit = 1000000 - - -c = s.contract('modexp.se') -print "Starting modexp tests" - -for i in range(0, len(vals) - 2, 3): - o1 = substitutes.modexp_substitute(vals[i], vals[i+1], vals[i+2]) - o2 = s.profile(t.k0, c, 0, funid=0, abi=vals[i:i+3]) - #assert o1["gas"] == o2["gas"], (o1, o2) - assert o1["output"] == o2["output"], (o1, o2) - -c = s.contract('jacobian_add.se') -print "Starting addition tests" - -for i in range(2): - P = test_points[i * 2] - Q = test_points[i * 2 + 1] - NP = neg_point(P) - - o1 = substitutes.jacobian_add_substitute(*(P + Q)) - o2 = s.profile(t.k0, c, 0, funid=0, abi=P + Q) - #assert o1["gas"] == o2["gas"], (o1, o2) - assert o1["output"] == o2["output"], (o1, o2) - - o1 = substitutes.jacobian_add_substitute(*(P + NP)) - o2 = s.profile(t.k0, c, 0, funid=0, abi=P + NP) - #assert o1["gas"] == o2["gas"], (o1, o2) - assert o1["output"] == o2["output"], (o1, o2) - - o1 = substitutes.jacobian_add_substitute(*(P + P)) - o2 = s.profile(t.k0, c, 0, funid=0, abi=P + P) - #assert o1["gas"] == o2["gas"], (o1, o2) - assert o1["output"] == o2["output"], (o1, o2) - - o1 = substitutes.jacobian_add_substitute(*(P + Z)) - o2 = s.profile(t.k0, c, 0, funid=0, abi=P + Z) - #assert o1["gas"] == o2["gas"], (o1, o2) - assert o1["output"] == o2["output"], (o1, o2) - - o1 = substitutes.jacobian_add_substitute(*(Z + P)) - o2 = s.profile(t.k0, c, 0, funid=0, abi=Z + P) - #assert o1["gas"] == o2["gas"], (o1, o2) - assert o1["output"] == o2["output"], (o1, o2) - - -c = s.contract('jacobian_mul.se') -print "Starting multiplication tests" - - -mul_tests = [ - Z + [0], - Z + [vals[0]], - test_points[0] + [0], - test_points[1] + [b.N], - test_points[2] + [1], - test_points[2] + [2], - test_points[2] + [3], - test_points[2] + [4], - test_points[3] + [5], - test_points[3] + [6], - test_points[4] + [7], - test_points[4] + [2**254], - test_points[4] + [vals[1]], - test_points[4] + [vals[2]], - test_points[4] + [vals[3]], - test_points[5] + [2**256 - 1], -] - -for i, test in enumerate(mul_tests): - print 'trying mul_test %i' % i, test - o1 = substitutes.jacobian_mul_substitute(*test) - o2 = s.profile(t.k0, c, 0, funid=0, abi=test) - # assert o1["gas"] == o2["gas"], (o1, o2, test) - assert o1["output"] == o2["output"], (o1, o2, test) - -c = s.contract('ecrecover.se') -print "Starting ecrecover tests" - -for i in range(5): - print 'trying ecrecover_test', vals[i*2], vals[i*2+1] - k = vals[i*2] - h = vals[i*2+1] - V, R, S = b.ecdsa_raw_sign(b.encode(h, 256, 32), k) - aa = time.time() - o1 = substitutes.ecrecover_substitute(h, V, R, S) - print 'sub', time.time() - aa - a = time.time() - o2 = s.profile(t.k0, c, 0, funid=0, abi=[h, V, R, S]) - print time.time() - a - # assert o1["gas"] == o2["gas"], (o1, o2, h, V, R, S) - assert o1["output"] == o2["output"], (o1, o2, h, V, R, S) - -# Explicit tests - -data = [[ - 0xf007a9c78a4b2213220adaaf50c89a49d533fbefe09d52bbf9b0da55b0b90b60, - 0x1b, - 0x5228fc9e2fabfe470c32f459f4dc17ef6a0a81026e57e4d61abc3bc268fc92b5, - 0x697d4221cd7bc5943b482173de95d3114b9f54c5f37cc7f02c6910c6dd8bd107 -]] - -for datum in data: - o1 = substitutes.ecrecover_substitute(*datum) - o2 = s.profile(t.k0, c, 0, funid=0, abi=datum) - #assert o1["gas"] == o2["gas"], (o1, o2, datum) - assert o1["output"] == o2["output"], (o1, o2, datum) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/channel.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/channel.se deleted file mode 100644 index 733f4a95b..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/channel.se +++ /dev/null @@ -1,45 +0,0 @@ -if msg.data[0] == 0: - new_id = contract.storage[-1] - # store [from, to, value, maxvalue, timeout] in contract storage - contract.storage[new_id] = msg.sender - contract.storage[new_id + 1] = msg.data[1] - contract.storage[new_id + 2] = 0 - contract.storage[new_id + 3] = msg.value - contract.storage[new_id + 4] = 2^254 - # increment next id - contract.storage[-1] = new_id + 10 - # return id of this channel - return(new_id) - -# Increase payment on channel: [1, id, value, v, r, s] -elif msg.data[0] == 1: - # Ecrecover native extension; will be a different address in testnet and live - ecrecover = 0x46a8d0b21b1336d83b06829f568d7450df36883f - # Message data parameters - id = msg.data[1] % 2^160 - value = msg.data[2] - # Determine sender from signature - h = sha3([id, value], 2) - sender = call(ecrecover, [h, msg.data[3], msg.data[4], msg.data[5]], 4) - # Check sender matches and new value is greater than old - if sender == contract.storage[id]: - if value > contract.storage[id + 2] and value <= contract.storage[id + 3]: - # Update channel, increasing value and setting timeout - contract.storage[id + 2] = value - contract.storage[id + 4] = block.number + 1000 - -# Cash out channel: [2, id] -elif msg.data[0] == 2: - id = msg.data[1] % 2^160 - # Check if timeout has run out - if block.number >= contract.storage[id + 3]: - # Send funds - send(contract.storage[id + 1], contract.storage[id + 2]) - # Send refund - send(contract.storage[id], contract.storage[id + 3] - contract.storage[id + 2]) - # Clear storage - contract.storage[id] = 0 - contract.storage[id + 1] = 0 - contract.storage[id + 2] = 0 - contract.storage[id + 3] = 0 - contract.storage[id + 4] = 0 diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/map.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/map.se deleted file mode 100644 index 768dfb9fc..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/map.se +++ /dev/null @@ -1,19 +0,0 @@ -# An implementation of a contract for storing a key/value binding -init: - # Set owner - contract.storage[0] = msg.sender -code: - # Check ownership - if msg.sender == contract.storage[0]: - # Get: returns (found, val) - if msg.data[0] == 0: - s = sha3(msg.data[1]) - return([contract.storage[s], contract.storage[s+1]], 2) - # Set: sets map[k] = v - elif msg.data[0] == 1: - s = sha3(msg.data[1]) - contract.storage[s] = 1 - contract.storage[s + 1] = msg.data[2] - # Suicide - elif msg.data[2] == 1: - suicide(0) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/multiforward.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/multiforward.se deleted file mode 100644 index 577794d97..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/multiforward.se +++ /dev/null @@ -1,14 +0,0 @@ -init: - contract.storage[0] = msg.sender -code: - if msg.sender != contract.storage[0]: - stop - i = 0 - while i < ~calldatasize(): - to = ~calldataload(i) - value = ~calldataload(i+20) / 256^12 - datasize = ~calldataload(i+32) / 256^30 - data = alloc(datasize) - ~calldatacopy(data, i+34, datasize) - ~call(tx.gas - 25, to, value, data, datasize, 0, 0) - i += 34 + datasize diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/shadowchain.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/shadowchain.se deleted file mode 100644 index 1e466a355..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/eth15/shadowchain.se +++ /dev/null @@ -1,166 +0,0 @@ -# Exists in state: -# (i) last committed block -# (ii) chain of uncommitted blocks (linear only) -# (iii) transactions, each tx with an associated block number -# -# Uncommitted block = -# [ numtxs, numkvs, tx1 (N words), tx2 (N words) ..., [k1, v1], [k2, v2], [k3, v3] ... ] -# -# Block checking process -# -# Suppose last committed state is m -# Last uncommitted state is n -# Contested block is b -# -# 1. Temporarily apply all state transitions from -# m to b -# 2. Run code, get list of changes -# 3. Check is list of changes matches deltas -# * if yes, do nothing -# * if no, set last uncommitted state to pre-b -# -# Storage variables: -# -# Last committed block: 0 -# Last uncommitted block: 1 -# Contract holding code: 2 -# Uncommitted map: 3 -# Transaction length (parameter): 4 -# Block b: 2^160 + b * 2^40: -# + 1: submission blknum -# + 2: submitter -# + 3: data in uncommitted block format above -# Last committed storage: -# sha3(k): index k - -# Initialize: [0, c, txlength], set address of the code-holding contract and the transaction -# length -if not contract.storage[2]: - contract.storage[2] = msg.data[1] - contract.storage[4] = msg.data[2] - stop - -# Sequentially commit all uncommitted blocks that are more than 1000 mainchain-blocks old -last_committed_block = contract.storage[0] -last_uncommitted_block = contract.storage[1] -lcb_storage_index = 2^160 + last_committed_block * 2^40 -while contract.storage[lcb_storage_index + 1] < block.number - 1000 and last_committed_block < last_uncommitted_block: - kvpairs = contract.storage[lcb_storage_index] - i = 0 - while i < kvpairs: - k = contract.storage[lcb_storage_index + 3 + i * 2] - v = contract.storage[lcb_storage_index + 4 + i * 2] - contract.storage[sha3(k)] = v - i += 1 - last_committed_block += 1 - lcb_storage_index += 2^40 -contract.storage[0] = last_committed_block - - -# Propose block: [ 0, block number, data in block format above ... ] -if msg.data[0] == 0: - blknumber = msg.data[1] - # Block number must be correct - if blknumber != contract.storage[1]: - stop - # Deposit requirement - if msg.value < 10^19: - stop - # Store the proposal in storage as - # [ 0, main-chain block number, sender, block data...] - start_index = 2^160 + blknumber * 2^40 - numkvs = (msg.datasize - 2) / 2 - contract.storage[start_index + 1] = block.number - 1ontract.storage[start_index + 2] = msg.sender - i = 0 - while i < msg.datasize - 2: - contract.storage[start_index + 3 + i] = msg.data[2 + i] - i += 1 - contract.storage[1] = blknumber + 1 - -# Challenge block: [ 1, b ] -elif msg.data[0] == 1: - blknumber = msg.data[1] - txwidth = contract.storage[4] - last_uncommitted_block = contract.storage[1] - last_committed_block = contract.storage[0] - # Cannot challenge nonexistent or committed blocks - if blknumber <= last_uncommitted_block or blknumber > last_committed_block: - stop - # Create a contract to serve as a map that maintains keys and values - # temporarily - tempstore = create('map.se') - contract.storage[3] = tempstore - # Unquestioningly apply the state transitions from the last committed block - # up to b - b = last_committed_block - cur_storage_index = 2^160 + last_committed_block * 2^40 - while b < blknumber: - numtxs = contract.storage[cur_storage_index + 3] - numkvs = contract.storage[cur_storage_index + 4] - kv0index = cur_storage_index + 5 + numtxs * txwidth - i = 0 - while i < numkvs: - k = contract.storage[kv0index + i * 2] - v = contract.storage[kx0index + i * 2 + 1] - call(tempstore, [1, k, v], 3) - i += 1 - b += 1 - cur_storage_index += 2^40 - # Run the actual code, and see what state transitions it outputs - # The way that the code is expected to work is to: - # - # (1) take as input the list of transactions (the contract should - # use msg.datasize to determine how many txs there are, and it should - # be aware of the value of txwidth) - # (2) call this contract with [2, k] to read current state data - # (3) call this contract with [3, k, v] to write current state data - # (4) return as output a list of all state transitions that it made - # in the form [kvcount, k1, v1, k2, v2 ... ] - # - # The reason for separating (2) from (3) is that sometimes the state - # transition may end up changing a given key many times, and we don't - # need to inefficiently store that in storage - numkvs = contract.storage[cur_storage_index + 3] - numtxs = contract.storage[cur_storage_index + 4] - # Populate input array - inpwidth = numtxs * txwidth - inp = array(inpwidth) - i = 0 - while i < inpwidth: - inp[i] = contract.storage[cur_storage_index + 5 + i] - i += 1 - out = call(contract.storage[2], inp, inpwidth, numkvs * 2 + 1) - # Check that the number of state transitions is the same - if out[0] != kvcount: - send(msg.sender, 10^19) - contract.storage[0] = last_committed_block - stop - kv0index = cur_storage_index + 5 + numtxs * txwidth - i = 0 - while i < kvcount: - # Check that each individual state transition matches - k = contract.storage[kv0index + i * 2 + 1] - v = contract.storage[kv0index + i * 2 + 2] - if k != out[i * 2 + 1] or v != out[i * 2 + 2]: - send(msg.sender, 10^19) - contract.storage[0] = last_committed_block - stop - i += 1 - # Suicide tempstore - call(tempstore, 2) - - -# Read data [2, k] -elif msg.data[0] == 2: - tempstore = contract.storage[3] - o = call(tempstore, [0, msg.data[1]], 2, 2) - if o[0]: - return(o[1]) - else: - return contract.storage[sha3(msg.data[1])] - -# Write data [3, k, v] -elif msg.data[0] == 3: - tempstore = contract.storage[3] - call(tempstore, [1, msg.data[1], msg.data[2]], 3, 2) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/fixedpoint.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/fixedpoint.se deleted file mode 100644 index a8073c685..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/fixedpoint.se +++ /dev/null @@ -1,31 +0,0 @@ -type f: [a, b, c, d, e] - -macro f($a) + f($b): - f(add($a, $b)) - -macro f($a) - f($b): - f(sub($a, $b)) - -macro f($a) * f($b): - f(mul($a, $b) / 10000) - -macro f($a) / f($b): - f(sdiv($a * 10000, $b)) - -macro f($a) % f($b): - f(smod($a, $b)) - -macro f($v) = f($w): - $v = $w - -macro unfify(f($a)): - $a / 10000 - -macro fify($a): - f($a * 10000) - -a = fify(5) -b = fify(2) -c = a / b -e = c + (a / b) -return(unfify(e)) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/long_integer_macros.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/long_integer_macros.se deleted file mode 100644 index 58cdce6ab..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/long_integer_macros.se +++ /dev/null @@ -1,116 +0,0 @@ -macro smin($a, $b): - with $1 = $a: - with $2 = $b: - if(slt($1, $2), $1, $2) - -macro smax($a, $b): - with $1 = $a: - with $2 = $b: - if(slt($1, $2), $2, $1) - -def omul(x, y): - o = expose(mklong(x) * mklong(y)) - return(slice(o, 1), o[0]+1) - -def oadd(x, y): - o = expose(mklong(x) + mklong(y)) - return(slice(o, 1), o[0]+1) - -def osub(x, y): - o = expose(mklong(x) - mklong(y)) - return(slice(o, 1), o[0]+1) - -def odiv(x, y): - o = expose(mklong(x) / mklong(y)) - return(slice(o, 1), o[0]+1) - -def comb(a:a, b:a, sign): - sz = smax(a[0], b[0]) - msz = smin(a[0], b[0]) - c = array(sz + 2) - c[0] = sz - i = 0 - carry = 0 - while i < msz: - m = a[i + 1] + sign * b[i + 1] + carry - c[i + 1] = mod(m + 2^127, 2^128) - 2^127 - carry = (div(m + 2^127, 2^128) + 2^127) % 2^128 - 2^127 - i += 1 - u = if(a[0] > msz, a, b) - s = if(a[0] > msz, 1, sign) - while i < sz: - m = s * u[i + 1] + carry - c[i + 1] = mod(m + 2^127, 2^128) - 2^127 - carry = (div(m + 2^127, 2^128) + 2^127) % 2^128 - 2^127 - i += 1 - if carry: - c[0] += 1 - c[sz + 1] = carry - return(c, c[0]+1) - -def mul(a:a, b:a): - c = array(a[0] + b[0] + 2) - c[0] = a[0] + b[0] - i = 0 - while i < a[0]: - j = 0 - carry = 0 - while j < b[0]: - m = c[i + j + 1] + a[i + 1] * b[j + 1] + carry - c[i + j + 1] = mod(m + 2^127, 2^128) - 2^127 - carry = (div(m + 2^127, 2^128) + 2^127) % 2^128 - 2^127 - j += 1 - if carry: - c[0] = a[0] + b[0] + 1 - c[i + j + 1] += carry - i += 1 - return(c, c[0]+1) - -macro long($a) + long($b): - long(self.comb($a:$a[0]+1, $b:$b[0]+1, 1, outsz=$a[0]+$b[0]+2)) - -macro long($a) - long($b): - long(self.comb($a:$a[0]+1, $b:$b[0]+1, -1, outsz=$a[0]+$b[0]+2)) - -macro long($a) * long($b): - long(self.mul($a:$a[0]+1, $b:$b[0]+1, outsz=$a[0]+$b[0]+2)) - -macro long($a) / long($b): - long(self.div($a:$a[0]+1, $b:$b[0]+1, outsz=$a[0]+$b[0]+2)) - -macro mulexpand(long($a), $k, $m): - long: - with $c = array($a[0]+k+2): - $c[0] = $a[0]+$k - with i = 0: - while i < $a[0]: - v = $a[i+1] * $m + $c[i+$k+1] - $c[i+$k+1] = mod(v + 2^127, 2^128) - 2^127 - $c[i+$k+2] = div(v + 2^127, 2^128) - i += 1 - $c - -def div(a:a, b:a): - asz = a[0] - bsz = b[0] - while b[bsz] == 0 and bsz > 0: - bsz -= 1 - c = array(asz+2) - c[0] = asz+1 - while 1: - while a[asz] == 0 and asz > 0: - asz -= 1 - if asz < bsz: - return(c, c[0]+1) - sub = expose(mulexpand(long(b), asz - bsz, a[asz] / b[bsz])) - c[asz - bsz+1] = a[asz] / b[bsz] - a = expose(long(a) - long(sub)) - a[asz-1] += 2^128 * a[asz] - a[asz] = 0 - -macro mklong($i): - long([2, mod($i + 2^127, 2^128) - 2^127, div($i + 2^127, 2^128)]) - -macro expose(long($i)): - $i - diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/mul2.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/mul2.se deleted file mode 100644 index 65adff1e6..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/mul2.se +++ /dev/null @@ -1,2 +0,0 @@ -def double(v): - return(v*2) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/mutuala.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/mutuala.se deleted file mode 100644 index 3efb0edeb..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/mutuala.se +++ /dev/null @@ -1,187 +0,0 @@ -# mutuala - subcurrency - -# We want to issue a currency that reduces in value as you store it through negative interest. -# That negative interest would be stored in a commons account. It's like the p2p version of a -# capital tax - -# the same things goes for transactions - you pay as you use the currency. However, the more -# you pay, the more you get to say about what the tax is used for - -# each participant can propose a recipient for a payout to be made out of the commons account, -# others can vote on it by awarding it tax_credits. - -# TODO should proposal have expiration timestamp?, after which the tax_credits are refunded -# TODO multiple proposals can take more credits that available in the Commons, how to handle this -# TODO how to handle lost accounts, after which no longer possible to get 2/3 majority - -shared: - COMMONS = 42 - ADMIN = 666 - CAPITAL_TAX_PER_DAY = 7305 # 5% per year - PAYMENT_TAX = 20 # 5% - - ACCOUNT_LIST_OFFSET = 2^160 - ACCOUNT_MAP_OFFSET = 2^161 - PROPOSAL_LIST_OFFSET = 2^162 - PROPOSAL_MAP_OFFSET = 2^163 - -init: - contract.storage[ADMIN] = msg.sender - contract.storage[ACCOUNT_LIST_OFFSET - 1] = 1 - contract.storage[ACCOUNT_LIST_OFFSET] = msg.sender - contract.storage[ACCOUNT_MAP_OFFSET + msg.sender] = 10^12 - contract.storage[ACCOUNT_MAP_OFFSET + msg.sender + 1] = block.timestamp - -# contract.storage[COMMONS] = balance commons - -# contract.storage[ACCOUNT_LIST_OFFSET - 1] = number of accounts -# contract.storage[ACCOUNT_LIST_OFFSET + n] = account n - -# contract.storage[PROPOSAL_LIST_OFFSET - 1] contains the number of proposals -# contract.storage[PROPOSAL_LIST_OFFSET + n] = proposal n - -# per account: -# contract.storage[ACCOUNT_MAP_OFFSET + account] = balance -# contract.storage[ACCOUNT_MAP_OFFSET + account+1] = timestamp_last_transaction -# contract.storage[ACCOUNT_MAP_OFFSET + account+2] = tax_credits - -# per proposal: -# contract.storage[PROPOSAL_MAP_OFFSET + proposal_id] = recipient -# contract.storage[PROPOSAL_MAP_OFFSET + proposal_id+1] = amount -# contract.storage[PROPOSAL_MAP_OFFSET + proposal_id+2] = total vote credits - -code: - if msg.data[0] == "suicide" and msg.sender == contract.storage[ADMIN]: - suicide(msg.sender) - - elif msg.data[0] == "balance": - addr = msg.data[1] - return(contract.storage[ACCOUNT_MAP_OFFSET + addr]) - - elif msg.data[0] == "pay": - from = msg.sender - fromvalue = contract.storage[ACCOUNT_MAP_OFFSET + from] - to = msg.data[1] - if to == 0 or to >= 2^160: - return([0, "invalid address"], 2) - value = msg.data[2] - tax = value / PAYMENT_TAX - - if fromvalue >= value + tax: - contract.storage[ACCOUNT_MAP_OFFSET + from] = fromvalue - (value + tax) - contract.storage[ACCOUNT_MAP_OFFSET + to] += value - # tax - contract.storage[COMMONS] += tax - contract.storage[ACCOUNT_MAP_OFFSET + from + 2] += tax - - # check timestamp field to see if target account exists - if contract.storage[ACCOUNT_MAP_OFFSET + to + 1] == 0: - # register new account - nr_accounts = contract.storage[ACCOUNT_LIST_OFFSET - 1] - contract.storage[ACCOUNT_LIST_OFFSET + nr_accounts] = to - contract.storage[ACCOUNT_LIST_OFFSET - 1] += 1 - contract.storage[ACCOUNT_MAP_OFFSET + to + 1] = block.timestamp - - return(1) - else: - return([0, "insufficient balance"], 2) - - elif msg.data[0] == "hash": - proposal_id = sha3(msg.data[1]) - return(proposal_id) - - elif msg.data[0] == "propose": - from = msg.sender - # check if sender has an account and has tax credits - if contract.storage[ACCOUNT_MAP_OFFSET + from + 2] == 0: - return([0, "sender has no tax credits"], 2) - - proposal_id = sha3(msg.data[1]) - # check if proposal doesn't already exist - if contract.storage[PROPOSAL_MAP_OFFSET + proposal_id]: - return([0, "proposal already exists"]) - - to = msg.data[2] - # check if recipient is a valid address and has an account (with timestamp) - if to == 0 or to >= 2^160: - return([0, "invalid address"], 2) - if contract.storage[ACCOUNT_MAP_OFFSET + to + 1] == 0: - return([0, "invalid to account"], 2) - - value = msg.data[3] - # check if there is enough money in the commons account - if value > contract.storage[COMMONS]: - return([0, "not enough credits in commons"], 2) - - # record proposal in list - nr_proposals = contract.storage[PROPOSAL_LIST_OFFSET - 1] - contract.storage[PROPOSAL_LIST_OFFSET + nr_proposals] = proposal_id - contract.storage[PROPOSAL_LIST_OFFSET - 1] += 1 - - # record proposal in map - contract.storage[PROPOSAL_MAP_OFFSET + proposal_id] = to - contract.storage[PROPOSAL_MAP_OFFSET + proposal_id + 1] = value - - return(proposal_id) - - elif msg.data[0] == "vote": - from = msg.sender - proposal_id = sha3(msg.data[1]) - value = msg.data[2] - # check if sender has an account and has tax credits - if value < contract.storage[ACCOUNT_MAP_OFFSET + from + 2]: - return([0, "sender doesn't have enough tax credits"], 2) - - # check if proposal exist - if contract.storage[PROPOSAL_MAP_OFFSET + proposal_id] == 0: - return([0, "proposal doesn't exist"], 2) - - # increase votes - contract.storage[PROPOSAL_MAP_OFFSET + proposal_id + 2] += value - # withdraw tax credits - contract.storage[ACCOUNT_MAP_OFFSET + from + 2] -= value - - # did we reach 2/3 threshold? - if contract.storage[PROPOSAL_MAP_OFFSET + proposal_id + 2] >= contract.storage[COMMONS] * 2 / 3: - # got majority - to = contract.storage[PROPOSAL_MAP_OFFSET + proposal_id] - amount = contract.storage[PROPOSAL_MAP_OFFSET + proposal_id + 1] - - # adjust balances - contract.storage[ACCOUNT_MAP_OFFSET + to] += amount - contract.storage[COMMONS] -= amount - - # reset proposal - contract.storage[PROPOSAL_MAP_OFFSET + proposal_id] = 0 - contract.storage[PROPOSAL_MAP_OFFSET + proposal_id + 1] = 0 - contract.storage[PROPOSAL_MAP_OFFSET + proposal_id + 2] = 0 - return(1) - - return(proposal_id) - - elif msg.data[0] == "tick": - nr_accounts = contract.storage[ACCOUNT_LIST_OFFSET - 1] - account_idx = 0 - tax_paid = 0 - # process all accounts and see if they have to pay their daily capital tax - while account_idx < nr_accounts: - cur_account = contract.storage[ACCOUNT_LIST_OFFSET + account_idx] - last_timestamp = contract.storage[ACCOUNT_MAP_OFFSET + cur_account + 1] - time_diff = block.timestamp - last_timestamp - if time_diff >= 86400: - tax_days = time_diff / 86400 - balance = contract.storage[ACCOUNT_MAP_OFFSET + cur_account] - tax = tax_days * (balance / CAPITAL_TAX_PER_DAY) - if tax > 0: - # charge capital tax, but give tax credits in return - contract.storage[ACCOUNT_MAP_OFFSET + cur_account] -= tax - contract.storage[ACCOUNT_MAP_OFFSET + cur_account + 1] += tax_days * 86400 - contract.storage[ACCOUNT_MAP_OFFSET + cur_account + 2] += tax - - contract.storage[COMMONS] += tax - tax_paid += 1 - account_idx += 1 - return(tax_paid) # how many accounts did we charge tax on - - else: - return([0, "unknown command"], 2) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/namecoin.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/namecoin.se deleted file mode 100644 index 11d6274ae..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/namecoin.se +++ /dev/null @@ -1,7 +0,0 @@ -def register(k, v): - if !self.storage[k]: # Is the key not yet taken? - # Then take it! - self.storage[k] = v - return(1) - else: - return(0) // Otherwise do nothing diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/peano.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/peano.se deleted file mode 100644 index 979854444..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/peano.se +++ /dev/null @@ -1,43 +0,0 @@ -macro padd($x, psuc($y)): - psuc(padd($x, $y)) - -macro padd($x, z()): - $x - -macro dec(psuc($x)): - dec($x) + 1 - -macro dec(z()): - 0 - -macro pmul($x, z()): - z() - -macro pmul($x, psuc($y)): - padd(pmul($x, $y), $x) - -macro pexp($x, z()): - one() - -macro pexp($x, psuc($y)): - pmul($x, pexp($x, $y)) - -macro fac(z()): - one() - -macro fac(psuc($x)): - pmul(psuc($x), fac($x)) - -macro one(): - psuc(z()) - -macro two(): - psuc(psuc(z())) - -macro three(): - psuc(psuc(psuc(z()))) - -macro five(): - padd(three(), two()) - -return([dec(pmul(three(), pmul(three(), three()))), dec(fac(five()))], 2) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/returnten.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/returnten.se deleted file mode 100644 index 7969c9eb8..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/returnten.se +++ /dev/null @@ -1,4 +0,0 @@ -extern mul2: [double] - -x = create("mul2.se") -return(x.double(5)) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/quicksort.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/quicksort.se deleted file mode 100644 index be5d97fc7..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/quicksort.se +++ /dev/null @@ -1,33 +0,0 @@ -def kall(): - argcount = ~calldatasize() / 32 - if argcount == 1: - return(~calldataload(1)) - - args = array(argcount) - ~calldatacopy(args, 1, argcount * 32) - low = array(argcount) - lsz = 0 - high = array(argcount) - hsz = 0 - i = 1 - while i < argcount: - if args[i] < args[0]: - low[lsz] = args[i] - lsz += 1 - else: - high[hsz] = args[i] - hsz += 1 - i += 1 - low = self.kall(data=low, datasz=lsz, outsz=lsz) - high = self.kall(data=high, datasz=hsz, outsz=hsz) - o = array(argcount) - i = 0 - while i < lsz: - o[i] = low[i] - i += 1 - o[lsz] = args[0] - j = 0 - while j < hsz: - o[lsz + 1 + j] = high[j] - j += 1 - return(o, argcount) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/quicksort_pairs.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/quicksort_pairs.se deleted file mode 100644 index 0e603a238..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/quicksort_pairs.se +++ /dev/null @@ -1,46 +0,0 @@ -# Quicksort pairs -# eg. input of the form [ 30, 1, 90, 2, 70, 3, 50, 4] -# outputs [ 30, 1, 50, 4, 70, 3, 90, 2 ] -# -# Note: this can be used as a generalized sorting algorithm: -# map every object to [ key, ref ] where `ref` is the index -# in memory to all of the properties and `key` is the key to -# sort by - - -def kall(): - argcount = ~calldatasize() / 64 - if argcount == 1: - return([~calldataload(1), ~calldataload(33)], 2) - - args = array(argcount * 2) - ~calldatacopy(args, 1, argcount * 64) - low = array(argcount * 2) - lsz = 0 - high = array(argcount * 2) - hsz = 0 - i = 2 - while i < argcount * 2: - if args[i] < args[0]: - low[lsz] = args[i] - low[lsz + 1] = args[i + 1] - lsz += 2 - else: - high[hsz] = args[i] - high[hsz + 1] = args[i + 1] - hsz += 2 - i = i + 2 - low = self.kall(data=low, datasz=lsz, outsz=lsz) - high = self.kall(data=high, datasz=hsz, outsz=hsz) - o = array(argcount * 2) - i = 0 - while i < lsz: - o[i] = low[i] - i += 1 - o[lsz] = args[0] - o[lsz + 1] = args[1] - j = 0 - while j < hsz: - o[lsz + 2 + j] = high[j] - j += 1 - return(o, argcount * 2) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/schellingcoin.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/schellingcoin.se deleted file mode 100644 index a7d7da9c5..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/schellingcoin.se +++ /dev/null @@ -1,94 +0,0 @@ -# SchellingCoin implementation -# -# Epoch length: 100 blocks -# Target savings depletion rate: 0.1% per epoch - -data epoch -data hashes_submitted -data output -data quicksort_pairs -data accounts[2^160] -data submissions[2^80](hash, deposit, address, value) -extern any: [call] - - -def init(): - self.epoch = block.number / 100 - self.quicksort_pairs = create('quicksort_pairs.se') - -def any(): - if block.number / 100 > epoch: - # Sort all values submitted - N = self.hashes_submitted - o = array(N * 2) - i = 0 - j = 0 - while i < N: - v = self.submissions[i].value - if v: - o[j] = v - o[j + 1] = i - j += 2 - i += 1 - values = self.quicksort_pairs.call(data=o, datasz=j, outsz=j) - - # Calculate total deposit, refund non-submitters and - # cleanup - - deposits = array(j / 2) - addresses = array(j / 2) - - i = 0 - total_deposit = 0 - while i < j / 2: - base_index = HASHES + values[i * 2 + 1] * 3 - deposits[i] = self.submissions[i].deposit - addresses[i] = self.submissions[i].address - if self.submissions[values[i * 2 + 1]].value: - total_deposit += deposits[i] - else: - send(addresses[i], deposits[i] * 999 / 1000) - i += 1 - - inverse_profit_ratio = total_deposit / (contract.balance / 1000) + 1 - - # Reward everyone - i = 0 - running_deposit_sum = 0 - halfway_passed = 0 - while i < j / 2: - new_deposit_sum = running_deposit_sum + deposits[i] - if new_deposit_sum > total_deposit / 4 and running_deposit_sum < total_deposit * 3 / 4: - send(addresses[i], deposits[i] + deposits[i] / inverse_profit_ratio * 2) - else: - send(addresses[i], deposits[i] - deposits[i] / inverse_profit_ratio) - - if not halfway_passed and new_deposit_sum > total_deposit / 2: - self.output = self.submissions[i].value - halfway_passed = 1 - self.submissions[i].value = 0 - running_deposit_sum = new_deposit_sum - i += 1 - self.epoch = block.number / 100 - self.hashes_submitted = 0 - -def submit_hash(h): - if block.number % 100 < 50: - cur = self.hashes_submitted - pos = HASHES + cur * 3 - self.submissions[cur].hash = h - self.submissions[cur].deposit = msg.value - self.submissions[cur].address = msg.sender - self.hashes_submitted = cur + 1 - return(cur) - -def submit_value(index, v): - if sha3([msg.sender, v], 2) == self.submissions[index].hash: - self.submissions[index].value = v - return(1) - -def request_balance(): - return(contract.balance) - -def request_output(): - return(self.output) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/schellingdollar.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/schellingdollar.se deleted file mode 100644 index a34f42ce2..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellingcoin/schellingdollar.se +++ /dev/null @@ -1,171 +0,0 @@ -# Hedged zero-supply dollar implementation -# Uses SchellingCoin as price-determining backend -# -# Stored variables: -# -# 0: Schelling coin contract -# 1: Last epoch -# 2: Genesis block of contract -# 3: USD exposure -# 4: ETH exposure -# 5: Cached price -# 6: Last interest rate -# 2^160 + k: interest rate accumulator at k epochs -# 2^161 + ADDR * 3: eth-balance of a particular address -# 2^161 + ADDR * 3 + 1: usd-balance of a particular address -# 2^161 + ADDR * 3 + 1: last accessed epoch of a particular address -# -# Transaction types: -# -# [1, to, val]: send ETH -# [2, to, val]: send USD -# [3, wei_amount]: convert ETH to USD -# [4, usd_amount]: converts USD to ETH -# [5]: deposit -# [6, amount]: withdraw -# [7]: my balance query -# [7, acct]: balance query for any acct -# [8]: global state query -# [9]: liquidation test any account -# -# The purpose of the contract is to serve as a sort of cryptographic -# bank account where users can store both ETH and USD. ETH must be -# stored in zero or positive quantities, but USD balances can be -# positive or negative. If the USD balance is negative, the invariant -# usdbal * 10 >= ethbal * 9 must be satisfied; if any account falls -# below this value, then that account's balances are zeroed. Note -# that there is a 2% bounty to ping the app if an account does go -# below zero; one weakness is that if no one does ping then it is -# quite possible for accounts to go negative-net-worth, then zero -# themselves out, draining the reserves of the "bank" and potentially -# bankrupting it. A 0.1% fee on ETH <-> USD trade is charged to -# minimize this risk. Additionally, the bank itself will inevitably -# end up with positive or negative USD exposure; to mitigate this, -# it automatically updates interest rates on USD to keep exposure -# near zero. - -data schelling_coin -data last_epoch -data starting_block -data usd_exposure -data eth_exposure -data price -data last_interest_rate -data interest_rate_accum[2^50] -data accounts[2^160](eth, usd, last_epoch) - -extern sc: [submit_hash, submit_value, request_balance, request_output] - -def init(): - self.schelling_coin = create('schellingcoin.se') - self.price = self.schelling_coin.request_output() - self.interest_rate_accum[0] = 10^18 - self.starting_block = block.number - -def any(): - sender = msg.sender - epoch = (block.number - self.starting_block) / 100 - last_epoch = self.last_epoch - usdprice = self.price - - # Update contract epochs - if epoch > last_epoch: - delta = epoch - last_epoch - last_interest_rate = self.last_interest_rate - usd_exposure - self.usd_exposure - last_accum = self.interest_rate_accum[last_epoch] - - if usd_exposure < 0: - self.last_interest_rate = last_interest_rate - 10000 * delta - elif usd_exposure > 0: - self.last_interest_rate = last_interest_rate + 10000 * delta - - self.interest_rate_accum[epoch] = last_accum + last_accum * last_interest_rate * delta / 10^9 - - # Proceeds go to support the SchellingCoin feeding it price data, ultimately providing the depositors - # of the SchellingCoin an interest rate - bal = max(self.balance - self.eth_exposure, 0) / 10000 - usdprice = self.schelling_coin.request_output() - self.price = usdprice - self.last_epoch = epoch - - ethbal = self.accounts[msg.sender].eth - usdbal = self.accounts[msg.sender].usd - - # Apply interest rates to sender and liquidation-test self - if msg.sender != self: - self.ping(self) - -def send_eth(to, value): - if value > 0 and value <= ethbal and usdbal * usdprice * 2 + (ethbal - value) >= 0: - self.accounts[msg.sender].eth = ethbal - value - self.ping(to) - self.accounts[to].eth += value - return(1) - -def send_usd(to, value): - if value > 0 and value <= usdbal and (usdbal - value) * usdprice * 2 + ethbal >= 0: - self.accounts[msg.sender].usd = usdbal - value - self.ping(to) - self.accounts[to].usd += value - return(1) - -def convert_to_eth(usdvalue): - ethplus = usdvalue * usdprice * 999 / 1000 - if usdvalue > 0 and (usdbal - usdvalue) * usdprice * 2 + (ethbal + ethplus) >= 0: - self.accounts[msg.sender].eth = ethbal + ethplus - self.accounts[msg.sender].usd = usdbal - usdvalue - self.eth_exposure += ethplus - self.usd_exposure -= usdvalue - return([ethbal + ethplus, usdbal - usdvalue], 2) - -def convert_to_usd(ethvalue): - usdplus = ethvalue / usdprice * 999 / 1000 - if ethvalue > 0 and (usdbal + usdplus) * usdprice * 2 + (ethbal - ethvalue) >= 0: - self.accounts[msg.sender].eth = ethbal - ethvalue - self.accounts[msg.sender].usd = usdbal + usdplus - self.eth_exposure -= ethvalue - self.usd_exposure += usdplus - return([ethbal - ethvalue, usdbal + usdplus], 2) - -def deposit(): - self.accounts[msg.sender].eth = ethbal + msg.value - self.eth_exposure += msg.value - return(ethbal + msg.value) - -def withdraw(value): - if value > 0 and value <= ethbal and usdbal * usdprice * 2 + (ethbal - value) >= 0: - self.accounts[msg.sender].eth -= value - self.eth_exposure -= value - return(ethbal - value) - -def balance(acct): - self.ping(acct) - return([self.accounts[acct].eth, self.accounts[acct].usd], 2) - -def global_state_query(acct): - interest = self.last_interest_rate - usd_exposure = self.usd_exposure - eth_exposure = self.eth_exposure - eth_balance = self.balance - return([epoch, usdprice, interest, usd_exposure, eth_exposure, eth_balance], 6) - -def ping(acct): - account_last_epoch = self.accounts[acct].last_epoch - if account_last_epoch != epoch: - cur_usd_balance = self.accounts[acct].usd - new_usd_balance = cur_usd_balance * self.interest_rate_accum[epoch] / self.interest_rate_accum[account_last_epoch] - self.accounts[acct].usd = new_usd_balance - self.accounts[acct].last_epoch = epoch - self.usd_exposure += new_usd_balance - cur_usd_balance - - ethbal = self.accounts[acct].eth - - if new_usd_balance * usdval * 10 + ethbal * 9 < 0: - self.accounts[acct].eth = 0 - self.accounts[acct].usd = 0 - self.accounts[msg.sender].eth += ethbal / 50 - self.eth_exposure += -ethbal + ethbal / 50 - self.usd_exposure += new_usd_balance - return(1) - return(0) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellinghelper.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellinghelper.se deleted file mode 100644 index 0e522d6e8..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/schellinghelper.se +++ /dev/null @@ -1 +0,0 @@ -return(sha3([msg.sender, msg.data[0]], 2)) diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/short_namecoin.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/short_namecoin.se deleted file mode 100644 index db327a77d..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/short_namecoin.se +++ /dev/null @@ -1,3 +0,0 @@ -def register(k, v): - if !self.storage[k]: - self.storage[k] = v diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/subcurrency.se b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/subcurrency.se deleted file mode 100644 index fbda822b6..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/examples/subcurrency.se +++ /dev/null @@ -1,11 +0,0 @@ -def init(): - self.storage[msg.sender] = 1000000 - -def balance_query(k): - return(self.storage[addr]) - -def send(to, value): - fromvalue = self.storage[msg.sender] - if fromvalue >= value: - self.storage[from] = fromvalue - value - self.storage[to] += value diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/funcs.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/funcs.cpp deleted file mode 100644 index ea9be14a6..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/funcs.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include "funcs.h" -#include "bignum.h" -#include "util.h" -#include "parser.h" -#include "lllparser.h" -#include "compiler.h" -#include "rewriter.h" -#include "tokenize.h" - -Node compileToLLL(std::string input) { - return rewrite(parseSerpent(input)); -} - -Node compileChunkToLLL(std::string input) { - return rewriteChunk(parseSerpent(input)); -} - -std::string compile(std::string input) { - return compileLLL(compileToLLL(input)); -} - -std::vector<Node> prettyCompile(std::string input) { - return prettyCompileLLL(compileToLLL(input)); -} - -std::string compileChunk(std::string input) { - return compileLLL(compileChunkToLLL(input)); -} - -std::vector<Node> prettyCompileChunk(std::string input) { - return prettyCompileLLL(compileChunkToLLL(input)); -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/funcs.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/funcs.h deleted file mode 100644 index d9bf44549..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/funcs.h +++ /dev/null @@ -1,35 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include "bignum.h" -#include "util.h" -#include "parser.h" -#include "lllparser.h" -#include "compiler.h" -#include "rewriter.h" -#include "tokenize.h" - -// Function listing: -// -// parseSerpent (serpent -> AST) std::string -> Node -// parseLLL (LLL -> AST) std::string -> Node -// rewrite (apply rewrite rules) Node -> Node -// compileToLLL (serpent -> LLL) std::string -> Node -// compileLLL (LLL -> EVMhex) Node -> std::string -// prettyCompileLLL (LLL -> EVMasm) Node -> std::vector<Node> -// prettyCompile (serpent -> EVMasm) std::string -> std::vector>Node> -// compile (serpent -> EVMhex) std::string -> std::string -// get_file_contents (filename -> file) std::string -> std::string -// exists (does file exist?) std::string -> bool - -Node compileToLLL(std::string input); - -Node compileChunkToLLL(std::string input); - -std::string compile(std::string input); - -std::vector<Node> prettyCompile(std::string input); - -std::string compileChunk(std::string input); - -std::vector<Node> prettyCompileChunk(std::string input); diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/functions.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/functions.cpp deleted file mode 100644 index 78e12e84a..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/functions.cpp +++ /dev/null @@ -1,203 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" -#include "lllparser.h" -#include "bignum.h" -#include "optimize.h" -#include "rewriteutils.h" -#include "preprocess.h" -#include "functions.h" - -std::string getSignature(std::vector<Node> args) { - std::string o; - for (unsigned i = 0; i < args.size(); i++) { - if (args[i].val == ":" && args[i].args[1].val == "s") - o += "s"; - else if (args[i].val == ":" && args[i].args[1].val == "a") - o += "a"; - else - o += "i"; - } - return o; -} - -// Convert a list of arguments into a node containing a -// < datastart, datasz > pair - -Node packArguments(std::vector<Node> args, std::string sig, - int funId, Metadata m) { - // Plain old 32 byte arguments - std::vector<Node> nargs; - // Variable-sized arguments - std::vector<Node> vargs; - // Variable sizes - std::vector<Node> sizes; - // Is a variable an array? - std::vector<bool> isArray; - // Fill up above three argument lists - int argCount = 0; - for (unsigned i = 0; i < args.size(); i++) { - Metadata m = args[i].metadata; - if (args[i].val == "=") { - // do nothing - } - else { - // Determine the correct argument type - char argType; - if (sig.size() > 0) { - if (argCount >= (signed)sig.size()) - err("Too many args", m); - argType = sig[argCount]; - } - else argType = 'i'; - // Integer (also usable for short strings) - if (argType == 'i') { - if (args[i].val == ":") - err("Function asks for int, provided string or array", m); - nargs.push_back(args[i]); - } - // Long string - else if (argType == 's') { - if (args[i].val != ":") - err("Must specify string length", m); - vargs.push_back(args[i].args[0]); - sizes.push_back(args[i].args[1]); - isArray.push_back(false); - } - // Array - else if (argType == 'a') { - if (args[i].val != ":") - err("Must specify array length", m); - vargs.push_back(args[i].args[0]); - sizes.push_back(args[i].args[1]); - isArray.push_back(true); - } - else err("Invalid arg type in signature", m); - argCount++; - } - } - int static_arg_size = 1 + (vargs.size() + nargs.size()) * 32; - // Start off by saving the size variables and calculating the total - msn kwargs; - kwargs["funid"] = tkn(utd(funId), m); - std::string pattern = - "(with _sztot "+utd(static_arg_size)+" " - " (with _sizes (alloc "+utd(sizes.size() * 32)+") " - " (seq "; - for (unsigned i = 0; i < sizes.size(); i++) { - std::string sizeIncrement = - isArray[i] ? "(mul 32 _x)" : "_x"; - pattern += - "(with _x $sz"+utd(i)+"(seq " - " (mstore (add _sizes "+utd(i * 32)+") _x) " - " (set _sztot (add _sztot "+sizeIncrement+" )))) "; - kwargs["sz"+utd(i)] = sizes[i]; - } - // Allocate memory, and set first data byte - pattern += - "(with _datastart (alloc (add _sztot 32)) (seq " - " (mstore8 _datastart $funid) "; - // Copy over size variables - for (unsigned i = 0; i < sizes.size(); i++) { - int v = 1 + i * 32; - pattern += - " (mstore " - " (add _datastart "+utd(v)+") " - " (mload (add _sizes "+utd(v-1)+"))) "; - } - // Store normal arguments - for (unsigned i = 0; i < nargs.size(); i++) { - int v = 1 + (i + sizes.size()) * 32; - pattern += - " (mstore (add _datastart "+utd(v)+") $"+utd(i)+") "; - kwargs[utd(i)] = nargs[i]; - } - // Loop through variable-sized arguments, store them - pattern += - " (with _pos (add _datastart "+utd(static_arg_size)+") (seq"; - for (unsigned i = 0; i < vargs.size(); i++) { - std::string copySize = - isArray[i] ? "(mul 32 (mload (add _sizes "+utd(i * 32)+")))" - : "(mload (add _sizes "+utd(i * 32)+"))"; - pattern += - " (unsafe_mcopy _pos $vl"+utd(i)+" "+copySize+") " - " (set _pos (add _pos "+copySize+")) "; - kwargs["vl"+utd(i)] = vargs[i]; - } - // Return a 2-item array containing the start and size - pattern += " (array_lit _datastart _sztot))))))))"; - std::string prefix = "_temp_"+mkUniqueToken(); - // Fill in pattern, return triple - return subst(parseLLL(pattern), kwargs, prefix, m); -} - -// Create a node for argument unpacking -Node unpackArguments(std::vector<Node> vars, Metadata m) { - std::vector<std::string> varNames; - std::vector<std::string> longVarNames; - std::vector<bool> longVarIsArray; - // Fill in variable and long variable names, as well as which - // long variables are arrays and which are strings - for (unsigned i = 0; i < vars.size(); i++) { - if (vars[i].val == ":") { - if (vars[i].args.size() != 2) - err("Malformed def!", m); - longVarNames.push_back(vars[i].args[0].val); - std::string tag = vars[i].args[1].val; - if (tag == "s") - longVarIsArray.push_back(false); - else if (tag == "a") - longVarIsArray.push_back(true); - else - err("Function value can only be string or array", m); - } - else { - varNames.push_back(vars[i].val); - } - } - std::vector<Node> sub; - if (!varNames.size() && !longVarNames.size()) { - // do nothing if we have no arguments - } - else { - std::vector<Node> varNodes; - for (unsigned i = 0; i < longVarNames.size(); i++) - varNodes.push_back(token(longVarNames[i], m)); - for (unsigned i = 0; i < varNames.size(); i++) - varNodes.push_back(token(varNames[i], m)); - // Copy over variable lengths and short variables - for (unsigned i = 0; i < varNodes.size(); i++) { - int pos = 1 + i * 32; - std::string prefix = (i < longVarNames.size()) ? "_len_" : ""; - sub.push_back(asn("untyped", asn("set", - token(prefix+varNodes[i].val, m), - asn("calldataload", tkn(utd(pos), m), m), - m))); - } - // Copy over long variables - if (longVarNames.size() > 0) { - std::vector<Node> sub2; - int pos = varNodes.size() * 32 + 1; - Node tot = tkn("_tot", m); - for (unsigned i = 0; i < longVarNames.size(); i++) { - Node var = tkn(longVarNames[i], m); - Node varlen = longVarIsArray[i] - ? asn("mul", tkn("32", m), tkn("_len_"+longVarNames[i], m)) - : tkn("_len_"+longVarNames[i], m); - sub2.push_back(asn("untyped", - asn("set", var, asn("alloc", varlen)))); - sub2.push_back(asn("calldatacopy", var, tot, varlen)); - sub2.push_back(asn("set", tot, asn("add", tot, varlen))); - } - std::string prefix = "_temp_"+mkUniqueToken(); - sub.push_back(subst( - astnode("with", tot, tkn(utd(pos), m), asn("seq", sub2)), - msn(), - prefix, - m)); - } - } - return asn("seq", sub, m); -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/functions.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/functions.h deleted file mode 100644 index 68a1c69ce..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/functions.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef ETHSERP_FUNCTIONS -#define ETHSERP_FUNCTIONS - -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" -#include "lllparser.h" -#include "bignum.h" -#include "optimize.h" -#include "rewriteutils.h" -#include "preprocess.h" - - -class argPack { - public: - argPack(Node a, Node b, Node c) { - pre = a; - datastart = b; - datasz = c; - } - Node pre; - Node datastart; - Node datasz; -}; - -// Get a signature from a function -std::string getSignature(std::vector<Node> args); - -// Convert a list of arguments into a <pre, mstart, msize> node -// triple, given the signature of a function -Node packArguments(std::vector<Node> args, std::string sig, - int funId, Metadata m); - -// Create a node for argument unpacking -Node unpackArguments(std::vector<Node> vars, Metadata m); - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/lllparser.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/lllparser.cpp deleted file mode 100644 index ad4fbd52d..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/lllparser.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" -#include "lllparser.h" -#include "tokenize.h" - -struct _parseOutput { - Node node; - int newpos; -}; - -// Helper, returns subtree and position of start of next node -_parseOutput _parse(std::vector<Node> inp, int pos) { - Metadata met = inp[pos].metadata; - _parseOutput o; - // Bracket: keep grabbing tokens until we get to the - // corresponding closing bracket - if (inp[pos].val == "(" || inp[pos].val == "[") { - std::string fun, rbrack; - std::vector<Node> args; - pos += 1; - if (inp[pos].val == "[") { - fun = "access"; - rbrack = "]"; - } - else rbrack = ")"; - // First argument is the function - while (inp[pos].val != ")") { - _parseOutput po = _parse(inp, pos); - if (fun.length() == 0 && po.node.type == 1) { - std::cerr << "Error: first arg must be function\n"; - fun = po.node.val; - } - else if (fun.length() == 0) { - fun = po.node.val; - } - else { - args.push_back(po.node); - } - pos = po.newpos; - } - o.newpos = pos + 1; - o.node = astnode(fun, args, met); - } - // Normal token, return it and advance to next token - else { - o.newpos = pos + 1; - o.node = token(inp[pos].val, met); - } - return o; -} - -// stream of tokens -> lisp parse tree -Node parseLLLTokenStream(std::vector<Node> inp) { - _parseOutput o = _parse(inp, 0); - return o.node; -} - -// Parses LLL -Node parseLLL(std::string s, bool allowFileRead) { - std::string input = s; - std::string file = "main"; - if (exists(s) && allowFileRead) { - file = s; - input = get_file_contents(s); - } - return parseLLLTokenStream(tokenize(s, Metadata(file, 0, 0), true)); -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/lllparser.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/lllparser.h deleted file mode 100644 index 4bfa7b82e..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/lllparser.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef ETHSERP_LLLPARSER -#define ETHSERP_LLLPARSER - -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" - -// LLL text -> parse tree -Node parseLLL(std::string s, bool allowFileRead=false); - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/opcodes.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/opcodes.cpp deleted file mode 100644 index b24144e46..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/opcodes.cpp +++ /dev/null @@ -1,154 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "opcodes.h" -#include "util.h" -#include "bignum.h" - -Mapping mapping[] = { - Mapping("STOP", 0x00, 0, 0), - Mapping("ADD", 0x01, 2, 1), - Mapping("MUL", 0x02, 2, 1), - Mapping("SUB", 0x03, 2, 1), - Mapping("DIV", 0x04, 2, 1), - Mapping("SDIV", 0x05, 2, 1), - Mapping("MOD", 0x06, 2, 1), - Mapping("SMOD", 0x07, 2, 1), - Mapping("ADDMOD", 0x08, 3, 1), - Mapping("MULMOD", 0x09, 3, 1), - Mapping("EXP", 0x0a, 2, 1), - Mapping("SIGNEXTEND", 0x0b, 2, 1), - Mapping("LT", 0x10, 2, 1), - Mapping("GT", 0x11, 2, 1), - Mapping("SLT", 0x12, 2, 1), - Mapping("SGT", 0x13, 2, 1), - Mapping("EQ", 0x14, 2, 1), - Mapping("ISZERO", 0x15, 1, 1), - Mapping("AND", 0x16, 2, 1), - Mapping("OR", 0x17, 2, 1), - Mapping("XOR", 0x18, 2, 1), - Mapping("NOT", 0x19, 1, 1), - Mapping("BYTE", 0x1a, 2, 1), - Mapping("SHA3", 0x20, 2, 1), - Mapping("ADDRESS", 0x30, 0, 1), - Mapping("BALANCE", 0x31, 1, 1), - Mapping("ORIGIN", 0x32, 0, 1), - Mapping("CALLER", 0x33, 0, 1), - Mapping("CALLVALUE", 0x34, 0, 1), - Mapping("CALLDATALOAD", 0x35, 1, 1), - Mapping("CALLDATASIZE", 0x36, 0, 1), - Mapping("CALLDATACOPY", 0x37, 3, 0), - Mapping("CODESIZE", 0x38, 0, 1), - Mapping("CODECOPY", 0x39, 3, 0), - Mapping("GASPRICE", 0x3a, 0, 1), - Mapping("EXTCODESIZE", 0x3b, 1, 1), - Mapping("EXTCODECOPY", 0x3c, 4, 0), - Mapping("PREVHASH", 0x40, 0, 1), - Mapping("COINBASE", 0x41, 0, 1), - Mapping("TIMESTAMP", 0x42, 0, 1), - Mapping("NUMBER", 0x43, 0, 1), - Mapping("DIFFICULTY", 0x44, 0, 1), - Mapping("GASLIMIT", 0x45, 0, 1), - Mapping("POP", 0x50, 1, 0), - Mapping("MLOAD", 0x51, 1, 1), - Mapping("MSTORE", 0x52, 2, 0), - Mapping("MSTORE8", 0x53, 2, 0), - Mapping("SLOAD", 0x54, 1, 1), - Mapping("SSTORE", 0x55, 2, 0), - Mapping("JUMP", 0x56, 1, 0), - Mapping("JUMPI", 0x57, 2, 0), - Mapping("PC", 0x58, 0, 1), - Mapping("MSIZE", 0x59, 0, 1), - Mapping("GAS", 0x5a, 0, 1), - Mapping("JUMPDEST", 0x5b, 0, 0), - Mapping("LOG0", 0xa0, 2, 0), - Mapping("LOG1", 0xa1, 3, 0), - Mapping("LOG2", 0xa2, 4, 0), - Mapping("LOG3", 0xa3, 5, 0), - Mapping("LOG4", 0xa4, 6, 0), - Mapping("CREATE", 0xf0, 3, 1), - Mapping("CALL", 0xf1, 7, 1), - Mapping("CALLCODE", 0xf2, 7, 1), - Mapping("RETURN", 0xf3, 2, 0), - Mapping("SUICIDE", 0xff, 1, 0), - Mapping("---END---", 0x00, 0, 0), -}; - -std::map<std::string, std::vector<int> > opcodes; -std::map<int, std::string> reverseOpcodes; - -// Fetches everything EXCEPT PUSH1..32 -std::pair<std::string, std::vector<int> > _opdata(std::string ops, int opi) { - if (!opcodes.size()) { - int i = 0; - while (mapping[i].op != "---END---") { - Mapping mi = mapping[i]; - opcodes[mi.op] = triple(mi.opcode, mi.in, mi.out); - i++; - } - for (i = 1; i <= 16; i++) { - opcodes["DUP"+unsignedToDecimal(i)] = triple(0x7f + i, i, i+1); - opcodes["SWAP"+unsignedToDecimal(i)] = triple(0x8f + i, i+1, i+1); - } - for (std::map<std::string, std::vector<int> >::iterator it=opcodes.begin(); - it != opcodes.end(); - it++) { - reverseOpcodes[(*it).second[0]] = (*it).first; - } - } - ops = upperCase(ops); - std::string op; - std::vector<int> opdata; - op = reverseOpcodes.count(opi) ? reverseOpcodes[opi] : ""; - opdata = opcodes.count(ops) ? opcodes[ops] : triple(-1, -1, -1); - return std::pair<std::string, std::vector<int> >(op, opdata); -} - -int opcode(std::string op) { - return _opdata(op, -1).second[0]; -} - -int opinputs(std::string op) { - return _opdata(op, -1).second[1]; -} - -int opoutputs(std::string op) { - return _opdata(op, -1).second[2]; -} - -std::string op(int opcode) { - return _opdata("", opcode).first; -} - -std::string lllSpecials[][3] = { - { "ref", "1", "1" }, - { "get", "1", "1" }, - { "set", "2", "2" }, - { "with", "3", "3" }, - { "comment", "0", "2147483647" }, - { "ops", "0", "2147483647" }, - { "lll", "2", "2" }, - { "seq", "0", "2147483647" }, - { "if", "3", "3" }, - { "unless", "2", "2" }, - { "until", "2", "2" }, - { "alloc", "1", "1" }, - { "---END---", "0", "0" }, -}; - -std::map<std::string, std::pair<int, int> > lllMap; - -// Is a function name one of the valid functions above? -bool isValidLLLFunc(std::string f, int argc) { - if (lllMap.size() == 0) { - for (int i = 0; ; i++) { - if (lllSpecials[i][0] == "---END---") break; - lllMap[lllSpecials[i][0]] = std::pair<int, int>( - dtu(lllSpecials[i][1]), dtu(lllSpecials[i][2])); - } - } - return lllMap.count(f) - && argc >= lllMap[f].first - && argc <= lllMap[f].second; -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/opcodes.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/opcodes.h deleted file mode 100644 index 41423c169..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/opcodes.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef ETHSERP_OPCODES -#define ETHSERP_OPCODES - -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" - -class Mapping { - public: - Mapping(std::string Op, int Opcode, int In, int Out) { - op = Op; - opcode = Opcode; - in = In; - out = Out; - } - std::string op; - int opcode; - int in; - int out; -}; - -extern Mapping mapping[]; - -extern std::map<std::string, std::vector<int> > opcodes; -extern std::map<int, std::string> reverseOpcodes; - -std::pair<std::string, std::vector<int> > _opdata(std::string ops, int opi); - -int opcode(std::string op); - -int opinputs(std::string op); - -int opoutputs(std::string op); - -std::string op(int opcode); - -extern std::string lllSpecials[][3]; - -extern std::map<std::string, std::pair<int, int> > lllMap; - -bool isValidLLLFunc(std::string f, int argc); - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/optimize.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/optimize.cpp deleted file mode 100644 index e689fcb69..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/optimize.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" -#include "lllparser.h" -#include "bignum.h" - -// Compile-time arithmetic calculations -Node optimize(Node inp) { - if (inp.type == TOKEN) { - Node o = tryNumberize(inp); - if (decimalGt(o.val, tt256, true)) - err("Value too large (exceeds 32 bytes or 2^256)", inp.metadata); - return o; - } - for (unsigned i = 0; i < inp.args.size(); i++) { - inp.args[i] = optimize(inp.args[i]); - } - // Arithmetic-specific transform - if (inp.val == "+") inp.val = "add"; - if (inp.val == "*") inp.val = "mul"; - if (inp.val == "-") inp.val = "sub"; - if (inp.val == "/") inp.val = "sdiv"; - if (inp.val == "^") inp.val = "exp"; - if (inp.val == "**") inp.val = "exp"; - if (inp.val == "%") inp.val = "smod"; - // Degenerate cases for add and mul - if (inp.args.size() == 2) { - if (inp.val == "add" && inp.args[0].type == TOKEN && - inp.args[0].val == "0") { - Node x = inp.args[1]; - inp = x; - } - if (inp.val == "add" && inp.args[1].type == TOKEN && - inp.args[1].val == "0") { - Node x = inp.args[0]; - inp = x; - } - if (inp.val == "mul" && inp.args[0].type == TOKEN && - inp.args[0].val == "1") { - Node x = inp.args[1]; - inp = x; - } - if (inp.val == "mul" && inp.args[1].type == TOKEN && - inp.args[1].val == "1") { - Node x = inp.args[0]; - inp = x; - } - } - // Arithmetic computation - if (inp.args.size() == 2 - && inp.args[0].type == TOKEN - && inp.args[1].type == TOKEN) { - std::string o; - if (inp.val == "add") { - o = decimalMod(decimalAdd(inp.args[0].val, inp.args[1].val), tt256); - } - else if (inp.val == "sub") { - if (decimalGt(inp.args[0].val, inp.args[1].val, true)) - o = decimalSub(inp.args[0].val, inp.args[1].val); - } - else if (inp.val == "mul") { - o = decimalMod(decimalMul(inp.args[0].val, inp.args[1].val), tt256); - } - else if (inp.val == "div" && inp.args[1].val != "0") { - o = decimalDiv(inp.args[0].val, inp.args[1].val); - } - else if (inp.val == "sdiv" && inp.args[1].val != "0" - && decimalGt(tt255, inp.args[0].val) - && decimalGt(tt255, inp.args[1].val)) { - o = decimalDiv(inp.args[0].val, inp.args[1].val); - } - else if (inp.val == "mod" && inp.args[1].val != "0") { - o = decimalMod(inp.args[0].val, inp.args[1].val); - } - else if (inp.val == "smod" && inp.args[1].val != "0" - && decimalGt(tt255, inp.args[0].val) - && decimalGt(tt255, inp.args[1].val)) { - o = decimalMod(inp.args[0].val, inp.args[1].val); - } - else if (inp.val == "exp") { - o = decimalModExp(inp.args[0].val, inp.args[1].val, tt256); - } - if (o.length()) return token(o, inp.metadata); - } - return inp; -} - -// Is a node degenerate (ie. trivial to calculate) ? -bool isDegenerate(Node n) { - return optimize(n).type == TOKEN; -} - -// Is a node purely arithmetic? -bool isPureArithmetic(Node n) { - return isNumberLike(optimize(n)); -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/optimize.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/optimize.h deleted file mode 100644 index 06ea3bba1..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/optimize.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef ETHSERP_OPTIMIZER -#define ETHSERP_OPTIMIZER - -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" - -// Compile-time arithmetic calculations -Node optimize(Node inp); - -// Is a node degenerate (ie. trivial to calculate) ? -bool isDegenerate(Node n); - -// Is a node purely arithmetic? -bool isPureArithmetic(Node n); - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/parser.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/parser.cpp deleted file mode 100644 index 5e8c459c3..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/parser.cpp +++ /dev/null @@ -1,430 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" -#include "parser.h" -#include "tokenize.h" - -// Extended BEDMAS precedence order -int precedence(Node tok) { - std::string v = tok.val; - if (v == ".") return -1; - else if (v == "!" || v == "not") return 1; - else if (v=="^" || v == "**") return 2; - else if (v=="*" || v=="/" || v=="%") return 3; - else if (v=="+" || v=="-") return 4; - else if (v=="<" || v==">" || v=="<=" || v==">=") return 5; - else if (v=="&" || v=="|" || v=="xor" || v=="==" || v == "!=") return 6; - else if (v=="&&" || v=="and") return 7; - else if (v=="||" || v=="or") return 8; - else if (v=="=") return 10; - else if (v=="+=" || v=="-=" || v=="*=" || v=="/=" || v=="%=") return 10; - else if (v==":" || v == "::") return 11; - else return 0; -} - -// Token classification for shunting-yard purposes -int toktype(Node tok) { - if (tok.type == ASTNODE) return COMPOUND; - std::string v = tok.val; - if (v == "(" || v == "[" || v == "{") return LPAREN; - else if (v == ")" || v == "]" || v == "}") return RPAREN; - else if (v == ",") return COMMA; - else if (v == "!" || v == "~" || v == "not") return UNARY_OP; - else if (precedence(tok) > 0) return BINARY_OP; - else if (precedence(tok) < 0) return TOKEN_SPLITTER; - if (tok.val[0] != '"' && tok.val[0] != '\'') { - for (unsigned i = 0; i < tok.val.length(); i++) { - if (chartype(tok.val[i]) == SYMB) { - err("Invalid symbol: "+tok.val, tok.metadata); - } - } - } - return ALPHANUM; -} - - -// Converts to reverse polish notation -std::vector<Node> shuntingYard(std::vector<Node> tokens) { - std::vector<Node> iq; - for (int i = tokens.size() - 1; i >= 0; i--) { - iq.push_back(tokens[i]); - } - std::vector<Node> oq; - std::vector<Node> stack; - Node prev, tok; - int prevtyp = 0, toktyp = 0; - - while (iq.size()) { - prev = tok; - prevtyp = toktyp; - tok = iq.back(); - toktyp = toktype(tok); - iq.pop_back(); - // Alphanumerics go straight to output queue - if (toktyp == ALPHANUM) { - oq.push_back(tok); - } - // Left parens go on stack and output queue - else if (toktyp == LPAREN) { - while (stack.size() && toktype(stack.back()) == TOKEN_SPLITTER) { - oq.push_back(stack.back()); - stack.pop_back(); - } - if (prevtyp != ALPHANUM && prevtyp != RPAREN) { - oq.push_back(token("id", tok.metadata)); - } - stack.push_back(tok); - oq.push_back(tok); - } - // If rparen, keep moving from stack to output queue until lparen - else if (toktyp == RPAREN) { - while (stack.size() && toktype(stack.back()) != LPAREN) { - oq.push_back(stack.back()); - stack.pop_back(); - } - if (stack.size()) { - stack.pop_back(); - } - oq.push_back(tok); - } - else if (toktyp == UNARY_OP) { - stack.push_back(tok); - } - // If token splitter, just push it to the stack - else if (toktyp == TOKEN_SPLITTER) { - while (stack.size() && toktype(stack.back()) == TOKEN_SPLITTER) { - oq.push_back(stack.back()); - stack.pop_back(); - } - stack.push_back(tok); - } - // If binary op, keep popping from stack while higher bedmas precedence - else if (toktyp == BINARY_OP) { - if (tok.val == "-" && prevtyp != ALPHANUM && prevtyp != RPAREN) { - stack.push_back(tok); - oq.push_back(token("0", tok.metadata)); - } - else { - int prec = precedence(tok); - while (stack.size() - && (toktype(stack.back()) == BINARY_OP - || toktype(stack.back()) == UNARY_OP - || toktype(stack.back()) == TOKEN_SPLITTER) - && precedence(stack.back()) <= prec) { - oq.push_back(stack.back()); - stack.pop_back(); - } - stack.push_back(tok); - } - } - // Comma means finish evaluating the argument - else if (toktyp == COMMA) { - while (stack.size() && toktype(stack.back()) != LPAREN) { - oq.push_back(stack.back()); - stack.pop_back(); - } - } - } - while (stack.size()) { - oq.push_back(stack.back()); - stack.pop_back(); - } - return oq; -} - -// Converts reverse polish notation into tree -Node treefy(std::vector<Node> stream) { - std::vector<Node> iq; - for (int i = stream.size() -1; i >= 0; i--) { - iq.push_back(stream[i]); - } - std::vector<Node> oq; - while (iq.size()) { - Node tok = iq.back(); - iq.pop_back(); - int typ = toktype(tok); - // If unary, take node off end of oq and wrap it with the operator - // If binary, do the same with two nodes - if (typ == UNARY_OP || typ == BINARY_OP || typ == TOKEN_SPLITTER) { - std::vector<Node> args; - int rounds = (typ == UNARY_OP) ? 1 : 2; - for (int i = 0; i < rounds; i++) { - if (oq.size() == 0) { - err("Line malformed, not enough args for "+tok.val, - tok.metadata); - } - args.push_back(oq.back()); - oq.pop_back(); - } - std::vector<Node> args2; - while (args.size()) { - args2.push_back(args.back()); - args.pop_back(); - } - oq.push_back(astnode(tok.val, args2, tok.metadata)); - } - // If rparen, keep grabbing until we get to an lparen - else if (typ == RPAREN) { - std::vector<Node> args; - while (1) { - if (toktype(oq.back()) == LPAREN) break; - args.push_back(oq.back()); - oq.pop_back(); - if (!oq.size()) err("Bracket without matching", tok.metadata); - } - oq.pop_back(); - args.push_back(oq.back()); - oq.pop_back(); - // We represent a[b] as (access a b) - if (tok.val == "]") - args.push_back(token("access", tok.metadata)); - if (args.back().type == ASTNODE) - args.push_back(token("fun", tok.metadata)); - std::string fun = args.back().val; - args.pop_back(); - // We represent [1,2,3] as (array_lit 1 2 3) - if (fun == "access" && args.size() && args.back().val == "id") { - fun = "array_lit"; - args.pop_back(); - } - std::vector<Node> args2; - while (args.size()) { - args2.push_back(args.back()); - args.pop_back(); - } - // When evaluating 2 + (3 * 5), the shunting yard algo turns that - // into 2 ( id 3 5 * ) +, effectively putting "id" as a dummy - // function where the algo was expecting a function to call the - // thing inside the brackets. This reverses that step - if (fun == "id" && args2.size() == 1) { - oq.push_back(args2[0]); - } - else { - oq.push_back(astnode(fun, args2, tok.metadata)); - } - } - else oq.push_back(tok); - // This is messy, but has to be done. Import/inset other files here - std::string v = oq.back().val; - if ((v == "inset" || v == "import" || v == "create") - && oq.back().args.size() == 1 - && oq.back().args[0].type == TOKEN) { - int lastSlashPos = tok.metadata.file.rfind("/"); - std::string root; - if (lastSlashPos >= 0) - root = tok.metadata.file.substr(0, lastSlashPos) + "/"; - else - root = ""; - std::string filename = oq.back().args[0].val; - filename = filename.substr(1, filename.length() - 2); - if (!exists(root + filename)) - err("File does not exist: "+root + filename, tok.metadata); - oq.back().args.pop_back(); - oq.back().args.push_back(parseSerpent(root + filename)); - } - //Useful for debugging - //for (int i = 0; i < oq.size(); i++) { - // std::cerr << printSimple(oq[i]) << " "; - //} - //std::cerr << " <-\n"; - } - // Output must have one argument - if (oq.size() == 0) { - err("Output blank", Metadata()); - } - else if (oq.size() > 1) { - return asn("multi", oq, oq[0].metadata); - } - - return oq[0]; -} - - -// Parses one line of serpent -Node parseSerpentTokenStream(std::vector<Node> s) { - return treefy(shuntingYard(s)); -} - - -// Count spaces at beginning of line -int spaceCount(std::string s) { - unsigned pos = 0; - while (pos < s.length() && (s[pos] == ' ' || s[pos] == '\t')) - pos++; - return pos; -} - -// Is this a command that takes an argument on the same line? -bool bodied(std::string tok) { - return tok == "if" || tok == "elif" || tok == "while" - || tok == "with" || tok == "def" || tok == "extern" - || tok == "data" || tok == "assert" || tok == "return" - || tok == "fun" || tok == "scope" || tok == "macro" - || tok == "type"; -} - -// Are the two commands meant to continue each other? -bool bodiedContinued(std::string prev, std::string tok) { - return (prev == "if" && tok == "elif") - || (prev == "elif" && tok == "else") - || (prev == "elif" && tok == "elif") - || (prev == "if" && tok == "else"); -} - -// Is a line of code empty? -bool isLineEmpty(std::string line) { - std::vector<Node> tokens = tokenize(line); - if (!tokens.size() || tokens[0].val == "#" || tokens[0].val == "//") - return true; - return false; -} - -// Parse lines of serpent (helper function) -Node parseLines(std::vector<std::string> lines, Metadata metadata, int sp) { - std::vector<Node> o; - int origLine = metadata.ln; - unsigned i = 0; - while (i < lines.size()) { - metadata.ln = origLine + i; - std::string main = lines[i]; - if (isLineEmpty(main)) { - i += 1; - continue; - } - int spaces = spaceCount(main); - if (spaces != sp) { - err("Indent mismatch", metadata); - } - // Tokenize current line - std::vector<Node> tokens = tokenize(main.substr(sp), metadata); - // Remove comments - std::vector<Node> tokens2; - for (unsigned j = 0; j < tokens.size(); j++) { - if (tokens[j].val == "#" || tokens[j].val == "//") break; - tokens2.push_back(tokens[j]); - } - bool expectingChildBlock = false; - if (tokens2.size() > 0 && tokens2.back().val == ":") { - tokens2.pop_back(); - expectingChildBlock = true; - } - // Parse current line - Node out = parseSerpentTokenStream(tokens2); - // Parse child block - int childIndent = 999999; - std::vector<std::string> childBlock; - while (1) { - i++; - if (i >= lines.size()) - break; - bool ile = isLineEmpty(lines[i]); - if (!ile) { - int spaces = spaceCount(lines[i]); - if (spaces <= sp) break; - childBlock.push_back(lines[i]); - if (spaces < childIndent) childIndent = spaces; - } - else childBlock.push_back(""); - } - // Child block empty? - bool cbe = true; - for (unsigned i = 0; i < childBlock.size(); i++) { - if (childBlock[i].length() > 0) { cbe = false; break; } - } - // Add child block to AST - if (expectingChildBlock) { - if (cbe) - err("Expected indented child block!", out.metadata); - out.type = ASTNODE; - metadata.ln += 1; - out.args.push_back(parseLines(childBlock, metadata, childIndent)); - metadata.ln -= 1; - } - else if (!cbe) - err("Did not expect indented child block!", out.metadata); - else if (out.args.size() && out.args[out.args.size() - 1].val == ":") { - Node n = out.args[out.args.size() - 1]; - out.args.pop_back(); - out.args.push_back(n.args[0]); - out.args.push_back(n.args[1]); - } - // Bring back if / elif into AST - if (bodied(tokens[0].val)) { - if (out.val != "multi") { - // token not being used in bodied form - } - else if (out.args[0].val == "id") - out = astnode(tokens[0].val, out.args[1].args, out.metadata); - else if (out.args[0].type == TOKEN) { - std::vector<Node> out2; - for (unsigned i = 1; i < out.args.size(); i++) - out2.push_back(out.args[i]); - out = astnode(tokens[0].val, out2, out.metadata); - } - else - out = astnode("fun", out.args, out.metadata); - } - // Multi not supported - if (out.val == "multi") - err("Multiple expressions or unclosed bracket", out.metadata); - // Convert top-level colon expressions into non-colon expressions; - // makes if statements and the like equivalent indented or not - //if (out.val == ":" && out.args[0].type == TOKEN) - // out = asn(out.args[0].val, out.args[1], out.metadata); - //if (bodied(tokens[0].val) && out.args[0].val == ":") - // out = asn(tokens[0].val, out.args[0].args); - if (o.size() == 0 || o.back().type == TOKEN) { - o.push_back(out); - continue; - } - // This is a little complicated. Basically, the idea here is to build - // constructions like [if [< x 5] [a] [elif [< x 10] [b] [else [c]]]] - std::vector<Node> u; - u.push_back(o.back()); - if (bodiedContinued(o.back().val, out.val)) { - while (1) { - if (!bodiedContinued(u.back().val, out.val)) { - u.pop_back(); - break; - } - if (!u.back().args.size() - || !bodiedContinued(u.back().val, u.back().args.back().val)) { - break; - } - u.push_back(u.back().args.back()); - } - u.back().args.push_back(out); - while (u.size() > 1) { - Node v = u.back(); - u.pop_back(); - u.back().args.pop_back(); - u.back().args.push_back(v); - } - o.pop_back(); - o.push_back(u[0]); - } - else o.push_back(out); - } - if (o.size() == 1) - return o[0]; - else if (o.size()) - return astnode("seq", o, o[0].metadata); - else - return astnode("seq", o, Metadata()); -} - -// Parses serpent code -Node parseSerpent(std::string s) { - std::string input = s; - std::string file = "main"; - if (exists(s)) { - file = s; - input = get_file_contents(s); - } - return parseLines(splitLines(input), Metadata(file, 0, 0), 0); -} - - -using namespace std; diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/parser.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/parser.h deleted file mode 100644 index e3632220a..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/parser.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef ETHSERP_PARSER -#define ETHSERP_PARSER - -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" - -// Serpent text -> parse tree -Node parseSerpent(std::string s); - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/preprocess.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/preprocess.cpp deleted file mode 100644 index 3f08ea8b1..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/preprocess.cpp +++ /dev/null @@ -1,299 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" -#include "lllparser.h" -#include "bignum.h" -#include "rewriteutils.h" -#include "optimize.h" -#include "preprocess.h" -#include "functions.h" -#include "opcodes.h" - -// Convert a function of the form (def (f x y z) (do stuff)) into -// (if (first byte of ABI is correct) (seq (setup x y z) (do stuff))) -Node convFunction(Node node, int functionCount) { - std::string prefix = "_temp"+mkUniqueToken()+"_"; - Metadata m = node.metadata; - - if (node.args.size() != 2) - err("Malformed def!", m); - // Collect the list of variable names and variable byte counts - Node unpack = unpackArguments(node.args[0].args, m); - // And the actual code - Node body = node.args[1]; - // Main LLL-based function body - return astnode("if", - astnode("eq", - astnode("get", token("__funid", m), m), - token(unsignedToDecimal(functionCount), m), - m), - astnode("seq", unpack, body, m)); -} - -// Populate an svObj with the arguments needed to determine -// the storage position of a node -svObj getStorageVars(svObj pre, Node node, std::string prefix, - int index) { - Metadata m = node.metadata; - if (!pre.globalOffset.size()) pre.globalOffset = "0"; - std::vector<Node> h; - std::vector<std::string> coefficients; - // Array accesses or atoms - if (node.val == "access" || node.type == TOKEN) { - std::string tot = "1"; - h = listfyStorageAccess(node); - coefficients.push_back("1"); - for (unsigned i = h.size() - 1; i >= 1; i--) { - // Array sizes must be constant or at least arithmetically - // evaluable at compile time - if (!isPureArithmetic(h[i])) - err("Array size must be fixed value", m); - // Create a list of the coefficient associated with each - // array index - coefficients.push_back(decimalMul(coefficients.back(), h[i].val)); - } - } - // Tuples - else { - int startc; - // Handle the (fun <fun_astnode> args...) case - if (node.val == "fun") { - startc = 1; - h = listfyStorageAccess(node.args[0]); - } - // Handle the (<fun_name> args...) case, which - // the serpent parser produces when the function - // is a simple name and not a complex astnode - else { - startc = 0; - h = listfyStorageAccess(token(node.val, m)); - } - svObj sub = pre; - sub.globalOffset = "0"; - // Evaluate tuple elements recursively - for (unsigned i = startc; i < node.args.size(); i++) { - sub = getStorageVars(sub, - node.args[i], - prefix+h[0].val.substr(2)+".", - i-startc); - } - coefficients.push_back(sub.globalOffset); - for (unsigned i = h.size() - 1; i >= 1; i--) { - // Array sizes must be constant or at least arithmetically - // evaluable at compile time - if (!isPureArithmetic(h[i])) - err("Array size must be fixed value", m); - // Create a list of the coefficient associated with each - // array index - coefficients.push_back(decimalMul(coefficients.back(), h[i].val)); - } - pre.offsets = sub.offsets; - pre.coefficients = sub.coefficients; - pre.nonfinal = sub.nonfinal; - pre.nonfinal[prefix+h[0].val.substr(2)] = true; - } - pre.coefficients[prefix+h[0].val.substr(2)] = coefficients; - pre.offsets[prefix+h[0].val.substr(2)] = pre.globalOffset; - pre.indices[prefix+h[0].val.substr(2)] = index; - if (decimalGt(tt176, coefficients.back())) - pre.globalOffset = decimalAdd(pre.globalOffset, coefficients.back()); - return pre; -} - -// Preprocess input containing functions -// -// localExterns is a map of the form, eg, -// -// { x: { foo: 0, bar: 1, baz: 2 }, y: { qux: 0, foo: 1 } ... } -// -// localExternSigs is a map of the form, eg, -// -// { x : { foo: iii, bar: iis, baz: ia }, y: { qux: i, foo: as } ... } -// -// Signifying that x.foo = 0, x.baz = 2, y.foo = 1, etc -// and that x.foo has three integers as arguments, x.bar has two -// integers and a variable-length string, and baz has an integer -// and an array -// -// globalExterns is a one-level map, eg from above -// -// { foo: 1, bar: 1, baz: 2, qux: 0 } -// -// globalExternSigs is a one-level map, eg from above -// -// { foo: as, bar: iis, baz: ia, qux: i} -// -// Note that globalExterns and globalExternSigs may be ambiguous -// Also, a null signature implies an infinite tail of integers -preprocessResult preprocessInit(Node inp) { - Metadata m = inp.metadata; - if (inp.val != "seq") - inp = astnode("seq", inp, m); - std::vector<Node> empty = std::vector<Node>(); - Node init = astnode("seq", empty, m); - Node shared = astnode("seq", empty, m); - std::vector<Node> any; - std::vector<Node> functions; - preprocessAux out = preprocessAux(); - out.localExterns["self"] = std::map<std::string, int>(); - int functionCount = 0; - int storageDataCount = 0; - for (unsigned i = 0; i < inp.args.size(); i++) { - Node obj = inp.args[i]; - // Functions - if (obj.val == "def") { - if (obj.args.size() == 0) - err("Empty def", m); - std::string funName = obj.args[0].val; - // Init, shared and any are special functions - if (funName == "init" || funName == "shared" || funName == "any") { - if (obj.args[0].args.size()) - err(funName+" cannot have arguments", m); - } - if (funName == "init") init = obj.args[1]; - else if (funName == "shared") shared = obj.args[1]; - else if (funName == "any") any.push_back(obj.args[1]); - else { - // Other functions - functions.push_back(convFunction(obj, functionCount)); - out.localExterns["self"][obj.args[0].val] = functionCount; - out.localExternSigs["self"][obj.args[0].val] - = getSignature(obj.args[0].args); - functionCount++; - } - } - // Extern declarations - else if (obj.val == "extern") { - std::string externName = obj.args[0].val; - Node al = obj.args[1]; - if (!out.localExterns.count(externName)) - out.localExterns[externName] = std::map<std::string, int>(); - for (unsigned i = 0; i < al.args.size(); i++) { - if (al.args[i].val == ":") { - std::string v = al.args[i].args[0].val; - std::string sig = al.args[i].args[1].val; - out.globalExterns[v] = i; - out.globalExternSigs[v] = sig; - out.localExterns[externName][v] = i; - out.localExternSigs[externName][v] = sig; - } - else { - std::string v = al.args[i].val; - out.globalExterns[v] = i; - out.globalExternSigs[v] = ""; - out.localExterns[externName][v] = i; - out.localExternSigs[externName][v] = ""; - } - } - } - // Custom macros - else if (obj.val == "macro") { - // Rules for valid macros: - // - // There are only four categories of valid macros: - // - // 1. a macro where the outer function is something - // which is NOT an existing valid function/extern/datum - // 2. a macro of the form set(c(x), d) where c must NOT - // be an existing valid function/extern/datum - // 3. something of the form access(c(x)), where c must NOT - // be an existing valid function/extern/datum - // 4. something of the form set(access(c(x)), d) where c must - // NOT be an existing valid function/extern/datum - bool valid = false; - Node pattern = obj.args[0]; - Node substitution = obj.args[1]; - if (opcode(pattern.val) < 0 && !isValidFunctionName(pattern.val)) - valid = true; - if (pattern.val == "set" && - opcode(pattern.args[0].val) < 0 && - !isValidFunctionName(pattern.args[0].val)) - valid = true; - if (pattern.val == "access" && - opcode(pattern.args[0].val) < 0 && - !isValidFunctionName(pattern.args[0].val)) - if (pattern.val == "set" && - pattern.args[0].val == "access" && - opcode(pattern.args[0].args[0].val) < 0 && - !isValidFunctionName(pattern.args[0].args[0].val)) - valid = true; - if (valid) { - out.customMacros.push_back(rewriteRule(pattern, substitution)); - } - } - // Variable types - else if (obj.val == "type") { - std::string typeName = obj.args[0].val; - std::vector<Node> vars = obj.args[1].args; - for (unsigned i = 0; i < vars.size(); i++) - out.types[vars[i].val] = typeName; - } - // Storage variables/structures - else if (obj.val == "data") { - out.storageVars = getStorageVars(out.storageVars, - obj.args[0], - "", - storageDataCount); - storageDataCount += 1; - } - else any.push_back(obj); - } - std::vector<Node> main; - if (shared.args.size()) main.push_back(shared); - if (init.args.size()) main.push_back(init); - - std::vector<Node> code; - if (shared.args.size()) code.push_back(shared); - for (unsigned i = 0; i < any.size(); i++) - code.push_back(any[i]); - for (unsigned i = 0; i < functions.size(); i++) - code.push_back(functions[i]); - Node codeNode; - if (functions.size() > 0) { - codeNode = astnode("with", - token("__funid", m), - astnode("byte", - token("0", m), - astnode("calldataload", token("0", m), m), - m), - astnode("seq", code, m), - m); - } - else codeNode = astnode("seq", code, m); - main.push_back(astnode("~return", - token("0", m), - astnode("lll", - codeNode, - token("0", m), - m), - m)); - - - Node result; - if (main.size() == 1) result = main[0]; - else result = astnode("seq", main, inp.metadata); - return preprocessResult(result, out); -} - -preprocessResult processTypes (preprocessResult pr) { - preprocessAux aux = pr.second; - Node node = pr.first; - if (node.type == TOKEN && aux.types.count(node.val)) { - node = asn(aux.types[node.val], node, node.metadata); - } - else if (node.val == "untyped") - return preprocessResult(node.args[0], aux); - else { - for (unsigned i = 0; i < node.args.size(); i++) { - node.args[i] = - processTypes(preprocessResult(node.args[i], aux)).first; - } - } - return preprocessResult(node, aux); -} - -preprocessResult preprocess(Node n) { - return processTypes(preprocessInit(n)); -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/preprocess.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/preprocess.h deleted file mode 100644 index 944436aef..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/preprocess.h +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef ETHSERP_PREPROCESSOR -#define ETHSERP_PREPROCESSOR - -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" - -// Storage variable index storing object -struct svObj { - std::map<std::string, std::string> offsets; - std::map<std::string, int> indices; - std::map<std::string, std::vector<std::string> > coefficients; - std::map<std::string, bool> nonfinal; - std::string globalOffset; -}; - -class rewriteRule { - public: - rewriteRule(Node p, Node s) { - pattern = p; - substitution = s; - } - Node pattern; - Node substitution; -}; - - -// Preprocessing result storing object -class preprocessAux { - public: - preprocessAux() { - globalExterns = std::map<std::string, int>(); - localExterns = std::map<std::string, std::map<std::string, int> >(); - localExterns["self"] = std::map<std::string, int>(); - } - std::map<std::string, int> globalExterns; - std::map<std::string, std::string> globalExternSigs; - std::map<std::string, std::map<std::string, int> > localExterns; - std::map<std::string, std::map<std::string, std::string> > localExternSigs; - std::vector<rewriteRule> customMacros; - std::map<std::string, std::string> types; - svObj storageVars; -}; - -#define preprocessResult std::pair<Node, preprocessAux> - -// Populate an svObj with the arguments needed to determine -// the storage position of a node -svObj getStorageVars(svObj pre, Node node, std::string prefix="", - int index=0); - -// Preprocess a function (see cpp for details) -preprocessResult preprocess(Node inp); - - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/pyserpent.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/pyserpent.cpp deleted file mode 100644 index 38398aa46..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/pyserpent.cpp +++ /dev/null @@ -1,173 +0,0 @@ -#include <Python.h> -#include "structmember.h" - -#include <stdlib.h> -#include <stdio.h> -#include <iostream> -#include "funcs.h" - -#define PYMETHOD(name, FROM, method, TO) \ - static PyObject * name(PyObject *, PyObject *args) { \ - try { \ - FROM(med) \ - return TO(method(med)); \ - } \ - catch (std::string e) { \ - PyErr_SetString(PyExc_Exception, e.c_str()); \ - return NULL; \ - } \ - } - -#define FROMSTR(v) \ - const char *command; \ - int len; \ - if (!PyArg_ParseTuple(args, "s#", &command, &len)) \ - return NULL; \ - std::string v = std::string(command, len); \ - -#define FROMNODE(v) \ - PyObject *node; \ - if (!PyArg_ParseTuple(args, "O", &node)) \ - return NULL; \ - Node v = cppifyNode(node); - -#define FROMLIST(v) \ - PyObject *node; \ - if (!PyArg_ParseTuple(args, "O", &node)) \ - return NULL; \ - std::vector<Node> v = cppifyNodeList(node); - -// Convert metadata into python wrapper form [file, ln, ch] -PyObject* pyifyMetadata(Metadata m) { - PyObject* a = PyList_New(0); - PyList_Append(a, Py_BuildValue("s#", m.file.c_str(), m.file.length())); - PyList_Append(a, Py_BuildValue("i", m.ln)); - PyList_Append(a, Py_BuildValue("i", m.ch)); - return a; -} - -// Convert node into python wrapper form -// [token=0/astnode=1, val, metadata, args] -PyObject* pyifyNode(Node n) { - PyObject* a = PyList_New(0); - PyList_Append(a, Py_BuildValue("i", n.type == ASTNODE)); - PyList_Append(a, Py_BuildValue("s#", n.val.c_str(), n.val.length())); - PyList_Append(a, pyifyMetadata(n.metadata)); - for (unsigned i = 0; i < n.args.size(); i++) - PyList_Append(a, pyifyNode(n.args[i])); - return a; -} - -// Convert string into python wrapper form -PyObject* pyifyString(std::string s) { - return Py_BuildValue("s#", s.c_str(), s.length()); -} - -// Convert list of nodes into python wrapper form -PyObject* pyifyNodeList(std::vector<Node> n) { - PyObject* a = PyList_New(0); - for (unsigned i = 0; i < n.size(); i++) - PyList_Append(a, pyifyNode(n[i])); - return a; -} - -// Convert pyobject int into normal form -int cppifyInt(PyObject* o) { - int out; - if (!PyArg_Parse(o, "i", &out)) - err("Argument should be integer", Metadata()); - return out; -} - -// Convert pyobject string into normal form -std::string cppifyString(PyObject* o) { - const char *command; - if (!PyArg_Parse(o, "s", &command)) - err("Argument should be string", Metadata()); - return std::string(command); -} - -// Convert metadata from python wrapper form -Metadata cppifyMetadata(PyObject* o) { - std::string file = cppifyString(PyList_GetItem(o, 0)); - int ln = cppifyInt(PyList_GetItem(o, 1)); - int ch = cppifyInt(PyList_GetItem(o, 2)); - return Metadata(file, ln, ch); -} - -// Convert node from python wrapper form -Node cppifyNode(PyObject* o) { - Node n; - int isAstNode = cppifyInt(PyList_GetItem(o, 0)); - n.type = isAstNode ? ASTNODE : TOKEN; - n.val = cppifyString(PyList_GetItem(o, 1)); - n.metadata = cppifyMetadata(PyList_GetItem(o, 2)); - std::vector<Node> args; - for (int i = 3; i < PyList_Size(o); i++) { - args.push_back(cppifyNode(PyList_GetItem(o, i))); - } - n.args = args; - return n; -} - -//Convert list of nodes into normal form -std::vector<Node> cppifyNodeList(PyObject* o) { - std::vector<Node> out; - for (int i = 0; i < PyList_Size(o); i++) { - out.push_back(cppifyNode(PyList_GetItem(o,i))); - } - return out; -} - -PYMETHOD(ps_compile, FROMSTR, compile, pyifyString) -PYMETHOD(ps_compile_chunk, FROMSTR, compileChunk, pyifyString) -PYMETHOD(ps_compile_to_lll, FROMSTR, compileToLLL, pyifyNode) -PYMETHOD(ps_compile_chunk_to_lll, FROMSTR, compileChunkToLLL, pyifyNode) -PYMETHOD(ps_compile_lll, FROMNODE, compileLLL, pyifyString) -PYMETHOD(ps_parse, FROMSTR, parseSerpent, pyifyNode) -PYMETHOD(ps_rewrite, FROMNODE, rewrite, pyifyNode) -PYMETHOD(ps_rewrite_chunk, FROMNODE, rewriteChunk, pyifyNode) -PYMETHOD(ps_pretty_compile, FROMSTR, prettyCompile, pyifyNodeList) -PYMETHOD(ps_pretty_compile_chunk, FROMSTR, prettyCompileChunk, pyifyNodeList) -PYMETHOD(ps_pretty_compile_lll, FROMNODE, prettyCompileLLL, pyifyNodeList) -PYMETHOD(ps_serialize, FROMLIST, serialize, pyifyString) -PYMETHOD(ps_deserialize, FROMSTR, deserialize, pyifyNodeList) -PYMETHOD(ps_parse_lll, FROMSTR, parseLLL, pyifyNode) - - -static PyMethodDef PyextMethods[] = { - {"compile", ps_compile, METH_VARARGS, - "Compile code."}, - {"compile_chunk", ps_compile_chunk, METH_VARARGS, - "Compile code chunk (no wrappers)."}, - {"compile_to_lll", ps_compile_to_lll, METH_VARARGS, - "Compile code to LLL."}, - {"compile_chunk_to_lll", ps_compile_chunk_to_lll, METH_VARARGS, - "Compile code chunk to LLL (no wrappers)."}, - {"compile_lll", ps_compile_lll, METH_VARARGS, - "Compile LLL to EVM."}, - {"parse", ps_parse, METH_VARARGS, - "Parse serpent"}, - {"rewrite", ps_rewrite, METH_VARARGS, - "Rewrite parsed serpent to LLL"}, - {"rewrite_chunk", ps_rewrite_chunk, METH_VARARGS, - "Rewrite parsed serpent to LLL (no wrappers)"}, - {"pretty_compile", ps_pretty_compile, METH_VARARGS, - "Compile to EVM opcodes"}, - {"pretty_compile_chunk", ps_pretty_compile_chunk, METH_VARARGS, - "Compile chunk to EVM opcodes (no wrappers)"}, - {"pretty_compile_lll", ps_pretty_compile_lll, METH_VARARGS, - "Compile LLL to EVM opcodes"}, - {"serialize", ps_serialize, METH_VARARGS, - "Convert EVM opcodes to bin"}, - {"deserialize", ps_deserialize, METH_VARARGS, - "Convert EVM bin to opcodes"}, - {"parse_lll", ps_parse_lll, METH_VARARGS, - "Parse LLL"}, - {NULL, NULL, 0, NULL} /* Sentinel */ -}; - -PyMODINIT_FUNC initserpent_pyext(void) -{ - Py_InitModule( "serpent_pyext", PyextMethods ); -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/pyserpent.py b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/pyserpent.py deleted file mode 100644 index 2103b48fe..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/pyserpent.py +++ /dev/null @@ -1 +0,0 @@ -from serpent import * diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriter.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriter.cpp deleted file mode 100644 index 4cdce4f0a..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriter.cpp +++ /dev/null @@ -1,804 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" -#include "lllparser.h" -#include "bignum.h" -#include "optimize.h" -#include "rewriteutils.h" -#include "preprocess.h" -#include "functions.h" -#include "opcodes.h" - -// Rewrite rules -std::string macros[][2] = { - { - "(seq $x)", - "$x" - }, - { - "(seq (seq) $x)", - "$x" - }, - { - "(+= $a $b)", - "(set $a (+ $a $b))" - }, - { - "(*= $a $b)", - "(set $a (* $a $b))" - }, - { - "(-= $a $b)", - "(set $a (- $a $b))" - }, - { - "(/= $a $b)", - "(set $a (/ $a $b))" - }, - { - "(%= $a $b)", - "(set $a (% $a $b))" - }, - { - "(^= $a $b)", - "(set $a (^ $a $b))" - }, - { - "(!= $a $b)", - "(iszero (eq $a $b))" - }, - { - "(assert $x)", - "(unless $x (stop))" - }, - { - "(min $a $b)", - "(with $1 $a (with $2 $b (if (lt $1 $2) $1 $2)))" - }, - { - "(max $a $b)", - "(with $1 $a (with $2 $b (if (lt $1 $2) $2 $1)))" - }, - { - "(smin $a $b)", - "(with $1 $a (with $2 $b (if (slt $1 $2) $1 $2)))" - }, - { - "(smax $a $b)", - "(with $1 $a (with $2 $b (if (slt $1 $2) $2 $1)))" - }, - { - "(if $cond $do (else $else))", - "(if $cond $do $else)" - }, - { - "(code $code)", - "$code" - }, - { - "(slice $arr $pos)", - "(add $arr (mul 32 $pos))", - }, - { - "(array $len)", - "(alloc (mul 32 $len))" - }, - { - "(while $cond $do)", - "(until (iszero $cond) $do)", - }, - { - "(while (iszero $cond) $do)", - "(until $cond $do)", - }, - { - "(if $cond $do)", - "(unless (iszero $cond) $do)", - }, - { - "(if (iszero $cond) $do)", - "(unless $cond $do)", - }, - { - "(access (. self storage) $ind)", - "(sload $ind)" - }, - { - "(access $var $ind)", - "(mload (add $var (mul 32 $ind)))" - }, - { - "(set (access (. self storage) $ind) $val)", - "(sstore $ind $val)" - }, - { - "(set (access $var $ind) $val)", - "(mstore (add $var (mul 32 $ind)) $val)" - }, - { - "(getch $var $ind)", - "(mod (mload (sub (add $var $ind) 31)) 256)" - }, - { - "(setch $var $ind $val)", - "(mstore8 (add $var $ind) $val)", - }, - { - "(send $to $value)", - "(~call (sub (gas) 25) $to $value 0 0 0 0)" - }, - { - "(send $gas $to $value)", - "(~call $gas $to $value 0 0 0 0)" - }, - { - "(sha3 $x)", - "(seq (set $1 $x) (~sha3 (ref $1) 32))" - }, - { - "(sha3 $mstart (= chars $msize))", - "(~sha3 $mstart $msize)" - }, - { - "(sha3 $mstart $msize)", - "(~sha3 $mstart (mul 32 $msize))" - }, - { - "(id $0)", - "$0" - }, - { - "(return $x)", - "(seq (set $1 $x) (~return (ref $1) 32))" - }, - { - "(return $mstart (= chars $msize))", - "(~return $mstart $msize)" - }, - { - "(return $start $len)", - "(~return $start (mul 32 $len))" - }, - { - "(&& $x $y)", - "(if $x $y 0)" - }, - { - "(|| $x $y)", - "(with $1 $x (if $1 $1 $y))" - }, - { - "(>= $x $y)", - "(iszero (slt $x $y))" - }, - { - "(<= $x $y)", - "(iszero (sgt $x $y))" - }, - { - "(create $code)", - "(create 0 $code)" - }, - { - "(create $endowment $code)", - "(with $1 (msize) (create $endowment (get $1) (lll (outer $code) (msize))))" - }, - { - "(sha256 $x)", - "(with $1 (alloc 64) (seq (mstore (add (get $1) 32) $x) (pop (~call 101 2 0 (add (get $1) 32) 32 (get $1) 32)) (mload (get $1))))" - }, - { - "(sha256 $arr (= chars $sz))", - "(with $1 (alloc 32) (seq (pop (~call 101 2 0 $arr $sz (get $1) 32)) (mload (get $1))))" - }, - { - "(sha256 $arr $sz)", - "(with $1 (alloc 32) (seq (pop (~call 101 2 0 $arr (mul 32 $sz) (get $1) 32)) (mload (get $1))))" - }, - { - "(ripemd160 $x)", - "(with $1 (alloc 64) (seq (mstore (add (get $1) 32) $x) (pop (~call 101 3 0 (add (get $1) 32) 32 (get $1) 32)) (mload (get $1))))" - }, - { - "(ripemd160 $arr (= chars $sz))", - "(with $1 (alloc 32) (seq (pop (~call 101 3 0 $arr $sz (mload $1) 32)) (mload (get $1))))" - }, - { - "(ripemd160 $arr $sz)", - "(with $1 (alloc 32) (seq (pop (~call 101 3 0 $arr (mul 32 $sz) (get $1) 32)) (mload (get $1))))" - }, - { - "(ecrecover $h $v $r $s)", - "(with $1 (alloc 160) (seq (mstore (get $1) $h) (mstore (add (get $1) 32) $v) (mstore (add (get $1) 64) $r) (mstore (add (get $1) 96) $s) (pop (~call 101 1 0 (get $1) 128 (add (get $1 128)) 32)) (mload (add (get $1) 128))))" - }, - { - "(inset $x)", - "$x" - }, - { - "(create $x)", - "(with $1 (msize) (create $val (get $1) (lll $code (get $1))))" - }, - { - "(with (= $var $val) $cond)", - "(with $var $val $cond)" - }, - { - "(log $t1)", - "(~log1 0 0 $t1)" - }, - { - "(log $t1 $t2)", - "(~log2 0 0 $t1 $t2)" - }, - { - "(log $t1 $t2 $t3)", - "(~log3 0 0 $t1 $t2 $t3)" - }, - { - "(log $t1 $t2 $t3 $t4)", - "(~log4 0 0 $t1 $t2 $t3 $t4)" - }, - { - "(logarr $a $sz)", - "(~log0 $a (mul 32 $sz))" - }, - { - "(logarr $a $sz $t1)", - "(~log1 $a (mul 32 $sz) $t1)" - }, - { - "(logarr $a $sz $t1 $t2)", - "(~log2 $a (mul 32 $sz) $t1 $t2)" - }, - { - "(logarr $a $sz $t1 $t2 $t3)", - "(~log3 $a (mul 32 $sz) $t1 $t2 $t3)" - }, - { - "(logarr $a $sz $t1 $t2 $t3 $t4)", - "(~log4 $a (mul 32 $sz) $t1 $t2 $t3 $t4)" - }, - { - "(save $loc $array (= chars $count))", - "(with $location (ref $loc) (with $c $count (with $end (div $c 32) (with $i 0 (seq (while (slt $i $end) (seq (sstore (add $i $location) (access $array $i)) (set $i (add $i 1)))) (sstore (add $i $location) (~and (access $array $i) (sub 0 (exp 256 (sub 32 (mod $c 32)))))))))))" - }, - { - "(save $loc $array $count)", - "(with $location (ref $loc) (with $end $count (with $i 0 (while (slt $i $end) (seq (sstore (add $i $location) (access $array $i)) (set $i (add $i 1)))))))" - }, - { - "(load $loc (= chars $count))", - "(with $location (ref $loc) (with $c $count (with $a (alloc $c) (with $i 0 (seq (while (slt $i (div $c 32)) (seq (set (access $a $i) (sload (add $location $i))) (set $i (add $i 1)))) (set (access $a $i) (~and (sload (add $location $i)) (sub 0 (exp 256 (sub 32 (mod $c 32)))))) $a)))))" - }, - { - "(load $loc $count)", - "(with $location (ref $loc) (with $c $count (with $a (alloc $c) (with $i 0 (seq (while (slt $i $c) (seq (set (access $a $i) (sload (add $location $i))) (set $i (add $i 1)))) $a)))))" - }, - { - "(unsafe_mcopy $to $from $sz)", - "(with _sz $sz (with _from $from (with _to $to (seq (comment STARTING UNSAFE MCOPY) (with _i 0 (while (lt _i _sz) (seq (mstore (add $to _i) (mload (add _from _i))) (set _i (add _i 32)))))))))" - }, - { - "(mcopy $to $from $_sz)", - "(with _to $to (with _from $from (with _sz $sz (seq (comment STARTING MCOPY (with _i 0 (seq (while (lt (add _i 31) _sz) (seq (mstore (add _to _i) (mload (add _from _i))) (set _i (add _i 32)))) (with _mask (exp 256 (sub 32 (mod _sz 32))) (mstore (add $to _i) (add (mod (mload (add $to _i)) _mask) (and (mload (add $from _i)) (sub 0 _mask))))))))))))" - }, - { "(. msg sender)", "(caller)" }, - { "(. msg value)", "(callvalue)" }, - { "(. tx gasprice)", "(gasprice)" }, - { "(. tx origin)", "(origin)" }, - { "(. tx gas)", "(gas)" }, - { "(. $x balance)", "(balance $x)" }, - { "self", "(address)" }, - { "(. block prevhash)", "(prevhash)" }, - { "(. block coinbase)", "(coinbase)" }, - { "(. block timestamp)", "(timestamp)" }, - { "(. block number)", "(number)" }, - { "(. block difficulty)", "(difficulty)" }, - { "(. block gaslimit)", "(gaslimit)" }, - { "stop", "(stop)" }, - { "---END---", "" } //Keep this line at the end of the list -}; - -std::vector<rewriteRule> nodeMacros; - -// Token synonyms -std::string synonyms[][2] = { - { "or", "||" }, - { "and", "&&" }, - { "|", "~or" }, - { "&", "~and" }, - { "elif", "if" }, - { "!", "iszero" }, - { "~", "~not" }, - { "not", "iszero" }, - { "string", "alloc" }, - { "+", "add" }, - { "-", "sub" }, - { "*", "mul" }, - { "/", "sdiv" }, - { "^", "exp" }, - { "**", "exp" }, - { "%", "smod" }, - { "<", "slt" }, - { ">", "sgt" }, - { "=", "set" }, - { "==", "eq" }, - { ":", "kv" }, - { "---END---", "" } //Keep this line at the end of the list -}; - -// Custom setters (need to be registered separately -// for use with managed storage) -std::string setters[][2] = { - { "+=", "+" }, - { "-=", "-" }, - { "*=", "*" }, - { "/=", "/" }, - { "%=", "%" }, - { "^=", "^" }, - { "---END---", "" } //Keep this line at the end of the list -}; - -// Processes mutable array literals -Node array_lit_transform(Node node) { - std::string prefix = "_temp"+mkUniqueToken() + "_"; - Metadata m = node.metadata; - std::map<std::string, Node> d; - std::string o = "(seq (set $arr (alloc "+utd(node.args.size()*32)+"))"; - for (unsigned i = 0; i < node.args.size(); i++) { - o += " (mstore (add (get $arr) "+utd(i * 32)+") $"+utd(i)+")"; - d[utd(i)] = node.args[i]; - } - o += " (get $arr))"; - return subst(parseLLL(o), d, prefix, m); -} - - -Node apply_rules(preprocessResult pr); - -// Transform "<variable>.<fun>(args...)" into -// a call -Node dotTransform(Node node, preprocessAux aux) { - Metadata m = node.metadata; - // We're gonna make lots of temporary variables, - // so set up a unique flag for them - std::string prefix = "_temp"+mkUniqueToken()+"_"; - // Check that the function name is a token - if (node.args[0].args[1].type == ASTNODE) - err("Function name must be static", m); - - Node dotOwner = node.args[0].args[0]; - std::string dotMember = node.args[0].args[1].val; - // kwargs = map of special arguments - std::map<std::string, Node> kwargs; - kwargs["value"] = token("0", m); - kwargs["gas"] = subst(parseLLL("(- (gas) 25)"), msn(), prefix, m); - // Search for as=? and call=code keywords, and isolate the actual - // function arguments - std::vector<Node> fnargs; - std::string as = ""; - std::string op = "call"; - for (unsigned i = 1; i < node.args.size(); i++) { - fnargs.push_back(node.args[i]); - Node arg = fnargs.back(); - if (arg.val == "=" || arg.val == "set") { - if (arg.args[0].val == "as") - as = arg.args[1].val; - if (arg.args[0].val == "call" && arg.args[1].val == "code") - op = "callcode"; - if (arg.args[0].val == "gas") - kwargs["gas"] = arg.args[1]; - if (arg.args[0].val == "value") - kwargs["value"] = arg.args[1]; - if (arg.args[0].val == "outsz") - kwargs["outsz"] = arg.args[1]; - } - } - if (dotOwner.val == "self") { - if (as.size()) err("Cannot use \"as\" when calling self!", m); - as = dotOwner.val; - } - // Determine the funId and sig assuming the "as" keyword was used - int funId = 0; - std::string sig; - if (as.size() > 0 && aux.localExterns.count(as)) { - if (!aux.localExterns[as].count(dotMember)) - err("Invalid call: "+printSimple(dotOwner)+"."+dotMember, m); - funId = aux.localExterns[as][dotMember]; - sig = aux.localExternSigs[as][dotMember]; - } - // Determine the funId and sig otherwise - else if (!as.size()) { - if (!aux.globalExterns.count(dotMember)) - err("Invalid call: "+printSimple(dotOwner)+"."+dotMember, m); - std::string key = unsignedToDecimal(aux.globalExterns[dotMember]); - funId = aux.globalExterns[dotMember]; - sig = aux.globalExternSigs[dotMember]; - } - else err("Invalid call: "+printSimple(dotOwner)+"."+dotMember, m); - // Pack arguments - kwargs["data"] = packArguments(fnargs, sig, funId, m); - kwargs["to"] = dotOwner; - Node main; - // Pack output - if (!kwargs.count("outsz")) { - main = parseLLL( - "(with _data $data (seq " - "(pop (~"+op+" $gas $to $value (access _data 0) (access _data 1) (ref $dataout) 32))" - "(get $dataout)))"); - } - else { - main = parseLLL( - "(with _data $data (with _outsz (mul 32 $outsz) (with _out (alloc _outsz) (seq " - "(pop (~"+op+" $gas $to $value (access _data 0) (access _data 1) _out _outsz))" - "(get _out)))))"); - } - // Set up main call - - Node o = subst(main, kwargs, prefix, m); - return o; -} - -// Transform an access of the form self.bob, self.users[5], etc into -// a storage access -// -// There exist two types of objects: finite objects, and infinite -// objects. Finite objects are packed optimally tightly into storage -// accesses; for example: -// -// data obj[100](a, b[2][4], c) -// -// obj[0].a -> 0 -// obj[0].b[0][0] -> 1 -// obj[0].b[1][3] -> 8 -// obj[45].c -> 459 -// -// Infinite objects are accessed by sha3([v1, v2, v3 ... ]), where -// the values are a list of array indices and keyword indices, for -// example: -// data obj[](a, b[2][4], c) -// data obj2[](a, b[][], c) -// -// obj[0].a -> sha3([0, 0, 0]) -// obj[5].b[1][3] -> sha3([0, 5, 1, 1, 3]) -// obj[45].c -> sha3([0, 45, 2]) -// obj2[0].a -> sha3([1, 0, 0]) -// obj2[5].b[1][3] -> sha3([1, 5, 1, 1, 3]) -// obj2[45].c -> sha3([1, 45, 2]) -Node storageTransform(Node node, preprocessAux aux, - bool mapstyle=false, bool ref=false) { - Metadata m = node.metadata; - // Get a list of all of the "access parameters" used in order - // eg. self.users[5].cow[4][m[2]][woof] -> - // [--self, --users, 5, --cow, 4, m[2], woof] - std::vector<Node> hlist = listfyStorageAccess(node); - // For infinite arrays, the terms array will just provide a list - // of indices. For finite arrays, it's a list of index*coefficient - std::vector<Node> terms; - std::string offset = "0"; - std::string prefix = ""; - std::string varPrefix = "_temp"+mkUniqueToken()+"_"; - int c = 0; - std::vector<std::string> coefficients; - coefficients.push_back(""); - for (unsigned i = 1; i < hlist.size(); i++) { - // We pre-add the -- flag to parameter-like terms. For example, - // self.users[m] -> [--self, --users, m] - // self.users.m -> [--self, --users, --m] - if (hlist[i].val.substr(0, 2) == "--") { - prefix += hlist[i].val.substr(2) + "."; - std::string tempPrefix = prefix.substr(0, prefix.size()-1); - if (!aux.storageVars.offsets.count(tempPrefix)) - return node; - if (c < (signed)coefficients.size() - 1) - err("Too few array index lookups", m); - if (c > (signed)coefficients.size() - 1) - err("Too many array index lookups", m); - coefficients = aux.storageVars.coefficients[tempPrefix]; - // If the size of an object exceeds 2^176, we make it an infinite - // array - if (decimalGt(coefficients.back(), tt176) && !mapstyle) - return storageTransform(node, aux, true, ref); - offset = decimalAdd(offset, aux.storageVars.offsets[tempPrefix]); - c = 0; - if (mapstyle) - terms.push_back(token(unsignedToDecimal( - aux.storageVars.indices[tempPrefix]))); - } - else if (mapstyle) { - terms.push_back(hlist[i]); - c += 1; - } - else { - if (c > (signed)coefficients.size() - 2) - err("Too many array index lookups", m); - terms.push_back( - astnode("mul", - hlist[i], - token(coefficients[coefficients.size() - 2 - c], m), - m)); - - c += 1; - } - } - if (aux.storageVars.nonfinal.count(prefix.substr(0, prefix.size()-1))) - err("Storage variable access not deep enough", m); - if (c < (signed)coefficients.size() - 1) { - err("Too few array index lookups", m); - } - if (c > (signed)coefficients.size() - 1) { - err("Too many array index lookups", m); - } - Node o; - if (mapstyle) { - std::string t = "_temp_"+mkUniqueToken(); - std::vector<Node> sub; - for (unsigned i = 0; i < terms.size(); i++) - sub.push_back(asn("mstore", - asn("add", - tkn(utd(i * 32), m), - asn("get", tkn(t+"pos", m), m), - m), - terms[i], - m)); - sub.push_back(tkn(t+"pos", m)); - Node main = asn("with", - tkn(t+"pos", m), - asn("alloc", tkn(utd(terms.size() * 32), m), m), - asn("seq", sub, m), - m); - Node sz = token(utd(terms.size() * 32), m); - o = astnode("~sha3", - main, - sz, - m); - } - else { - // We add up all the index*coefficients - Node out = token(offset, node.metadata); - for (unsigned i = 0; i < terms.size(); i++) { - std::vector<Node> temp; - temp.push_back(out); - temp.push_back(terms[i]); - out = astnode("add", temp, node.metadata); - } - o = out; - } - if (ref) return o; - else return astnode("sload", o, node.metadata); -} - - -// Recursively applies rewrite rules -std::pair<Node, bool> apply_rules_iter(preprocessResult pr) { - bool changed = false; - Node node = pr.first; - // If the rewrite rules have not yet been parsed, parse them - if (!nodeMacros.size()) { - for (int i = 0; i < 9999; i++) { - std::vector<Node> o; - if (macros[i][0] == "---END---") break; - nodeMacros.push_back(rewriteRule( - parseLLL(macros[i][0]), - parseLLL(macros[i][1]) - )); - } - } - // Assignment transformations - for (int i = 0; i < 9999; i++) { - if (setters[i][0] == "---END---") break; - if (node.val == setters[i][0]) { - node = astnode("=", - node.args[0], - astnode(setters[i][1], - node.args[0], - node.args[1], - node.metadata), - node.metadata); - } - } - // Do nothing to macros - if (node.val == "macro") { - return std::pair<Node, bool>(node, changed); - } - // Ignore comments - if (node.val == "comment") { - return std::pair<Node, bool>(node, changed); - } - // Special storage transformation - if (isNodeStorageVariable(node)) { - node = storageTransform(node, pr.second); - changed = true; - } - if (node.val == "ref" && isNodeStorageVariable(node.args[0])) { - node = storageTransform(node.args[0], pr.second, false, true); - changed = true; - } - if (node.val == "=" && isNodeStorageVariable(node.args[0])) { - Node t = storageTransform(node.args[0], pr.second); - if (t.val == "sload") { - std::vector<Node> o; - o.push_back(t.args[0]); - o.push_back(node.args[1]); - node = astnode("sstore", o, node.metadata); - } - changed = true; - } - // Main code - unsigned pos = 0; - std::string prefix = "_temp"+mkUniqueToken()+"_"; - while(1) { - if (synonyms[pos][0] == "---END---") { - break; - } - else if (node.type == ASTNODE && node.val == synonyms[pos][0]) { - node.val = synonyms[pos][1]; - changed = true; - } - pos++; - } - for (pos = 0; pos < nodeMacros.size() + pr.second.customMacros.size(); pos++) { - rewriteRule macro = pos < nodeMacros.size() - ? nodeMacros[pos] - : pr.second.customMacros[pos - nodeMacros.size()]; - matchResult mr = match(macro.pattern, node); - if (mr.success) { - node = subst(macro.substitution, mr.map, prefix, node.metadata); - std::pair<Node, bool> o = - apply_rules_iter(preprocessResult(node, pr.second)); - o.second = true; - return o; - } - } - // Special transformations - if (node.val == "outer") { - node = apply_rules(preprocess(node.args[0])); - changed = true; - } - if (node.val == "array_lit") { - node = array_lit_transform(node); - changed = true; - } - if (node.val == "fun" && node.args[0].val == ".") { - node = dotTransform(node, pr.second); - changed = true; - } - if (node.type == ASTNODE) { - unsigned i = 0; - if (node.val == "set" || node.val == "ref" - || node.val == "get" || node.val == "with") { - if (node.args[0].val.size() > 0 && node.args[0].val[0] != '\'' - && node.args[0].type == TOKEN && node.args[0].val[0] != '$') { - node.args[0].val = "'" + node.args[0].val; - changed = true; - } - i = 1; - } - else if (node.val == "arglen") { - node.val = "get"; - node.args[0].val = "'_len_" + node.args[0].val; - i = 1; - changed = true; - } - for (; i < node.args.size(); i++) { - std::pair<Node, bool> r = - apply_rules_iter(preprocessResult(node.args[i], pr.second)); - node.args[i] = r.first; - changed = changed || r.second; - } - } - else if (node.type == TOKEN && !isNumberLike(node)) { - if (node.val.size() >= 2 - && node.val[0] == '"' - && node.val[node.val.size() - 1] == '"') { - std::string bin = node.val.substr(1, node.val.size() - 2); - unsigned sz = bin.size(); - std::vector<Node> o; - for (unsigned i = 0; i < sz; i += 32) { - std::string t = binToNumeric(bin.substr(i, 32)); - if ((sz - i) < 32 && (sz - i) > 0) { - while ((sz - i) < 32) { - t = decimalMul(t, "256"); - i--; - } - i = sz; - } - o.push_back(token(t, node.metadata)); - } - node = astnode("array_lit", o, node.metadata); - std::pair<Node, bool> r = - apply_rules_iter(preprocessResult(node, pr.second)); - node = r.first; - changed = true; - } - else if (node.val.size() && node.val[0] != '\'' && node.val[0] != '$') { - node.val = "'" + node.val; - std::vector<Node> args; - args.push_back(node); - std::string v = node.val.substr(1); - node = astnode("get", args, node.metadata); - changed = true; - } - } - return std::pair<Node, bool>(node, changed); -} - -Node apply_rules(preprocessResult pr) { - for (unsigned i = 0; i < pr.second.customMacros.size(); i++) { - pr.second.customMacros[i].pattern = - apply_rules(preprocessResult(pr.second.customMacros[i].pattern, preprocessAux())); - } - while (1) { - //std::cerr << printAST(pr.first) << - // " " << pr.second.customMacros.size() << "\n"; - std::pair<Node, bool> r = apply_rules_iter(pr); - if (!r.second) { - return r.first; - } - pr.first = r.first; - } -} - -Node validate(Node inp) { - Metadata m = inp.metadata; - if (inp.type == ASTNODE) { - int i = 0; - while(validFunctions[i][0] != "---END---") { - if (inp.val == validFunctions[i][0]) { - std::string sz = unsignedToDecimal(inp.args.size()); - if (decimalGt(validFunctions[i][1], sz)) { - err("Too few arguments for "+inp.val, inp.metadata); - } - if (decimalGt(sz, validFunctions[i][2])) { - err("Too many arguments for "+inp.val, inp.metadata); - } - } - i++; - } - } - for (unsigned i = 0; i < inp.args.size(); i++) validate(inp.args[i]); - return inp; -} - -Node postValidate(Node inp) { - // This allows people to use ~x as a way of having functions with the same - // name and arity as macros; the idea is that ~x is a "final" form, and - // should not be remacroed, but it is converted back at the end - if (inp.val.size() > 0 && inp.val[0] == '~') { - inp.val = inp.val.substr(1); - } - if (inp.type == ASTNODE) { - if (inp.val == ".") - err("Invalid object member (ie. a foo.bar not mapped to anything)", - inp.metadata); - else if (opcode(inp.val) >= 0) { - if ((signed)inp.args.size() < opinputs(inp.val)) - err("Too few arguments for "+inp.val, inp.metadata); - if ((signed)inp.args.size() > opinputs(inp.val)) - err("Too many arguments for "+inp.val, inp.metadata); - } - else if (isValidLLLFunc(inp.val, inp.args.size())) { - // do nothing - } - else err ("Invalid argument count or LLL function: "+inp.val, inp.metadata); - for (unsigned i = 0; i < inp.args.size(); i++) { - inp.args[i] = postValidate(inp.args[i]); - } - } - return inp; -} - -Node rewrite(Node inp) { - return postValidate(optimize(apply_rules(preprocess(inp)))); -} - -Node rewriteChunk(Node inp) { - return postValidate(optimize(apply_rules( - preprocessResult( - validate(inp), preprocessAux())))); -} - -using namespace std; diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriter.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriter.h deleted file mode 100644 index 716815cee..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriter.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef ETHSERP_REWRITER -#define ETHSERP_REWRITER - -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" - -// Applies rewrite rules -Node rewrite(Node inp); - -// Applies rewrite rules adding without wrapper -Node rewriteChunk(Node inp); - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriteutils.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriteutils.cpp deleted file mode 100644 index 0d810bdbc..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriteutils.cpp +++ /dev/null @@ -1,211 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" -#include "lllparser.h" -#include "bignum.h" -#include "rewriteutils.h" -#include "optimize.h" - -// Valid functions and their min and max argument counts -std::string validFunctions[][3] = { - { "if", "2", "3" }, - { "unless", "2", "2" }, - { "while", "2", "2" }, - { "until", "2", "2" }, - { "alloc", "1", "1" }, - { "array", "1", "1" }, - { "call", "2", tt256 }, - { "callcode", "2", tt256 }, - { "create", "1", "4" }, - { "getch", "2", "2" }, - { "setch", "3", "3" }, - { "sha3", "1", "2" }, - { "return", "1", "2" }, - { "inset", "1", "1" }, - { "min", "2", "2" }, - { "max", "2", "2" }, - { "array_lit", "0", tt256 }, - { "seq", "0", tt256 }, - { "log", "1", "6" }, - { "outer", "1", "1" }, - { "set", "2", "2" }, - { "get", "1", "1" }, - { "ref", "1", "1" }, - { "declare", "1", tt256 }, - { "with", "3", "3" }, - { "outer", "1", "1" }, - { "mcopy", "3", "3" }, - { "unsafe_mcopy", "3", "3" }, - { "save", "3", "3" }, - { "load", "2", "2" }, - { "---END---", "", "" } //Keep this line at the end of the list -}; - -std::map<std::string, bool> vfMap; - -// Is a function name one of the valid functions above? -bool isValidFunctionName(std::string f) { - if (vfMap.size() == 0) { - for (int i = 0; ; i++) { - if (validFunctions[i][0] == "---END---") break; - vfMap[validFunctions[i][0]] = true; - } - } - return vfMap.count(f); -} - -// Cool function for debug purposes (named cerrStringList to make -// all prints searchable via 'cerr') -void cerrStringList(std::vector<std::string> s, std::string suffix) { - for (unsigned i = 0; i < s.size(); i++) std::cerr << s[i] << " "; - std::cerr << suffix << "\n"; -} - -// Convert: -// self.cow -> ["cow"] -// self.horse[0] -> ["horse", "0"] -// self.a[6][7][self.storage[3]].chicken[9] -> -// ["6", "7", (sload 3), "chicken", "9"] -std::vector<Node> listfyStorageAccess(Node node) { - std::vector<Node> out; - std::vector<Node> nodez; - nodez.push_back(node); - while (1) { - if (nodez.back().type == TOKEN) { - out.push_back(token("--" + nodez.back().val, node.metadata)); - std::vector<Node> outrev; - for (int i = (signed)out.size() - 1; i >= 0; i--) { - outrev.push_back(out[i]); - } - return outrev; - } - if (nodez.back().val == ".") - nodez.back().args[1].val = "--" + nodez.back().args[1].val; - if (nodez.back().args.size() == 0) - err("Error parsing storage variable statement", node.metadata); - if (nodez.back().args.size() == 1) - out.push_back(token(tt256m1, node.metadata)); - else - out.push_back(nodez.back().args[1]); - nodez.push_back(nodez.back().args[0]); - } -} - -// Is the given node something of the form -// self.cow -// self.horse[0] -// self.a[6][7][self.storage[3]].chicken[9] -bool isNodeStorageVariable(Node node) { - std::vector<Node> nodez; - nodez.push_back(node); - while (1) { - if (nodez.back().type == TOKEN) return false; - if (nodez.back().args.size() == 0) return false; - if (nodez.back().val != "." && nodez.back().val != "access") - return false; - if (nodez.back().args[0].val == "self") return true; - nodez.push_back(nodez.back().args[0]); - } -} - -// Main pattern matching routine, for those patterns that can be expressed -// using our standard mini-language above -// -// Returns two values. First, a boolean to determine whether the node matches -// the pattern, second, if the node does match then a map mapping variables -// in the pattern to nodes -matchResult match(Node p, Node n) { - matchResult o; - o.success = false; - if (p.type == TOKEN) { - if (p.val == n.val && n.type == TOKEN) o.success = true; - else if (p.val[0] == '$' || p.val[0] == '@') { - o.success = true; - o.map[p.val.substr(1)] = n; - } - } - else if (n.type==TOKEN || p.val!=n.val || p.args.size()!=n.args.size()) { - // do nothing - } - else { - for (unsigned i = 0; i < p.args.size(); i++) { - matchResult oPrime = match(p.args[i], n.args[i]); - if (!oPrime.success) { - o.success = false; - return o; - } - for (std::map<std::string, Node>::iterator it = oPrime.map.begin(); - it != oPrime.map.end(); - it++) { - o.map[(*it).first] = (*it).second; - } - } - o.success = true; - } - return o; -} - - -// Fills in the pattern with a dictionary mapping variable names to -// nodes (these dicts are generated by match). Match and subst together -// create a full pattern-matching engine. -Node subst(Node pattern, - std::map<std::string, Node> dict, - std::string varflag, - Metadata m) { - // Swap out patterns at the token level - if (pattern.metadata.ln == -1) - pattern.metadata = m; - if (pattern.type == TOKEN && - pattern.val[0] == '$') { - if (dict.count(pattern.val.substr(1))) { - return dict[pattern.val.substr(1)]; - } - else { - return token(varflag + pattern.val.substr(1), m); - } - } - // Other tokens are untouched - else if (pattern.type == TOKEN) { - return pattern; - } - // Substitute recursively for ASTs - else { - std::vector<Node> args; - for (unsigned i = 0; i < pattern.args.size(); i++) { - args.push_back(subst(pattern.args[i], dict, varflag, m)); - } - return asn(pattern.val, args, m); - } -} - -// Transforms a sequence containing two-argument with statements -// into a statement containing those statements in nested form -Node withTransform (Node source) { - Node o = token("--"); - Metadata m = source.metadata; - std::vector<Node> args; - for (int i = source.args.size() - 1; i >= 0; i--) { - Node a = source.args[i]; - if (a.val == "with" && a.args.size() == 2) { - std::vector<Node> flipargs; - for (int j = args.size() - 1; j >= 0; j--) - flipargs.push_back(args[i]); - if (o.val != "--") - flipargs.push_back(o); - o = asn("with", a.args[0], a.args[1], asn("seq", flipargs, m), m); - args = std::vector<Node>(); - } - else { - args.push_back(a); - } - } - std::vector<Node> flipargs; - for (int j = args.size() - 1; j >= 0; j--) - flipargs.push_back(args[j]); - if (o.val != "--") - flipargs.push_back(o); - return asn("seq", flipargs, m); -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriteutils.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriteutils.h deleted file mode 100644 index 8abf44a9f..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/rewriteutils.h +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef ETHSERP_REWRITEUTILS -#define ETHSERP_REWRITEUTILS - -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" - -// Valid functions and their min and max argument counts -extern std::string validFunctions[][3]; - -extern std::map<std::string, bool> vfMap; - -bool isValidFunctionName(std::string f); - -// Converts deep array access into ordered list of the arguments -// along the descent -std::vector<Node> listfyStorageAccess(Node node); - -// Cool function for debug purposes (named cerrStringList to make -// all prints searchable via 'cerr') -void cerrStringList(std::vector<std::string> s, std::string suffix=""); - -// Is the given node something of the form -// self.cow -// self.horse[0] -// self.a[6][7][self.storage[3]].chicken[9] -bool isNodeStorageVariable(Node node); - -// Applies rewrite rules adding without wrapper -Node rewriteChunk(Node inp); - -// Match result storing object -struct matchResult { - bool success; - std::map<std::string, Node> map; -}; - -// Match node to pattern -matchResult match(Node p, Node n); - -// Substitute node using pattern -Node subst(Node pattern, - std::map<std::string, Node> dict, - std::string varflag, - Metadata m); - -Node withTransform(Node source); - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/serpent.py b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/serpent.py deleted file mode 100644 index 8d6bedfe3..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/serpent.py +++ /dev/null @@ -1,201 +0,0 @@ -import serpent_pyext as pyext -import sys -import re - -VERSION = '1.7.7' - - -class Metadata(object): - def __init__(self, li): - self.file = li[0] - self.ln = li[1] - self.ch = li[2] - - def out(self): - return [self.file, self.ln, self.ch] - - -class Token(object): - def __init__(self, val, metadata): - self.val = val - self.metadata = Metadata(metadata) - - def out(self): - return [0, self.val, self.metadata.out()] - - def __repr__(self): - return str(self.val) - - -class Astnode(object): - def __init__(self, val, args, metadata): - self.val = val - self.args = map(node, args) - self.metadata = Metadata(metadata) - - def out(self): - o = [1, self.val, self.metadata.out()]+[x.out() for x in self.args] - return o - - def __repr__(self): - o = '(' + self.val - subs = map(repr, self.args) - k = 0 - out = " " - while k < len(subs) and o != "(seq": - if '\n' in subs[k] or len(out + subs[k]) >= 80: - break - out += subs[k] + " " - k += 1 - if k < len(subs): - o += out + "\n " - o += '\n '.join('\n'.join(subs[k:]).split('\n')) - o += '\n)' - else: - o += out[:-1] + ')' - return o - - -def node(li): - if li[0]: - return Astnode(li[1], li[3:], li[2]) - else: - return Token(li[1], li[2]) - - -def take(x): - return pyext.parse_lll(x) if isinstance(x, (str, unicode)) else x.out() - - -def takelist(x): - return map(take, parse(x).args if isinstance(x, (str, unicode)) else x) - - -compile = lambda x: pyext.compile(x) -compile_chunk = lambda x: pyext.compile_chunk(x) -compile_to_lll = lambda x: node(pyext.compile_to_lll(x)) -compile_chunk_to_lll = lambda x: node(pyext.compile_chunk_to_lll(x)) -compile_lll = lambda x: pyext.compile_lll(take(x)) -parse = lambda x: node(pyext.parse(x)) -rewrite = lambda x: node(pyext.rewrite(take(x))) -rewrite_chunk = lambda x: node(pyext.rewrite_chunk(take(x))) -pretty_compile = lambda x: map(node, pyext.pretty_compile(x)) -pretty_compile_chunk = lambda x: map(node, pyext.pretty_compile_chunk(x)) -pretty_compile_lll = lambda x: map(node, pyext.pretty_compile_lll(take(x))) -serialize = lambda x: pyext.serialize(takelist(x)) -deserialize = lambda x: map(node, pyext.deserialize(x)) - -is_numeric = lambda x: isinstance(x, (int, long)) -is_string = lambda x: isinstance(x, (str, unicode)) -tobytearr = lambda n, L: [] if L == 0 else tobytearr(n / 256, L - 1)+[n % 256] - - -# A set of methods for detecting raw values (numbers and strings) and -# converting them to integers -def frombytes(b): - return 0 if len(b) == 0 else ord(b[-1]) + 256 * frombytes(b[:-1]) - - -def fromhex(b): - hexord = lambda x: '0123456789abcdef'.find(x) - return 0 if len(b) == 0 else hexord(b[-1]) + 16 * fromhex(b[:-1]) - - -def numberize(b): - if is_numeric(b): - return b - elif b[0] in ["'", '"']: - return frombytes(b[1:-1]) - elif b[:2] == '0x': - return fromhex(b[2:]) - elif re.match('^[0-9]*$', b): - return int(b) - elif len(b) == 40: - return fromhex(b) - else: - raise Exception("Cannot identify data type: %r" % b) - - -def enc(n): - if is_numeric(n): - return ''.join(map(chr, tobytearr(n, 32))) - elif is_string(n) and len(n) == 40: - return '\x00' * 12 + n.decode('hex') - elif is_string(n): - return '\x00' * (32 - len(n)) + n - elif n is True: - return 1 - elif n is False or n is None: - return 0 - - -def encode_datalist(*args): - if isinstance(args, (tuple, list)): - return ''.join(map(enc, args)) - elif not len(args) or args[0] == '': - return '' - else: - # Assume you're getting in numbers or addresses or 0x... - return ''.join(map(enc, map(numberize, args))) - - -def decode_datalist(arr): - if isinstance(arr, list): - arr = ''.join(map(chr, arr)) - o = [] - for i in range(0, len(arr), 32): - o.append(frombytes(arr[i:i + 32])) - return o - - -def encode_abi(funid, *args): - len_args = '' - normal_args = '' - var_args = '' - for arg in args: - if isinstance(arg, str) and len(arg) and \ - arg[0] == '"' and arg[-1] == '"': - len_args += enc(numberize(len(arg[1:-1]))) - var_args += arg[1:-1] - elif isinstance(arg, list): - for a in arg: - var_args += enc(numberize(a)) - len_args += enc(numberize(len(arg))) - else: - normal_args += enc(numberize(arg)) - return chr(int(funid)) + len_args + normal_args + var_args - - -def decode_abi(arr, *lens): - o = [] - pos = 1 - i = 0 - if len(lens) == 1 and isinstance(lens[0], list): - lens = lens[0] - while pos < len(arr): - bytez = int(lens[i]) if i < len(lens) else 32 - o.append(frombytes(arr[pos: pos + bytez])) - i, pos = i + 1, pos + bytez - return o - - -def main(): - if len(sys.argv) == 1: - print "serpent <command> <arg1> <arg2> ..." - else: - cmd = sys.argv[2] if sys.argv[1] == '-s' else sys.argv[1] - if sys.argv[1] == '-s': - args = [sys.stdin.read()] + sys.argv[3:] - elif sys.argv[1] == '-v': - print VERSION - sys.exit() - else: - cmd = sys.argv[1] - args = sys.argv[2:] - if cmd in ['deserialize', 'decode_datalist', 'decode_abi']: - args[0] = args[0].strip().decode('hex') - o = globals()[cmd](*args) - if isinstance(o, (Token, Astnode, list)): - print repr(o) - else: - print o.encode('hex') diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/setup.py b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/setup.py deleted file mode 100644 index 5fdc1c16a..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -from setuptools import setup, Extension - -import os -from distutils.sysconfig import get_config_vars - -(opt,) = get_config_vars('OPT') -os.environ['OPT'] = " ".join( - flag for flag in opt.split() if flag != '-Wstrict-prototypes' -) - -setup( - # Name of this package - name="ethereum-serpent", - - # Package version - version='1.7.7', - - description='Serpent compiler', - maintainer='Vitalik Buterin', - maintainer_email='v@buterin.com', - license='WTFPL', - url='http://www.ethereum.org/', - - # Describes how to build the actual extension module from C source files. - ext_modules=[ - Extension( - 'serpent_pyext', # Python name of the module - ['bignum.cpp', 'util.cpp', 'tokenize.cpp', - 'lllparser.cpp', 'parser.cpp', 'functions.cpp', - 'optimize.cpp', 'opcodes.cpp', - 'rewriteutils.cpp', 'preprocess.cpp', 'rewriter.cpp', - 'compiler.cpp', 'funcs.cpp', 'pyserpent.cpp'] - )], - py_modules=[ - 'serpent', - 'pyserpent' - ], - scripts=[ - 'serpent.py' - ], - entry_points={ - 'console_scripts': [ - 'serpent = serpent:main', - ], - } - ), diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/tokenize.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/tokenize.cpp deleted file mode 100644 index b60cc8a44..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/tokenize.cpp +++ /dev/null @@ -1,115 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" - -// These appear as independent tokens even if inside a stream of symbols -const std::string atoms[] = { "#", "//", "(", ")", "[", "]", "{", "}" }; -const int numAtoms = 8; - -// Is the char alphanumeric, a space, a bracket, a quote, a symbol? -int chartype(char c) { - if (c >= '0' && c <= '9') return ALPHANUM; - else if (c >= 'a' && c <= 'z') return ALPHANUM; - else if (c >= 'A' && c <= 'Z') return ALPHANUM; - else if (std::string("~_$@").find(c) != std::string::npos) return ALPHANUM; - else if (c == '\t' || c == ' ' || c == '\n' || c == '\r') return SPACE; - else if (std::string("()[]{}").find(c) != std::string::npos) return BRACK; - else if (c == '"') return DQUOTE; - else if (c == '\'') return SQUOTE; - else return SYMB; -} - -// "y = f(45,124)/3" -> [ "y", "f", "(", "45", ",", "124", ")", "/", "3"] -std::vector<Node> tokenize(std::string inp, Metadata metadata, bool lispMode) { - int curtype = SPACE; - unsigned pos = 0; - int lastNewline = 0; - metadata.ch = 0; - std::string cur; - std::vector<Node> out; - - inp += " "; - while (pos < inp.length()) { - int headtype = chartype(inp[pos]); - if (lispMode) { - if (inp[pos] == '\'') headtype = ALPHANUM; - } - // Are we inside a quote? - if (curtype == SQUOTE || curtype == DQUOTE) { - // Close quote - if (headtype == curtype) { - cur += inp[pos]; - out.push_back(token(cur, metadata)); - cur = ""; - metadata.ch = pos - lastNewline; - curtype = SPACE; - pos += 1; - } - // eg. \xc3 - else if (inp.length() >= pos + 4 && inp.substr(pos, 2) == "\\x") { - cur += (std::string("0123456789abcdef").find(inp[pos+2]) * 16 - + std::string("0123456789abcdef").find(inp[pos+3])); - pos += 4; - } - // Newline - else if (inp.substr(pos, 2) == "\\n") { - cur += '\n'; - pos += 2; - } - // Backslash escape - else if (inp.length() >= pos + 2 && inp[pos] == '\\') { - cur += inp[pos + 1]; - pos += 2; - } - // Normal character - else { - cur += inp[pos]; - pos += 1; - } - } - else { - // Handle atoms ( '//', '#', brackets ) - for (int i = 0; i < numAtoms; i++) { - int split = cur.length() - atoms[i].length(); - if (split >= 0 && cur.substr(split) == atoms[i]) { - if (split > 0) { - out.push_back(token(cur.substr(0, split), metadata)); - } - metadata.ch += split; - out.push_back(token(cur.substr(split), metadata)); - metadata.ch = pos - lastNewline; - cur = ""; - curtype = SPACE; - } - } - // Special case the minus sign - if (cur.length() > 1 && (cur.substr(cur.length() - 1) == "-" - || cur.substr(cur.length() - 1) == "!")) { - out.push_back(token(cur.substr(0, cur.length() - 1), metadata)); - out.push_back(token(cur.substr(cur.length() - 1), metadata)); - cur = ""; - } - // Boundary between different char types - if (headtype != curtype) { - if (curtype != SPACE && cur != "") { - out.push_back(token(cur, metadata)); - } - metadata.ch = pos - lastNewline; - cur = ""; - } - cur += inp[pos]; - curtype = headtype; - pos += 1; - } - if (inp[pos] == '\n') { - lastNewline = pos; - metadata.ch = 0; - metadata.ln += 1; - } - } - return out; -} - - diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/tokenize.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/tokenize.h deleted file mode 100644 index 04a42f3c6..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/tokenize.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef ETHSERP_TOKENIZE -#define ETHSERP_TOKENIZE - -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" - -int chartype(char c); - -std::vector<Node> tokenize(std::string inp, - Metadata meta=Metadata(), - bool lispMode=false); - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/util.cpp b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/util.cpp deleted file mode 100644 index 56f642fc8..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/util.cpp +++ /dev/null @@ -1,305 +0,0 @@ -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include "util.h" -#include "bignum.h" -#include <fstream> -#include <cerrno> - -//Token or value node constructor -Node token(std::string val, Metadata met) { - Node o; - o.type = 0; - o.val = val; - o.metadata = met; - return o; -} - -//AST node constructor -Node astnode(std::string val, std::vector<Node> args, Metadata met) { - Node o; - o.type = 1; - o.val = val; - o.args = args; - o.metadata = met; - return o; -} - -//AST node constructors for a specific number of children -Node astnode(std::string val, Metadata met) { - std::vector<Node> args; - return astnode(val, args, met); -} - -Node astnode(std::string val, Node a, Metadata met) { - std::vector<Node> args; - args.push_back(a); - return astnode(val, args, met); -} - -Node astnode(std::string val, Node a, Node b, Metadata met) { - std::vector<Node> args; - args.push_back(a); - args.push_back(b); - return astnode(val, args, met); -} - -Node astnode(std::string val, Node a, Node b, Node c, Metadata met) { - std::vector<Node> args; - args.push_back(a); - args.push_back(b); - args.push_back(c); - return astnode(val, args, met); -} - -Node astnode(std::string val, Node a, Node b, Node c, Node d, Metadata met) { - std::vector<Node> args; - args.push_back(a); - args.push_back(b); - args.push_back(c); - args.push_back(d); - return astnode(val, args, met); -} - - -// Print token list -std::string printTokens(std::vector<Node> tokens) { - std::string s = ""; - for (unsigned i = 0; i < tokens.size(); i++) { - s += tokens[i].val + " "; - } - return s; -} - -// Prints a lisp AST on one line -std::string printSimple(Node ast) { - if (ast.type == TOKEN) return ast.val; - std::string o = "(" + ast.val; - std::vector<std::string> subs; - for (unsigned i = 0; i < ast.args.size(); i++) { - o += " " + printSimple(ast.args[i]); - } - return o + ")"; -} - -// Number of tokens in a tree -int treeSize(Node prog) { - if (prog.type == TOKEN) return 1; - int o = 0; - for (unsigned i = 0; i < prog.args.size(); i++) o += treeSize(prog.args[i]); - return o; -} - -// Pretty-prints a lisp AST -std::string printAST(Node ast, bool printMetadata) { - if (ast.type == TOKEN) return ast.val; - std::string o = "("; - if (printMetadata) { - o += ast.metadata.file + " "; - o += unsignedToDecimal(ast.metadata.ln) + " "; - o += unsignedToDecimal(ast.metadata.ch) + ": "; - } - o += ast.val; - std::vector<std::string> subs; - for (unsigned i = 0; i < ast.args.size(); i++) { - subs.push_back(printAST(ast.args[i], printMetadata)); - } - unsigned k = 0; - std::string out = " "; - // As many arguments as possible go on the same line as the function, - // except when seq is used - while (k < subs.size() && o != "(seq") { - if (subs[k].find("\n") != std::string::npos || (out + subs[k]).length() >= 80) break; - out += subs[k] + " "; - k += 1; - } - // All remaining arguments go on their own lines - if (k < subs.size()) { - o += out + "\n"; - std::vector<std::string> subsSliceK; - for (unsigned i = k; i < subs.size(); i++) subsSliceK.push_back(subs[i]); - o += indentLines(joinLines(subsSliceK)); - o += "\n)"; - } - else { - o += out.substr(0, out.size() - 1) + ")"; - } - return o; -} - -// Splits text by line -std::vector<std::string> splitLines(std::string s) { - unsigned pos = 0; - int lastNewline = 0; - std::vector<std::string> o; - while (pos < s.length()) { - if (s[pos] == '\n') { - o.push_back(s.substr(lastNewline, pos - lastNewline)); - lastNewline = pos + 1; - } - pos = pos + 1; - } - o.push_back(s.substr(lastNewline)); - return o; -} - -// Inverse of splitLines -std::string joinLines(std::vector<std::string> lines) { - std::string o = "\n"; - for (unsigned i = 0; i < lines.size(); i++) { - o += lines[i] + "\n"; - } - return o.substr(1, o.length() - 2); -} - -// Indent all lines by 4 spaces -std::string indentLines(std::string inp) { - std::vector<std::string> lines = splitLines(inp); - for (unsigned i = 0; i < lines.size(); i++) lines[i] = " "+lines[i]; - return joinLines(lines); -} - -// Binary to hexadecimal -std::string binToNumeric(std::string inp) { - std::string o = "0"; - for (unsigned i = 0; i < inp.length(); i++) { - o = decimalAdd(decimalMul(o,"256"), unsignedToDecimal((unsigned char)inp[i])); - } - return o; -} - -// Converts string to simple numeric format -std::string strToNumeric(std::string inp) { - std::string o = "0"; - if (inp == "") { - o = ""; - } - else if (inp.substr(0,2) == "0x") { - for (unsigned i = 2; i < inp.length(); i++) { - int dig = std::string("0123456789abcdef0123456789ABCDEF").find(inp[i]) % 16; - if (dig < 0) return ""; - o = decimalAdd(decimalMul(o,"16"), unsignedToDecimal(dig)); - } - } - else { - bool isPureNum = true; - for (unsigned i = 0; i < inp.length(); i++) { - isPureNum = isPureNum && inp[i] >= '0' && inp[i] <= '9'; - } - o = isPureNum ? inp : ""; - } - return o; -} - -// Does the node contain a number (eg. 124, 0xf012c, "george") -bool isNumberLike(Node node) { - if (node.type == ASTNODE) return false; - return strToNumeric(node.val) != ""; -} - -//Normalizes number representations -Node nodeToNumeric(Node node) { - std::string o = strToNumeric(node.val); - return token(o == "" ? node.val : o, node.metadata); -} - -Node tryNumberize(Node node) { - if (node.type == TOKEN && isNumberLike(node)) return nodeToNumeric(node); - return node; -} - -//Converts a value to an array of byte number nodes -std::vector<Node> toByteArr(std::string val, Metadata metadata, int minLen) { - std::vector<Node> o; - int L = 0; - while (val != "0" || L < minLen) { - o.push_back(token(decimalMod(val, "256"), metadata)); - val = decimalDiv(val, "256"); - L++; - } - std::vector<Node> o2; - for (int i = o.size() - 1; i >= 0; i--) o2.push_back(o[i]); - return o2; -} - -int counter = 0; - -//Makes a unique token -std::string mkUniqueToken() { - counter++; - return unsignedToDecimal(counter); -} - -//Does a file exist? http://stackoverflow.com/questions/12774207 -bool exists(std::string fileName) { - std::ifstream infile(fileName.c_str()); - return infile.good(); -} - -//Reads a file: http://stackoverflow.com/questions/2602013 -std::string get_file_contents(std::string filename) -{ - std::ifstream in(filename.c_str(), std::ios::in | std::ios::binary); - if (in) - { - std::string contents; - in.seekg(0, std::ios::end); - contents.resize(in.tellg()); - in.seekg(0, std::ios::beg); - in.read(&contents[0], contents.size()); - in.close(); - return(contents); - } - throw(errno); -} - -//Report error -void err(std::string errtext, Metadata met) { - std::string err = "Error (file \"" + met.file + "\", line " + - unsignedToDecimal(met.ln + 1) + ", char " + unsignedToDecimal(met.ch) + - "): " + errtext; - std::cerr << err << "\n"; - throw(err); -} - -//Bin to hex -std::string binToHex(std::string inp) { - std::string o = ""; - for (unsigned i = 0; i < inp.length(); i++) { - unsigned char v = inp[i]; - o += std::string("0123456789abcdef").substr(v/16, 1) - + std::string("0123456789abcdef").substr(v%16, 1); - } - return o; -} - -//Hex to bin -std::string hexToBin(std::string inp) { - std::string o = ""; - for (unsigned i = 0; i+1 < inp.length(); i+=2) { - char v = (char)(std::string("0123456789abcdef").find(inp[i]) * 16 + - std::string("0123456789abcdef").find(inp[i+1])); - o += v; - } - return o; -} - -//Lower to upper -std::string upperCase(std::string inp) { - std::string o = ""; - for (unsigned i = 0; i < inp.length(); i++) { - if (inp[i] >= 97 && inp[i] <= 122) o += inp[i] - 32; - else o += inp[i]; - } - return o; -} - -//Three-int vector -std::vector<int> triple(int a, int b, int c) { - std::vector<int> v; - v.push_back(a); - v.push_back(b); - v.push_back(c); - return v; -} diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/util.h b/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/util.h deleted file mode 100644 index f7d6744f9..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/serpent/util.h +++ /dev/null @@ -1,127 +0,0 @@ -#ifndef ETHSERP_UTIL -#define ETHSERP_UTIL - -#include <stdio.h> -#include <iostream> -#include <vector> -#include <map> -#include <fstream> -#include <cerrno> - -const int TOKEN = 0, - ASTNODE = 1, - SPACE = 2, - BRACK = 3, - SQUOTE = 4, - DQUOTE = 5, - SYMB = 6, - ALPHANUM = 7, - LPAREN = 8, - RPAREN = 9, - COMMA = 10, - COLON = 11, - UNARY_OP = 12, - BINARY_OP = 13, - COMPOUND = 14, - TOKEN_SPLITTER = 15; - -// Stores metadata about each token -class Metadata { - public: - Metadata(std::string File="main", int Ln=-1, int Ch=-1) { - file = File; - ln = Ln; - ch = Ch; - fixed = false; - } - std::string file; - int ln; - int ch; - bool fixed; -}; - -std::string mkUniqueToken(); - -// type can be TOKEN or ASTNODE -struct Node { - int type; - std::string val; - std::vector<Node> args; - Metadata metadata; -}; -Node token(std::string val, Metadata met=Metadata()); -Node astnode(std::string val, std::vector<Node> args, Metadata met=Metadata()); -Node astnode(std::string val, Metadata met=Metadata()); -Node astnode(std::string val, Node a, Metadata met=Metadata()); -Node astnode(std::string val, Node a, Node b, Metadata met=Metadata()); -Node astnode(std::string val, Node a, Node b, Node c, Metadata met=Metadata()); -Node astnode(std::string val, Node a, Node b, - Node c, Node d, Metadata met=Metadata()); - -// Number of tokens in a tree -int treeSize(Node prog); - -// Print token list -std::string printTokens(std::vector<Node> tokens); - -// Prints a lisp AST on one line -std::string printSimple(Node ast); - -// Pretty-prints a lisp AST -std::string printAST(Node ast, bool printMetadata=false); - -// Splits text by line -std::vector<std::string> splitLines(std::string s); - -// Inverse of splitLines -std::string joinLines(std::vector<std::string> lines); - -// Indent all lines by 4 spaces -std::string indentLines(std::string inp); - -// Converts binary to simple numeric format -std::string binToNumeric(std::string inp); - -// Converts string to simple numeric format -std::string strToNumeric(std::string inp); - -// Does the node contain a number (eg. 124, 0xf012c, "george") -bool isNumberLike(Node node); - -//Normalizes number representations -Node nodeToNumeric(Node node); - -//If a node is numeric, normalize its representation -Node tryNumberize(Node node); - -//Converts a value to an array of byte number nodes -std::vector<Node> toByteArr(std::string val, Metadata metadata, int minLen=1); - -//Reads a file -std::string get_file_contents(std::string filename); - -//Does a file exist? -bool exists(std::string fileName); - -//Report error -void err(std::string errtext, Metadata met); - -//Bin to hex -std::string binToHex(std::string inp); - -//Hex to bin -std::string hexToBin(std::string inp); - -//Lower to upper -std::string upperCase(std::string inp); - -//Three-int vector -std::vector<int> triple(int a, int b, int c); - -#define asn astnode -#define tkn token -#define msi std::map<std::string, int> -#define msn std::map<std::string, Node> -#define mss std::map<std::string, std::string> - -#endif diff --git a/Godeps/_workspace/src/github.com/ethereum/serpent-go/tests/main.go b/Godeps/_workspace/src/github.com/ethereum/serpent-go/tests/main.go deleted file mode 100644 index 2f2d17784..000000000 --- a/Godeps/_workspace/src/github.com/ethereum/serpent-go/tests/main.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/ethereum/serpent-go" -) - -func main() { - out, _ := serpent.Compile(` -// Namecoin -if !contract.storage[msg.data[0]]: # Is the key not yet taken? - # Then take it! - contract.storage[msg.data[0]] = msg.data[1] - return(1) -else: - return(0) // Otherwise do nothing - `) - - fmt.Printf("%x\n", out) -} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch.go index 0d7911eca..ccf390c9c 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch.go @@ -8,65 +8,84 @@ package leveldb import ( "encoding/binary" - "errors" + "fmt" + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/memdb" ) -var ( - errBatchTooShort = errors.New("leveldb: batch is too short") - errBatchBadRecord = errors.New("leveldb: bad record in batch") -) +type ErrBatchCorrupted struct { + Reason string +} + +func (e *ErrBatchCorrupted) Error() string { + return fmt.Sprintf("leveldb: batch corrupted: %s", e.Reason) +} + +func newErrBatchCorrupted(reason string) error { + return errors.NewErrCorrupted(nil, &ErrBatchCorrupted{reason}) +} -const kBatchHdrLen = 8 + 4 +const ( + batchHdrLen = 8 + 4 + batchGrowRec = 3000 +) -type batchReplay interface { - put(key, value []byte, seq uint64) - delete(key []byte, seq uint64) +type BatchReplay interface { + Put(key, value []byte) + Delete(key []byte) } // Batch is a write batch. type Batch struct { - buf []byte + data []byte rLen, bLen int seq uint64 sync bool } func (b *Batch) grow(n int) { - off := len(b.buf) + off := len(b.data) if off == 0 { - // include headers - off = kBatchHdrLen - n += off + off = batchHdrLen + if b.data != nil { + b.data = b.data[:off] + } } - if cap(b.buf)-off >= n { - return + if cap(b.data)-off < n { + if b.data == nil { + b.data = make([]byte, off, off+n) + } else { + odata := b.data + div := 1 + if b.rLen > batchGrowRec { + div = b.rLen / batchGrowRec + } + b.data = make([]byte, off, off+n+(off-batchHdrLen)/div) + copy(b.data, odata) + } } - buf := make([]byte, 2*cap(b.buf)+n) - copy(buf, b.buf) - b.buf = buf[:off] } -func (b *Batch) appendRec(t vType, key, value []byte) { +func (b *Batch) appendRec(kt kType, key, value []byte) { n := 1 + binary.MaxVarintLen32 + len(key) - if t == tVal { + if kt == ktVal { n += binary.MaxVarintLen32 + len(value) } b.grow(n) - off := len(b.buf) - buf := b.buf[:off+n] - buf[off] = byte(t) + off := len(b.data) + data := b.data[:off+n] + data[off] = byte(kt) off += 1 - off += binary.PutUvarint(buf[off:], uint64(len(key))) - copy(buf[off:], key) + off += binary.PutUvarint(data[off:], uint64(len(key))) + copy(data[off:], key) off += len(key) - if t == tVal { - off += binary.PutUvarint(buf[off:], uint64(len(value))) - copy(buf[off:], value) + if kt == ktVal { + off += binary.PutUvarint(data[off:], uint64(len(value))) + copy(data[off:], value) off += len(value) } - b.buf = buf[:off] + b.data = data[:off] b.rLen++ // Include 8-byte ikey header b.bLen += len(key) + len(value) + 8 @@ -75,18 +94,51 @@ func (b *Batch) appendRec(t vType, key, value []byte) { // Put appends 'put operation' of the given key/value pair to the batch. // It is safe to modify the contents of the argument after Put returns. func (b *Batch) Put(key, value []byte) { - b.appendRec(tVal, key, value) + b.appendRec(ktVal, key, value) } // Delete appends 'delete operation' of the given key to the batch. // It is safe to modify the contents of the argument after Delete returns. func (b *Batch) Delete(key []byte) { - b.appendRec(tDel, key, nil) + b.appendRec(ktDel, key, nil) +} + +// Dump dumps batch contents. The returned slice can be loaded into the +// batch using Load method. +// The returned slice is not its own copy, so the contents should not be +// modified. +func (b *Batch) Dump() []byte { + return b.encode() +} + +// Load loads given slice into the batch. Previous contents of the batch +// will be discarded. +// The given slice will not be copied and will be used as batch buffer, so +// it is not safe to modify the contents of the slice. +func (b *Batch) Load(data []byte) error { + return b.decode(0, data) +} + +// Replay replays batch contents. +func (b *Batch) Replay(r BatchReplay) error { + return b.decodeRec(func(i int, kt kType, key, value []byte) { + switch kt { + case ktVal: + r.Put(key, value) + case ktDel: + r.Delete(key) + } + }) +} + +// Len returns number of records in the batch. +func (b *Batch) Len() int { + return b.rLen } // Reset resets the batch. func (b *Batch) Reset() { - b.buf = nil + b.data = b.data[:0] b.seq = 0 b.rLen = 0 b.bLen = 0 @@ -97,24 +149,10 @@ func (b *Batch) init(sync bool) { b.sync = sync } -func (b *Batch) put(key, value []byte, seq uint64) { - if b.rLen == 0 { - b.seq = seq - } - b.Put(key, value) -} - -func (b *Batch) delete(key []byte, seq uint64) { - if b.rLen == 0 { - b.seq = seq - } - b.Delete(key) -} - func (b *Batch) append(p *Batch) { if p.rLen > 0 { - b.grow(len(p.buf) - kBatchHdrLen) - b.buf = append(b.buf, p.buf[kBatchHdrLen:]...) + b.grow(len(p.data) - batchHdrLen) + b.data = append(b.data, p.data[batchHdrLen:]...) b.rLen += p.rLen } if p.sync { @@ -122,95 +160,93 @@ func (b *Batch) append(p *Batch) { } } -func (b *Batch) len() int { - return b.rLen -} - +// size returns sums of key/value pair length plus 8-bytes ikey. func (b *Batch) size() int { return b.bLen } func (b *Batch) encode() []byte { b.grow(0) - binary.LittleEndian.PutUint64(b.buf, b.seq) - binary.LittleEndian.PutUint32(b.buf[8:], uint32(b.rLen)) + binary.LittleEndian.PutUint64(b.data, b.seq) + binary.LittleEndian.PutUint32(b.data[8:], uint32(b.rLen)) - return b.buf + return b.data } -func (b *Batch) decode(buf []byte) error { - if len(buf) < kBatchHdrLen { - return errBatchTooShort +func (b *Batch) decode(prevSeq uint64, data []byte) error { + if len(data) < batchHdrLen { + return newErrBatchCorrupted("too short") } - b.seq = binary.LittleEndian.Uint64(buf) - b.rLen = int(binary.LittleEndian.Uint32(buf[8:])) + b.seq = binary.LittleEndian.Uint64(data) + if b.seq < prevSeq { + return newErrBatchCorrupted("invalid sequence number") + } + b.rLen = int(binary.LittleEndian.Uint32(data[8:])) + if b.rLen < 0 { + return newErrBatchCorrupted("invalid records length") + } // No need to be precise at this point, it won't be used anyway - b.bLen = len(buf) - kBatchHdrLen - b.buf = buf + b.bLen = len(data) - batchHdrLen + b.data = data return nil } -func (b *Batch) decodeRec(f func(i int, t vType, key, value []byte)) error { - off := kBatchHdrLen +func (b *Batch) decodeRec(f func(i int, kt kType, key, value []byte)) (err error) { + off := batchHdrLen for i := 0; i < b.rLen; i++ { - if off >= len(b.buf) { - return errors.New("leveldb: invalid batch record length") + if off >= len(b.data) { + return newErrBatchCorrupted("invalid records length") } - t := vType(b.buf[off]) - if t > tVal { - return errors.New("leveldb: invalid batch record type in batch") + kt := kType(b.data[off]) + if kt > ktVal { + return newErrBatchCorrupted("bad record: invalid type") } off += 1 - x, n := binary.Uvarint(b.buf[off:]) + x, n := binary.Uvarint(b.data[off:]) off += n - if n <= 0 || off+int(x) > len(b.buf) { - return errBatchBadRecord + if n <= 0 || off+int(x) > len(b.data) { + return newErrBatchCorrupted("bad record: invalid key length") } - key := b.buf[off : off+int(x)] + key := b.data[off : off+int(x)] off += int(x) - var value []byte - if t == tVal { - x, n := binary.Uvarint(b.buf[off:]) + if kt == ktVal { + x, n := binary.Uvarint(b.data[off:]) off += n - if n <= 0 || off+int(x) > len(b.buf) { - return errBatchBadRecord + if n <= 0 || off+int(x) > len(b.data) { + return newErrBatchCorrupted("bad record: invalid value length") } - value = b.buf[off : off+int(x)] + value = b.data[off : off+int(x)] off += int(x) } - f(i, t, key, value) + f(i, kt, key, value) } return nil } -func (b *Batch) replay(to batchReplay) error { - return b.decodeRec(func(i int, t vType, key, value []byte) { - switch t { - case tVal: - to.put(key, value, b.seq+uint64(i)) - case tDel: - to.delete(key, b.seq+uint64(i)) - } - }) -} - func (b *Batch) memReplay(to *memdb.DB) error { - return b.decodeRec(func(i int, t vType, key, value []byte) { - ikey := newIKey(key, b.seq+uint64(i), t) + return b.decodeRec(func(i int, kt kType, key, value []byte) { + ikey := newIkey(key, b.seq+uint64(i), kt) to.Put(ikey, value) }) } +func (b *Batch) memDecodeAndReplay(prevSeq uint64, data []byte, to *memdb.DB) error { + if err := b.decode(prevSeq, data); err != nil { + return err + } + return b.memReplay(to) +} + func (b *Batch) revertMemReplay(to *memdb.DB) error { - return b.decodeRec(func(i int, t vType, key, value []byte) { - ikey := newIKey(key, b.seq+uint64(i), t) + return b.decodeRec(func(i int, kt kType, key, value []byte) { + ikey := newIkey(key, b.seq+uint64(i), kt) to.Delete(ikey) }) } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch_test.go index 19b749b8f..7fc842f4f 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch_test.go @@ -15,7 +15,7 @@ import ( ) type tbRec struct { - t vType + kt kType key, value []byte } @@ -23,39 +23,39 @@ type testBatch struct { rec []*tbRec } -func (p *testBatch) put(key, value []byte, seq uint64) { - p.rec = append(p.rec, &tbRec{tVal, key, value}) +func (p *testBatch) Put(key, value []byte) { + p.rec = append(p.rec, &tbRec{ktVal, key, value}) } -func (p *testBatch) delete(key []byte, seq uint64) { - p.rec = append(p.rec, &tbRec{tDel, key, nil}) +func (p *testBatch) Delete(key []byte) { + p.rec = append(p.rec, &tbRec{ktDel, key, nil}) } func compareBatch(t *testing.T, b1, b2 *Batch) { if b1.seq != b2.seq { t.Errorf("invalid seq number want %d, got %d", b1.seq, b2.seq) } - if b1.len() != b2.len() { - t.Fatalf("invalid record length want %d, got %d", b1.len(), b2.len()) + if b1.Len() != b2.Len() { + t.Fatalf("invalid record length want %d, got %d", b1.Len(), b2.Len()) } p1, p2 := new(testBatch), new(testBatch) - err := b1.replay(p1) + err := b1.Replay(p1) if err != nil { t.Fatal("error when replaying batch 1: ", err) } - err = b2.replay(p2) + err = b2.Replay(p2) if err != nil { t.Fatal("error when replaying batch 2: ", err) } for i := range p1.rec { r1, r2 := p1.rec[i], p2.rec[i] - if r1.t != r2.t { - t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.t, r2.t) + if r1.kt != r2.kt { + t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.kt, r2.kt) } if !bytes.Equal(r1.key, r2.key) { t.Errorf("invalid key on record '%d' want %s, got %s", i, string(r1.key), string(r2.key)) } - if r1.t == tVal { + if r1.kt == ktVal { if !bytes.Equal(r1.value, r2.value) { t.Errorf("invalid value on record '%d' want %s, got %s", i, string(r1.value), string(r2.value)) } @@ -75,7 +75,7 @@ func TestBatch_EncodeDecode(t *testing.T) { b1.Delete([]byte("k")) buf := b1.encode() b2 := new(Batch) - err := b2.decode(buf) + err := b2.decode(0, buf) if err != nil { t.Error("error when decoding batch: ", err) } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench2_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench2_test.go new file mode 100644 index 000000000..0dd60fd82 --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench2_test.go @@ -0,0 +1,58 @@ +// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com> +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// +build !go1.2 + +package leveldb + +import ( + "sync/atomic" + "testing" +) + +func BenchmarkDBReadConcurrent(b *testing.B) { + p := openDBBench(b, false) + p.populate(b.N) + p.fill() + p.gc() + defer p.close() + + b.ResetTimer() + b.SetBytes(116) + + b.RunParallel(func(pb *testing.PB) { + iter := p.newIter() + defer iter.Release() + for pb.Next() && iter.Next() { + } + }) +} + +func BenchmarkDBReadConcurrent2(b *testing.B) { + p := openDBBench(b, false) + p.populate(b.N) + p.fill() + p.gc() + defer p.close() + + b.ResetTimer() + b.SetBytes(116) + + var dir uint32 + b.RunParallel(func(pb *testing.PB) { + iter := p.newIter() + defer iter.Release() + if atomic.AddUint32(&dir, 1)%2 == 0 { + for pb.Next() && iter.Next() { + } + } else { + if pb.Next() && iter.Last() { + for pb.Next() && iter.Prev() { + } + } + } + }) +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench_test.go index ea6801a89..91b426709 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench_test.go @@ -170,7 +170,7 @@ func (p *dbBench) writes(perBatch int) { b.SetBytes(116) } -func (p *dbBench) drop() { +func (p *dbBench) gc() { p.keys, p.values = nil, nil runtime.GC() } @@ -249,6 +249,9 @@ func (p *dbBench) newIter() iterator.Iterator { } func (p *dbBench) close() { + if bp, err := p.db.GetProperty("leveldb.blockpool"); err == nil { + p.b.Log("Block pool stats: ", bp) + } p.db.Close() p.stor.Close() os.RemoveAll(benchDB) @@ -331,7 +334,7 @@ func BenchmarkDBRead(b *testing.B) { p := openDBBench(b, false) p.populate(b.N) p.fill() - p.drop() + p.gc() iter := p.newIter() b.ResetTimer() @@ -362,7 +365,7 @@ func BenchmarkDBReadUncompressed(b *testing.B) { p := openDBBench(b, true) p.populate(b.N) p.fill() - p.drop() + p.gc() iter := p.newIter() b.ResetTimer() @@ -379,7 +382,7 @@ func BenchmarkDBReadTable(b *testing.B) { p.populate(b.N) p.fill() p.reopen() - p.drop() + p.gc() iter := p.newIter() b.ResetTimer() @@ -395,7 +398,7 @@ func BenchmarkDBReadReverse(b *testing.B) { p := openDBBench(b, false) p.populate(b.N) p.fill() - p.drop() + p.gc() iter := p.newIter() b.ResetTimer() @@ -413,7 +416,7 @@ func BenchmarkDBReadReverseTable(b *testing.B) { p.populate(b.N) p.fill() p.reopen() - p.drop() + p.gc() iter := p.newIter() b.ResetTimer() diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/bench2_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/bench2_test.go new file mode 100644 index 000000000..175e22203 --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/bench2_test.go @@ -0,0 +1,30 @@ +// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com> +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// +build !go1.2 + +package cache + +import ( + "math/rand" + "testing" +) + +func BenchmarkLRUCache(b *testing.B) { + c := NewCache(NewLRU(10000)) + + b.SetParallelism(10) + b.RunParallel(func(pb *testing.PB) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + for pb.Next() { + key := uint64(r.Intn(1000000)) + c.Get(0, key, func() (int, Value) { + return 1, key + }).Release() + } + }) +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go index 9b6a74977..c9670de5d 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go @@ -8,118 +8,669 @@ package cache import ( + "sync" "sync/atomic" + "unsafe" + + "github.com/syndtr/goleveldb/leveldb/util" ) -// SetFunc used by Namespace.Get method to create a cache object. SetFunc -// may return ok false, in that case the cache object will not be created. -type SetFunc func() (ok bool, value interface{}, charge int, fin SetFin) +// Cacher provides interface to implements a caching functionality. +// An implementation must be goroutine-safe. +type Cacher interface { + // Capacity returns cache capacity. + Capacity() int -// SetFin will be called when corresponding cache object are released. -type SetFin func() + // SetCapacity sets cache capacity. + SetCapacity(capacity int) -// DelFin will be called when corresponding cache object are released. -// DelFin will be called after SetFin. The exist is true if the corresponding -// cache object is actually exist in the cache tree. -type DelFin func(exist bool) + // Promote promotes the 'cache node'. + Promote(n *Node) -// PurgeFin will be called when corresponding cache object are released. -// PurgeFin will be called after SetFin. If PurgeFin present DelFin will -// not be executed but passed to the PurgeFin, it is up to the caller -// to call it or not. -type PurgeFin func(ns, key uint64, delfin DelFin) + // Ban evicts the 'cache node' and prevent subsequent 'promote'. + Ban(n *Node) -// Cache is a cache tree. -type Cache interface { - // SetCapacity sets cache capacity. - SetCapacity(capacity int) + // Evict evicts the 'cache node'. + Evict(n *Node) - // GetNamespace gets or creates a cache namespace for the given id. - GetNamespace(id uint64) Namespace + // EvictNS evicts 'cache node' with the given namespace. + EvictNS(ns uint64) - // Purge purges all cache namespaces, read Namespace.Purge method documentation. - Purge(fin PurgeFin) + // EvictAll evicts all 'cache node'. + EvictAll() + + // Close closes the 'cache tree' + Close() error +} + +// Value is a 'cacheable object'. It may implements util.Releaser, if +// so the the Release method will be called once object is released. +type Value interface{} + +type CacheGetter struct { + Cache *Cache + NS uint64 +} - // Zap zaps all cache namespaces, read Namespace.Zap method documentation. - Zap(closed bool) +func (g *CacheGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle { + return g.Cache.Get(g.NS, key, setFunc) } -// Namespace is a cache namespace. -type Namespace interface { - // Get gets cache object for the given key. The given SetFunc (if not nil) will - // be called if the given key does not exist. - // If the given key does not exist, SetFunc is nil or SetFunc return ok false, Get - // will return ok false. - Get(key uint64, setf SetFunc) (obj Object, ok bool) +// The hash tables implementation is based on: +// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu, Kunlong Zhang, and Michael Spear. ACM Symposium on Principles of Distributed Computing, Jul 2014. - // Get deletes cache object for the given key. If exist the cache object will - // be deleted later when all of its handles have been released (i.e. no one use - // it anymore) and the given DelFin (if not nil) will finally be executed. If - // such cache object does not exist the given DelFin will be executed anyway. - // - // Delete returns true if such cache object exist. - Delete(key uint64, fin DelFin) bool +const ( + mInitialSize = 1 << 4 + mOverflowThreshold = 1 << 5 + mOverflowGrowThreshold = 1 << 7 +) - // Purge deletes all cache objects, read Delete method documentation. - Purge(fin PurgeFin) +type mBucket struct { + mu sync.Mutex + node []*Node + frozen bool +} - // Zap detaches the namespace from the cache tree and delete all its cache - // objects. The cache objects deletion and finalizers execution are happen - // immediately, even if its existing handles haven't yet been released. - // A zapped namespace can't never be filled again. - // If closed is false then the Get function will always call the given SetFunc - // if it is not nil, but resultant of the SetFunc will not be cached. - Zap(closed bool) +func (b *mBucket) freeze() []*Node { + b.mu.Lock() + defer b.mu.Unlock() + if !b.frozen { + b.frozen = true + } + return b.node } -// Object is a cache object. -type Object interface { - // Release releases the cache object. Other methods should not be called - // after the cache object has been released. - Release() +func (b *mBucket) get(r *Cache, h *mNode, hash uint32, ns, key uint64, noset bool) (done, added bool, n *Node) { + b.mu.Lock() + + if b.frozen { + b.mu.Unlock() + return + } + + // Scan the node. + for _, n := range b.node { + if n.hash == hash && n.ns == ns && n.key == key { + atomic.AddInt32(&n.ref, 1) + b.mu.Unlock() + return true, false, n + } + } + + // Get only. + if noset { + b.mu.Unlock() + return true, false, nil + } + + // Create node. + n = &Node{ + r: r, + hash: hash, + ns: ns, + key: key, + ref: 1, + } + // Add node to bucket. + b.node = append(b.node, n) + bLen := len(b.node) + b.mu.Unlock() + + // Update counter. + grow := atomic.AddInt32(&r.nodes, 1) >= h.growThreshold + if bLen > mOverflowThreshold { + grow = grow || atomic.AddInt32(&h.overflow, 1) >= mOverflowGrowThreshold + } - // Value returns value of the cache object. - Value() interface{} + // Grow. + if grow && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) { + nhLen := len(h.buckets) << 1 + nh := &mNode{ + buckets: make([]unsafe.Pointer, nhLen), + mask: uint32(nhLen) - 1, + pred: unsafe.Pointer(h), + growThreshold: int32(nhLen * mOverflowThreshold), + shrinkThreshold: int32(nhLen >> 1), + } + ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh)) + if !ok { + panic("BUG: failed swapping head") + } + go nh.initBuckets() + } + + return true, true, n } -// Namespace state. -type nsState int +func (b *mBucket) delete(r *Cache, h *mNode, hash uint32, ns, key uint64) (done, deleted bool) { + b.mu.Lock() -const ( - nsEffective nsState = iota - nsZapped - nsClosed -) + if b.frozen { + b.mu.Unlock() + return + } -// Node state. -type nodeState int + // Scan the node. + var ( + n *Node + bLen int + ) + for i := range b.node { + n = b.node[i] + if n.ns == ns && n.key == key { + if atomic.LoadInt32(&n.ref) == 0 { + deleted = true -const ( - nodeEffective nodeState = iota - nodeEvicted - nodeRemoved -) + // Call releaser. + if n.value != nil { + if r, ok := n.value.(util.Releaser); ok { + r.Release() + } + n.value = nil + } + + // Remove node from bucket. + b.node = append(b.node[:i], b.node[i+1:]...) + bLen = len(b.node) + } + break + } + } + b.mu.Unlock() -// Fake object. -type fakeObject struct { - value interface{} - fin func() - once uint32 + if deleted { + // Call OnDel. + for _, f := range n.onDel { + f() + } + + // Update counter. + atomic.AddInt32(&r.size, int32(n.size)*-1) + shrink := atomic.AddInt32(&r.nodes, -1) < h.shrinkThreshold + if bLen >= mOverflowThreshold { + atomic.AddInt32(&h.overflow, -1) + } + + // Shrink. + if shrink && len(h.buckets) > mInitialSize && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) { + nhLen := len(h.buckets) >> 1 + nh := &mNode{ + buckets: make([]unsafe.Pointer, nhLen), + mask: uint32(nhLen) - 1, + pred: unsafe.Pointer(h), + growThreshold: int32(nhLen * mOverflowThreshold), + shrinkThreshold: int32(nhLen >> 1), + } + ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh)) + if !ok { + panic("BUG: failed swapping head") + } + go nh.initBuckets() + } + } + + return true, deleted } -func (o *fakeObject) Value() interface{} { - if atomic.LoadUint32(&o.once) == 0 { - return o.value +type mNode struct { + buckets []unsafe.Pointer // []*mBucket + mask uint32 + pred unsafe.Pointer // *mNode + resizeInProgess int32 + + overflow int32 + growThreshold int32 + shrinkThreshold int32 +} + +func (n *mNode) initBucket(i uint32) *mBucket { + if b := (*mBucket)(atomic.LoadPointer(&n.buckets[i])); b != nil { + return b + } + + p := (*mNode)(atomic.LoadPointer(&n.pred)) + if p != nil { + var node []*Node + if n.mask > p.mask { + // Grow. + pb := (*mBucket)(atomic.LoadPointer(&p.buckets[i&p.mask])) + if pb == nil { + pb = p.initBucket(i & p.mask) + } + m := pb.freeze() + // Split nodes. + for _, x := range m { + if x.hash&n.mask == i { + node = append(node, x) + } + } + } else { + // Shrink. + pb0 := (*mBucket)(atomic.LoadPointer(&p.buckets[i])) + if pb0 == nil { + pb0 = p.initBucket(i) + } + pb1 := (*mBucket)(atomic.LoadPointer(&p.buckets[i+uint32(len(n.buckets))])) + if pb1 == nil { + pb1 = p.initBucket(i + uint32(len(n.buckets))) + } + m0 := pb0.freeze() + m1 := pb1.freeze() + // Merge nodes. + node = make([]*Node, 0, len(m0)+len(m1)) + node = append(node, m0...) + node = append(node, m1...) + } + b := &mBucket{node: node} + if atomic.CompareAndSwapPointer(&n.buckets[i], nil, unsafe.Pointer(b)) { + if len(node) > mOverflowThreshold { + atomic.AddInt32(&n.overflow, int32(len(node)-mOverflowThreshold)) + } + return b + } + } + + return (*mBucket)(atomic.LoadPointer(&n.buckets[i])) +} + +func (n *mNode) initBuckets() { + for i := range n.buckets { + n.initBucket(uint32(i)) + } + atomic.StorePointer(&n.pred, nil) +} + +// Cache is a 'cache map'. +type Cache struct { + mu sync.RWMutex + mHead unsafe.Pointer // *mNode + nodes int32 + size int32 + cacher Cacher + closed bool +} + +// NewCache creates a new 'cache map'. The cacher is optional and +// may be nil. +func NewCache(cacher Cacher) *Cache { + h := &mNode{ + buckets: make([]unsafe.Pointer, mInitialSize), + mask: mInitialSize - 1, + growThreshold: int32(mInitialSize * mOverflowThreshold), + shrinkThreshold: 0, + } + for i := range h.buckets { + h.buckets[i] = unsafe.Pointer(&mBucket{}) + } + r := &Cache{ + mHead: unsafe.Pointer(h), + cacher: cacher, + } + return r +} + +func (r *Cache) getBucket(hash uint32) (*mNode, *mBucket) { + h := (*mNode)(atomic.LoadPointer(&r.mHead)) + i := hash & h.mask + b := (*mBucket)(atomic.LoadPointer(&h.buckets[i])) + if b == nil { + b = h.initBucket(i) + } + return h, b +} + +func (r *Cache) delete(n *Node) bool { + for { + h, b := r.getBucket(n.hash) + done, deleted := b.delete(r, h, n.hash, n.ns, n.key) + if done { + return deleted + } + } + return false +} + +// Nodes returns number of 'cache node' in the map. +func (r *Cache) Nodes() int { + return int(atomic.LoadInt32(&r.nodes)) +} + +// Size returns sums of 'cache node' size in the map. +func (r *Cache) Size() int { + return int(atomic.LoadInt32(&r.size)) +} + +// Capacity returns cache capacity. +func (r *Cache) Capacity() int { + if r.cacher == nil { + return 0 + } + return r.cacher.Capacity() +} + +// SetCapacity sets cache capacity. +func (r *Cache) SetCapacity(capacity int) { + if r.cacher != nil { + r.cacher.SetCapacity(capacity) + } +} + +// Get gets 'cache node' with the given namespace and key. +// If cache node is not found and setFunc is not nil, Get will atomically creates +// the 'cache node' by calling setFunc. Otherwise Get will returns nil. +// +// The returned 'cache handle' should be released after use by calling Release +// method. +func (r *Cache) Get(ns, key uint64, setFunc func() (size int, value Value)) *Handle { + r.mu.RLock() + defer r.mu.RUnlock() + if r.closed { + return nil + } + + hash := murmur32(ns, key, 0xf00) + for { + h, b := r.getBucket(hash) + done, _, n := b.get(r, h, hash, ns, key, setFunc == nil) + if done { + if n != nil { + n.mu.Lock() + if n.value == nil { + if setFunc == nil { + n.mu.Unlock() + n.unref() + return nil + } + + n.size, n.value = setFunc() + if n.value == nil { + n.size = 0 + n.mu.Unlock() + n.unref() + return nil + } + atomic.AddInt32(&r.size, int32(n.size)) + } + n.mu.Unlock() + if r.cacher != nil { + r.cacher.Promote(n) + } + return &Handle{unsafe.Pointer(n)} + } + + break + } } return nil } -func (o *fakeObject) Release() { - if !atomic.CompareAndSwapUint32(&o.once, 0, 1) { +// Delete removes and ban 'cache node' with the given namespace and key. +// A banned 'cache node' will never inserted into the 'cache tree'. Ban +// only attributed to the particular 'cache node', so when a 'cache node' +// is recreated it will not be banned. +// +// If onDel is not nil, then it will be executed if such 'cache node' +// doesn't exist or once the 'cache node' is released. +// +// Delete return true is such 'cache node' exist. +func (r *Cache) Delete(ns, key uint64, onDel func()) bool { + r.mu.RLock() + defer r.mu.RUnlock() + if r.closed { + return false + } + + hash := murmur32(ns, key, 0xf00) + for { + h, b := r.getBucket(hash) + done, _, n := b.get(r, h, hash, ns, key, true) + if done { + if n != nil { + if onDel != nil { + n.mu.Lock() + n.onDel = append(n.onDel, onDel) + n.mu.Unlock() + } + if r.cacher != nil { + r.cacher.Ban(n) + } + n.unref() + return true + } + + break + } + } + + if onDel != nil { + onDel() + } + + return false +} + +// Evict evicts 'cache node' with the given namespace and key. This will +// simply call Cacher.Evict. +// +// Evict return true is such 'cache node' exist. +func (r *Cache) Evict(ns, key uint64) bool { + r.mu.RLock() + defer r.mu.RUnlock() + if r.closed { + return false + } + + hash := murmur32(ns, key, 0xf00) + for { + h, b := r.getBucket(hash) + done, _, n := b.get(r, h, hash, ns, key, true) + if done { + if n != nil { + if r.cacher != nil { + r.cacher.Evict(n) + } + n.unref() + return true + } + + break + } + } + + return false +} + +// EvictNS evicts 'cache node' with the given namespace. This will +// simply call Cacher.EvictNS. +func (r *Cache) EvictNS(ns uint64) { + r.mu.RLock() + defer r.mu.RUnlock() + if r.closed { + return + } + + if r.cacher != nil { + r.cacher.EvictNS(ns) + } +} + +// EvictAll evicts all 'cache node'. This will simply call Cacher.EvictAll. +func (r *Cache) EvictAll() { + r.mu.RLock() + defer r.mu.RUnlock() + if r.closed { return } - if o.fin != nil { - o.fin() - o.fin = nil + + if r.cacher != nil { + r.cacher.EvictAll() + } +} + +// Close closes the 'cache map' and releases all 'cache node'. +func (r *Cache) Close() error { + r.mu.Lock() + if !r.closed { + r.closed = true + + if r.cacher != nil { + if err := r.cacher.Close(); err != nil { + return err + } + } + + h := (*mNode)(r.mHead) + h.initBuckets() + + for i := range h.buckets { + b := (*mBucket)(h.buckets[i]) + for _, n := range b.node { + // Call releaser. + if n.value != nil { + if r, ok := n.value.(util.Releaser); ok { + r.Release() + } + n.value = nil + } + + // Call OnDel. + for _, f := range n.onDel { + f() + } + } + } } + r.mu.Unlock() + return nil +} + +// Node is a 'cache node'. +type Node struct { + r *Cache + + hash uint32 + ns, key uint64 + + mu sync.Mutex + size int + value Value + + ref int32 + onDel []func() + + CacheData unsafe.Pointer +} + +// NS returns this 'cache node' namespace. +func (n *Node) NS() uint64 { + return n.ns +} + +// Key returns this 'cache node' key. +func (n *Node) Key() uint64 { + return n.key +} + +// Size returns this 'cache node' size. +func (n *Node) Size() int { + return n.size +} + +// Value returns this 'cache node' value. +func (n *Node) Value() Value { + return n.value +} + +// Ref returns this 'cache node' ref counter. +func (n *Node) Ref() int32 { + return atomic.LoadInt32(&n.ref) +} + +// GetHandle returns an handle for this 'cache node'. +func (n *Node) GetHandle() *Handle { + if atomic.AddInt32(&n.ref, 1) <= 1 { + panic("BUG: Node.GetHandle on zero ref") + } + return &Handle{unsafe.Pointer(n)} +} + +func (n *Node) unref() { + if atomic.AddInt32(&n.ref, -1) == 0 { + n.r.delete(n) + } +} + +func (n *Node) unrefLocked() { + if atomic.AddInt32(&n.ref, -1) == 0 { + n.r.mu.RLock() + if !n.r.closed { + n.r.delete(n) + } + n.r.mu.RUnlock() + } +} + +type Handle struct { + n unsafe.Pointer // *Node +} + +func (h *Handle) Value() Value { + n := (*Node)(atomic.LoadPointer(&h.n)) + if n != nil { + return n.value + } + return nil +} + +func (h *Handle) Release() { + nPtr := atomic.LoadPointer(&h.n) + if nPtr != nil && atomic.CompareAndSwapPointer(&h.n, nPtr, nil) { + n := (*Node)(nPtr) + n.unrefLocked() + } +} + +func murmur32(ns, key uint64, seed uint32) uint32 { + const ( + m = uint32(0x5bd1e995) + r = 24 + ) + + k1 := uint32(ns >> 32) + k2 := uint32(ns) + k3 := uint32(key >> 32) + k4 := uint32(key) + + k1 *= m + k1 ^= k1 >> r + k1 *= m + + k2 *= m + k2 ^= k2 >> r + k2 *= m + + k3 *= m + k3 ^= k3 >> r + k3 *= m + + k4 *= m + k4 ^= k4 >> r + k4 *= m + + h := seed + + h *= m + h ^= k1 + h *= m + h ^= k2 + h *= m + h ^= k3 + h *= m + h ^= k4 + + h ^= h >> 13 + h *= m + h ^= h >> 15 + + return h } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go index 07a9939b2..c2a50156f 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go @@ -8,17 +8,289 @@ package cache import ( "math/rand" + "runtime" + "sync" + "sync/atomic" "testing" + "time" + "unsafe" ) -func set(ns Namespace, key uint64, value interface{}, charge int, fin func()) Object { - obj, _ := ns.Get(key, func() (bool, interface{}, int, SetFin) { - return true, value, charge, fin +type int32o int32 + +func (o *int32o) acquire() { + if atomic.AddInt32((*int32)(o), 1) != 1 { + panic("BUG: invalid ref") + } +} + +func (o *int32o) Release() { + if atomic.AddInt32((*int32)(o), -1) != 0 { + panic("BUG: invalid ref") + } +} + +type releaserFunc struct { + fn func() + value Value +} + +func (r releaserFunc) Release() { + if r.fn != nil { + r.fn() + } +} + +func set(c *Cache, ns, key uint64, value Value, charge int, relf func()) *Handle { + return c.Get(ns, key, func() (int, Value) { + if relf != nil { + return charge, releaserFunc{relf, value} + } else { + return charge, value + } + }) +} + +func TestCacheMap(t *testing.T) { + runtime.GOMAXPROCS(runtime.NumCPU()) + + nsx := []struct { + nobjects, nhandles, concurrent, repeat int + }{ + {10000, 400, 50, 3}, + {100000, 1000, 100, 10}, + } + + var ( + objects [][]int32o + handles [][]unsafe.Pointer + ) + + for _, x := range nsx { + objects = append(objects, make([]int32o, x.nobjects)) + handles = append(handles, make([]unsafe.Pointer, x.nhandles)) + } + + c := NewCache(nil) + + wg := new(sync.WaitGroup) + var done int32 + + for ns, x := range nsx { + for i := 0; i < x.concurrent; i++ { + wg.Add(1) + go func(ns, i, repeat int, objects []int32o, handles []unsafe.Pointer) { + defer wg.Done() + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + for j := len(objects) * repeat; j >= 0; j-- { + key := uint64(r.Intn(len(objects))) + h := c.Get(uint64(ns), key, func() (int, Value) { + o := &objects[key] + o.acquire() + return 1, o + }) + if v := h.Value().(*int32o); v != &objects[key] { + t.Fatalf("#%d invalid value: want=%p got=%p", ns, &objects[key], v) + } + if objects[key] != 1 { + t.Fatalf("#%d invalid object %d: %d", ns, key, objects[key]) + } + if !atomic.CompareAndSwapPointer(&handles[r.Intn(len(handles))], nil, unsafe.Pointer(h)) { + h.Release() + } + } + }(ns, i, x.repeat, objects[ns], handles[ns]) + } + + go func(handles []unsafe.Pointer) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + for atomic.LoadInt32(&done) == 0 { + i := r.Intn(len(handles)) + h := (*Handle)(atomic.LoadPointer(&handles[i])) + if h != nil && atomic.CompareAndSwapPointer(&handles[i], unsafe.Pointer(h), nil) { + h.Release() + } + time.Sleep(time.Millisecond) + } + }(handles[ns]) + } + + go func() { + handles := make([]*Handle, 100000) + for atomic.LoadInt32(&done) == 0 { + for i := range handles { + handles[i] = c.Get(999999999, uint64(i), func() (int, Value) { + return 1, 1 + }) + } + for _, h := range handles { + h.Release() + } + } + }() + + wg.Wait() + + atomic.StoreInt32(&done, 1) + + for _, handles0 := range handles { + for i := range handles0 { + h := (*Handle)(atomic.LoadPointer(&handles0[i])) + if h != nil && atomic.CompareAndSwapPointer(&handles0[i], unsafe.Pointer(h), nil) { + h.Release() + } + } + } + + for ns, objects0 := range objects { + for i, o := range objects0 { + if o != 0 { + t.Fatalf("invalid object #%d.%d: ref=%d", ns, i, o) + } + } + } +} + +func TestCacheMap_NodesAndSize(t *testing.T) { + c := NewCache(nil) + if c.Nodes() != 0 { + t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes()) + } + if c.Size() != 0 { + t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size()) + } + set(c, 0, 1, 1, 1, nil) + set(c, 0, 2, 2, 2, nil) + set(c, 1, 1, 3, 3, nil) + set(c, 2, 1, 4, 1, nil) + if c.Nodes() != 4 { + t.Errorf("invalid nodes counter: want=%d got=%d", 4, c.Nodes()) + } + if c.Size() != 7 { + t.Errorf("invalid size counter: want=%d got=%d", 4, c.Size()) + } +} + +func TestLRUCache_Capacity(t *testing.T) { + c := NewCache(NewLRU(10)) + if c.Capacity() != 10 { + t.Errorf("invalid capacity: want=%d got=%d", 10, c.Capacity()) + } + set(c, 0, 1, 1, 1, nil).Release() + set(c, 0, 2, 2, 2, nil).Release() + set(c, 1, 1, 3, 3, nil).Release() + set(c, 2, 1, 4, 1, nil).Release() + set(c, 2, 2, 5, 1, nil).Release() + set(c, 2, 3, 6, 1, nil).Release() + set(c, 2, 4, 7, 1, nil).Release() + set(c, 2, 5, 8, 1, nil).Release() + if c.Nodes() != 7 { + t.Errorf("invalid nodes counter: want=%d got=%d", 7, c.Nodes()) + } + if c.Size() != 10 { + t.Errorf("invalid size counter: want=%d got=%d", 10, c.Size()) + } + c.SetCapacity(9) + if c.Capacity() != 9 { + t.Errorf("invalid capacity: want=%d got=%d", 9, c.Capacity()) + } + if c.Nodes() != 6 { + t.Errorf("invalid nodes counter: want=%d got=%d", 6, c.Nodes()) + } + if c.Size() != 8 { + t.Errorf("invalid size counter: want=%d got=%d", 8, c.Size()) + } +} + +func TestCacheMap_NilValue(t *testing.T) { + c := NewCache(NewLRU(10)) + h := c.Get(0, 0, func() (size int, value Value) { + return 1, nil }) - return obj + if h != nil { + t.Error("cache handle is non-nil") + } + if c.Nodes() != 0 { + t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes()) + } + if c.Size() != 0 { + t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size()) + } } -func TestCache_HitMiss(t *testing.T) { +func TestLRUCache_GetLatency(t *testing.T) { + runtime.GOMAXPROCS(runtime.NumCPU()) + + const ( + concurrentSet = 30 + concurrentGet = 3 + duration = 3 * time.Second + delay = 3 * time.Millisecond + maxkey = 100000 + ) + + var ( + set, getHit, getAll int32 + getMaxLatency, getDuration int64 + ) + + c := NewCache(NewLRU(5000)) + wg := &sync.WaitGroup{} + until := time.Now().Add(duration) + for i := 0; i < concurrentSet; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for time.Now().Before(until) { + c.Get(0, uint64(r.Intn(maxkey)), func() (int, Value) { + time.Sleep(delay) + atomic.AddInt32(&set, 1) + return 1, 1 + }).Release() + } + }(i) + } + for i := 0; i < concurrentGet; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for { + mark := time.Now() + if mark.Before(until) { + h := c.Get(0, uint64(r.Intn(maxkey)), nil) + latency := int64(time.Now().Sub(mark)) + m := atomic.LoadInt64(&getMaxLatency) + if latency > m { + atomic.CompareAndSwapInt64(&getMaxLatency, m, latency) + } + atomic.AddInt64(&getDuration, latency) + if h != nil { + atomic.AddInt32(&getHit, 1) + h.Release() + } + atomic.AddInt32(&getAll, 1) + } else { + break + } + } + }(i) + } + + wg.Wait() + getAvglatency := time.Duration(getDuration) / time.Duration(getAll) + t.Logf("set=%d getHit=%d getAll=%d getMaxLatency=%v getAvgLatency=%v", + set, getHit, getAll, time.Duration(getMaxLatency), getAvglatency) + + if getAvglatency > delay/3 { + t.Errorf("get avg latency > %v: got=%v", delay/3, getAvglatency) + } +} + +func TestLRUCache_HitMiss(t *testing.T) { cases := []struct { key uint64 value string @@ -36,36 +308,37 @@ func TestCache_HitMiss(t *testing.T) { } setfin := 0 - c := NewLRUCache(1000) - ns := c.GetNamespace(0) + c := NewCache(NewLRU(1000)) for i, x := range cases { - set(ns, x.key, x.value, len(x.value), func() { + set(c, 0, x.key, x.value, len(x.value), func() { setfin++ }).Release() for j, y := range cases { - r, ok := ns.Get(y.key, nil) + h := c.Get(0, y.key, nil) if j <= i { // should hit - if !ok { + if h == nil { t.Errorf("case '%d' iteration '%d' is miss", i, j) - } else if r.Value().(string) != y.value { - t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value) + } else { + if x := h.Value().(releaserFunc).value.(string); x != y.value { + t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value) + } } } else { // should miss - if ok { - t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, r.Value().(string)) + if h != nil { + t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, h.Value().(releaserFunc).value.(string)) } } - if ok { - r.Release() + if h != nil { + h.Release() } } } for i, x := range cases { finalizerOk := false - ns.Delete(x.key, func(exist bool) { + c.Delete(0, x.key, func() { finalizerOk = true }) @@ -74,22 +347,24 @@ func TestCache_HitMiss(t *testing.T) { } for j, y := range cases { - r, ok := ns.Get(y.key, nil) + h := c.Get(0, y.key, nil) if j > i { // should hit - if !ok { + if h == nil { t.Errorf("case '%d' iteration '%d' is miss", i, j) - } else if r.Value().(string) != y.value { - t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value) + } else { + if x := h.Value().(releaserFunc).value.(string); x != y.value { + t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value) + } } } else { // should miss - if ok { - t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, r.Value().(string)) + if h != nil { + t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, h.Value().(releaserFunc).value.(string)) } } - if ok { - r.Release() + if h != nil { + h.Release() } } } @@ -100,137 +375,180 @@ func TestCache_HitMiss(t *testing.T) { } func TestLRUCache_Eviction(t *testing.T) { - c := NewLRUCache(12) - ns := c.GetNamespace(0) - o1 := set(ns, 1, 1, 1, nil) - set(ns, 2, 2, 1, nil).Release() - set(ns, 3, 3, 1, nil).Release() - set(ns, 4, 4, 1, nil).Release() - set(ns, 5, 5, 1, nil).Release() - if r, ok := ns.Get(2, nil); ok { // 1,3,4,5,2 - r.Release() - } - set(ns, 9, 9, 10, nil).Release() // 5,2,9 - - for _, x := range []uint64{9, 2, 5, 1} { - r, ok := ns.Get(x, nil) - if !ok { - t.Errorf("miss for key '%d'", x) + c := NewCache(NewLRU(12)) + o1 := set(c, 0, 1, 1, 1, nil) + set(c, 0, 2, 2, 1, nil).Release() + set(c, 0, 3, 3, 1, nil).Release() + set(c, 0, 4, 4, 1, nil).Release() + set(c, 0, 5, 5, 1, nil).Release() + if h := c.Get(0, 2, nil); h != nil { // 1,3,4,5,2 + h.Release() + } + set(c, 0, 9, 9, 10, nil).Release() // 5,2,9 + + for _, key := range []uint64{9, 2, 5, 1} { + h := c.Get(0, key, nil) + if h == nil { + t.Errorf("miss for key '%d'", key) } else { - if r.Value().(int) != int(x) { - t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int)) + if x := h.Value().(int); x != int(key) { + t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x) } - r.Release() + h.Release() } } o1.Release() - for _, x := range []uint64{1, 2, 5} { - r, ok := ns.Get(x, nil) - if !ok { - t.Errorf("miss for key '%d'", x) + for _, key := range []uint64{1, 2, 5} { + h := c.Get(0, key, nil) + if h == nil { + t.Errorf("miss for key '%d'", key) } else { - if r.Value().(int) != int(x) { - t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int)) + if x := h.Value().(int); x != int(key) { + t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x) } - r.Release() + h.Release() } } - for _, x := range []uint64{3, 4, 9} { - r, ok := ns.Get(x, nil) - if ok { - t.Errorf("hit for key '%d'", x) - if r.Value().(int) != int(x) { - t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int)) + for _, key := range []uint64{3, 4, 9} { + h := c.Get(0, key, nil) + if h != nil { + t.Errorf("hit for key '%d'", key) + if x := h.Value().(int); x != int(key) { + t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x) } - r.Release() + h.Release() } } } -func TestLRUCache_SetGet(t *testing.T) { - c := NewLRUCache(13) - ns := c.GetNamespace(0) - for i := 0; i < 200; i++ { - n := uint64(rand.Intn(99999) % 20) - set(ns, n, n, 1, nil).Release() - if p, ok := ns.Get(n, nil); ok { - if p.Value() == nil { - t.Errorf("key '%d' contains nil value", n) +func TestLRUCache_Evict(t *testing.T) { + c := NewCache(NewLRU(6)) + set(c, 0, 1, 1, 1, nil).Release() + set(c, 0, 2, 2, 1, nil).Release() + set(c, 1, 1, 4, 1, nil).Release() + set(c, 1, 2, 5, 1, nil).Release() + set(c, 2, 1, 6, 1, nil).Release() + set(c, 2, 2, 7, 1, nil).Release() + + for ns := 0; ns < 3; ns++ { + for key := 1; key < 3; key++ { + if h := c.Get(uint64(ns), uint64(key), nil); h != nil { + h.Release() } else { - got := p.Value().(uint64) - if got != n { - t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, got) - } + t.Errorf("Cache.Get on #%d.%d return nil", ns, key) } - p.Release() - } else { - t.Errorf("key '%d' doesn't exist", n) } } -} -func TestLRUCache_Purge(t *testing.T) { - c := NewLRUCache(3) - ns1 := c.GetNamespace(0) - o1 := set(ns1, 1, 1, 1, nil) - o2 := set(ns1, 2, 2, 1, nil) - ns1.Purge(nil) - set(ns1, 3, 3, 1, nil).Release() - for _, x := range []uint64{1, 2, 3} { - r, ok := ns1.Get(x, nil) - if !ok { - t.Errorf("miss for key '%d'", x) - } else { - if r.Value().(int) != int(x) { - t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int)) + if ok := c.Evict(0, 1); !ok { + t.Error("first Cache.Evict on #0.1 return false") + } + if ok := c.Evict(0, 1); ok { + t.Error("second Cache.Evict on #0.1 return true") + } + if h := c.Get(0, 1, nil); h != nil { + t.Errorf("Cache.Get on #0.1 return non-nil: %v", h.Value()) + } + + c.EvictNS(1) + if h := c.Get(1, 1, nil); h != nil { + t.Errorf("Cache.Get on #1.1 return non-nil: %v", h.Value()) + } + if h := c.Get(1, 2, nil); h != nil { + t.Errorf("Cache.Get on #1.2 return non-nil: %v", h.Value()) + } + + c.EvictAll() + for ns := 0; ns < 3; ns++ { + for key := 1; key < 3; key++ { + if h := c.Get(uint64(ns), uint64(key), nil); h != nil { + t.Errorf("Cache.Get on #%d.%d return non-nil: %v", ns, key, h.Value()) } - r.Release() } } - o1.Release() - o2.Release() - for _, x := range []uint64{1, 2} { - r, ok := ns1.Get(x, nil) - if ok { - t.Errorf("hit for key '%d'", x) - if r.Value().(int) != int(x) { - t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int)) - } - r.Release() +} + +func TestLRUCache_Delete(t *testing.T) { + delFuncCalled := 0 + delFunc := func() { + delFuncCalled++ + } + + c := NewCache(NewLRU(2)) + set(c, 0, 1, 1, 1, nil).Release() + set(c, 0, 2, 2, 1, nil).Release() + + if ok := c.Delete(0, 1, delFunc); !ok { + t.Error("Cache.Delete on #1 return false") + } + if h := c.Get(0, 1, nil); h != nil { + t.Errorf("Cache.Get on #1 return non-nil: %v", h.Value()) + } + if ok := c.Delete(0, 1, delFunc); ok { + t.Error("Cache.Delete on #1 return true") + } + + h2 := c.Get(0, 2, nil) + if h2 == nil { + t.Error("Cache.Get on #2 return nil") + } + if ok := c.Delete(0, 2, delFunc); !ok { + t.Error("(1) Cache.Delete on #2 return false") + } + if ok := c.Delete(0, 2, delFunc); !ok { + t.Error("(2) Cache.Delete on #2 return false") + } + + set(c, 0, 3, 3, 1, nil).Release() + set(c, 0, 4, 4, 1, nil).Release() + c.Get(0, 2, nil).Release() + + for key := 2; key <= 4; key++ { + if h := c.Get(0, uint64(key), nil); h != nil { + h.Release() + } else { + t.Errorf("Cache.Get on #%d return nil", key) } } -} -func BenchmarkLRUCache_SetRelease(b *testing.B) { - capacity := b.N / 100 - if capacity <= 0 { - capacity = 10 + h2.Release() + if h := c.Get(0, 2, nil); h != nil { + t.Errorf("Cache.Get on #2 return non-nil: %v", h.Value()) } - c := NewLRUCache(capacity) - ns := c.GetNamespace(0) - b.ResetTimer() - for i := uint64(0); i < uint64(b.N); i++ { - set(ns, i, nil, 1, nil).Release() + + if delFuncCalled != 4 { + t.Errorf("delFunc isn't called 4 times: got=%d", delFuncCalled) } } -func BenchmarkLRUCache_SetReleaseTwice(b *testing.B) { - capacity := b.N / 100 - if capacity <= 0 { - capacity = 10 +func TestLRUCache_Close(t *testing.T) { + relFuncCalled := 0 + relFunc := func() { + relFuncCalled++ + } + delFuncCalled := 0 + delFunc := func() { + delFuncCalled++ } - c := NewLRUCache(capacity) - ns := c.GetNamespace(0) - b.ResetTimer() - na := b.N / 2 - nb := b.N - na + c := NewCache(NewLRU(2)) + set(c, 0, 1, 1, 1, relFunc).Release() + set(c, 0, 2, 2, 1, relFunc).Release() - for i := uint64(0); i < uint64(na); i++ { - set(ns, i, nil, 1, nil).Release() + h3 := set(c, 0, 3, 3, 1, relFunc) + if h3 == nil { + t.Error("Cache.Get on #3 return nil") } + if ok := c.Delete(0, 3, delFunc); !ok { + t.Error("Cache.Delete on #3 return false") + } + + c.Close() - for i := uint64(0); i < uint64(nb); i++ { - set(ns, i, nil, 1, nil).Release() + if relFuncCalled != 3 { + t.Errorf("relFunc isn't called 3 times: got=%d", relFuncCalled) + } + if delFuncCalled != 1 { + t.Errorf("delFunc isn't called 1 times: got=%d", delFuncCalled) } } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/empty_cache.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/empty_cache.go deleted file mode 100644 index 1fbf81459..000000000 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/empty_cache.go +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com> -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package cache - -import ( - "sync" - "sync/atomic" -) - -type emptyCache struct { - sync.Mutex - table map[uint64]*emptyNS -} - -// NewEmptyCache creates a new initialized empty cache. -func NewEmptyCache() Cache { - return &emptyCache{ - table: make(map[uint64]*emptyNS), - } -} - -func (c *emptyCache) GetNamespace(id uint64) Namespace { - c.Lock() - defer c.Unlock() - - if ns, ok := c.table[id]; ok { - return ns - } - - ns := &emptyNS{ - cache: c, - id: id, - table: make(map[uint64]*emptyNode), - } - c.table[id] = ns - return ns -} - -func (c *emptyCache) Purge(fin PurgeFin) { - c.Lock() - for _, ns := range c.table { - ns.purgeNB(fin) - } - c.Unlock() -} - -func (c *emptyCache) Zap(closed bool) { - c.Lock() - for _, ns := range c.table { - ns.zapNB(closed) - } - c.table = make(map[uint64]*emptyNS) - c.Unlock() -} - -func (*emptyCache) SetCapacity(capacity int) {} - -type emptyNS struct { - cache *emptyCache - id uint64 - table map[uint64]*emptyNode - state nsState -} - -func (ns *emptyNS) Get(key uint64, setf SetFunc) (o Object, ok bool) { - ns.cache.Lock() - - switch ns.state { - case nsZapped: - ns.cache.Unlock() - if setf == nil { - return - } - - var value interface{} - var fin func() - ok, value, _, fin = setf() - if ok { - o = &fakeObject{ - value: value, - fin: fin, - } - } - return - case nsClosed: - ns.cache.Unlock() - return - } - - n, ok := ns.table[key] - if ok { - n.ref++ - } else { - if setf == nil { - ns.cache.Unlock() - return - } - - var value interface{} - var fin func() - ok, value, _, fin = setf() - if !ok { - ns.cache.Unlock() - return - } - - n = &emptyNode{ - ns: ns, - key: key, - value: value, - setfin: fin, - ref: 1, - } - ns.table[key] = n - } - - ns.cache.Unlock() - o = &emptyObject{node: n} - return -} - -func (ns *emptyNS) Delete(key uint64, fin DelFin) bool { - ns.cache.Lock() - - if ns.state != nsEffective { - ns.cache.Unlock() - if fin != nil { - fin(false) - } - return false - } - - n, ok := ns.table[key] - if !ok { - ns.cache.Unlock() - if fin != nil { - fin(false) - } - return false - } - n.delfin = fin - ns.cache.Unlock() - return true -} - -func (ns *emptyNS) purgeNB(fin PurgeFin) { - if ns.state != nsEffective { - return - } - for _, n := range ns.table { - n.purgefin = fin - } -} - -func (ns *emptyNS) Purge(fin PurgeFin) { - ns.cache.Lock() - ns.purgeNB(fin) - ns.cache.Unlock() -} - -func (ns *emptyNS) zapNB(closed bool) { - if ns.state != nsEffective { - return - } - for _, n := range ns.table { - n.execFin() - } - if closed { - ns.state = nsClosed - } else { - ns.state = nsZapped - } - ns.table = nil -} - -func (ns *emptyNS) Zap(closed bool) { - ns.cache.Lock() - ns.zapNB(closed) - delete(ns.cache.table, ns.id) - ns.cache.Unlock() -} - -type emptyNode struct { - ns *emptyNS - key uint64 - value interface{} - ref int - setfin SetFin - delfin DelFin - purgefin PurgeFin -} - -func (n *emptyNode) execFin() { - if n.setfin != nil { - n.setfin() - n.setfin = nil - } - if n.purgefin != nil { - n.purgefin(n.ns.id, n.key, n.delfin) - n.delfin = nil - n.purgefin = nil - } else if n.delfin != nil { - n.delfin(true) - n.delfin = nil - } -} - -func (n *emptyNode) evict() { - n.ns.cache.Lock() - n.ref-- - if n.ref == 0 { - if n.ns.state == nsEffective { - // Remove elem. - delete(n.ns.table, n.key) - // Execute finalizer. - n.execFin() - } - } else if n.ref < 0 { - panic("leveldb/cache: emptyNode: negative node reference") - } - n.ns.cache.Unlock() -} - -type emptyObject struct { - node *emptyNode - once uint32 -} - -func (o *emptyObject) Value() interface{} { - if atomic.LoadUint32(&o.once) == 0 { - return o.node.value - } - return nil -} - -func (o *emptyObject) Release() { - if !atomic.CompareAndSwapUint32(&o.once, 0, 1) { - return - } - o.node.evict() - o.node = nil -} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru.go new file mode 100644 index 000000000..d9a84cde1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru.go @@ -0,0 +1,195 @@ +// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com> +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package cache + +import ( + "sync" + "unsafe" +) + +type lruNode struct { + n *Node + h *Handle + ban bool + + next, prev *lruNode +} + +func (n *lruNode) insert(at *lruNode) { + x := at.next + at.next = n + n.prev = at + n.next = x + x.prev = n +} + +func (n *lruNode) remove() { + if n.prev != nil { + n.prev.next = n.next + n.next.prev = n.prev + n.prev = nil + n.next = nil + } else { + panic("BUG: removing removed node") + } +} + +type lru struct { + mu sync.Mutex + capacity int + used int + recent lruNode +} + +func (r *lru) reset() { + r.recent.next = &r.recent + r.recent.prev = &r.recent + r.used = 0 +} + +func (r *lru) Capacity() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.capacity +} + +func (r *lru) SetCapacity(capacity int) { + var evicted []*lruNode + + r.mu.Lock() + r.capacity = capacity + for r.used > r.capacity { + rn := r.recent.prev + if rn == nil { + panic("BUG: invalid LRU used or capacity counter") + } + rn.remove() + rn.n.CacheData = nil + r.used -= rn.n.Size() + evicted = append(evicted, rn) + } + r.mu.Unlock() + + for _, rn := range evicted { + rn.h.Release() + } +} + +func (r *lru) Promote(n *Node) { + var evicted []*lruNode + + r.mu.Lock() + if n.CacheData == nil { + if n.Size() <= r.capacity { + rn := &lruNode{n: n, h: n.GetHandle()} + rn.insert(&r.recent) + n.CacheData = unsafe.Pointer(rn) + r.used += n.Size() + + for r.used > r.capacity { + rn := r.recent.prev + if rn == nil { + panic("BUG: invalid LRU used or capacity counter") + } + rn.remove() + rn.n.CacheData = nil + r.used -= rn.n.Size() + evicted = append(evicted, rn) + } + } + } else { + rn := (*lruNode)(n.CacheData) + if !rn.ban { + rn.remove() + rn.insert(&r.recent) + } + } + r.mu.Unlock() + + for _, rn := range evicted { + rn.h.Release() + } +} + +func (r *lru) Ban(n *Node) { + r.mu.Lock() + if n.CacheData == nil { + n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true}) + } else { + rn := (*lruNode)(n.CacheData) + if !rn.ban { + rn.remove() + rn.ban = true + r.used -= rn.n.Size() + r.mu.Unlock() + + rn.h.Release() + rn.h = nil + return + } + } + r.mu.Unlock() +} + +func (r *lru) Evict(n *Node) { + r.mu.Lock() + rn := (*lruNode)(n.CacheData) + if rn == nil || rn.ban { + r.mu.Unlock() + return + } + n.CacheData = nil + r.mu.Unlock() + + rn.h.Release() +} + +func (r *lru) EvictNS(ns uint64) { + var evicted []*lruNode + + r.mu.Lock() + for e := r.recent.prev; e != &r.recent; { + rn := e + e = e.prev + if rn.n.NS() == ns { + rn.remove() + rn.n.CacheData = nil + r.used -= rn.n.Size() + evicted = append(evicted, rn) + } + } + r.mu.Unlock() + + for _, rn := range evicted { + rn.h.Release() + } +} + +func (r *lru) EvictAll() { + r.mu.Lock() + back := r.recent.prev + for rn := back; rn != &r.recent; rn = rn.prev { + rn.n.CacheData = nil + } + r.reset() + r.mu.Unlock() + + for rn := back; rn != &r.recent; rn = rn.prev { + rn.h.Release() + } +} + +func (r *lru) Close() error { + return nil +} + +// NewLRU create a new LRU-cache. +func NewLRU(capacity int) Cacher { + r := &lru{capacity: capacity} + r.reset() + return r +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go deleted file mode 100644 index 3c98e076b..000000000 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go +++ /dev/null @@ -1,354 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com> -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package cache - -import ( - "sync" - "sync/atomic" -) - -// lruCache represent a LRU cache state. -type lruCache struct { - sync.Mutex - - recent lruNode - table map[uint64]*lruNs - capacity int - size int -} - -// NewLRUCache creates a new initialized LRU cache with the given capacity. -func NewLRUCache(capacity int) Cache { - c := &lruCache{ - table: make(map[uint64]*lruNs), - capacity: capacity, - } - c.recent.rNext = &c.recent - c.recent.rPrev = &c.recent - return c -} - -// SetCapacity set cache capacity. -func (c *lruCache) SetCapacity(capacity int) { - c.Lock() - c.capacity = capacity - c.evict() - c.Unlock() -} - -// GetNamespace return namespace object for given id. -func (c *lruCache) GetNamespace(id uint64) Namespace { - c.Lock() - defer c.Unlock() - - if p, ok := c.table[id]; ok { - return p - } - - p := &lruNs{ - lru: c, - id: id, - table: make(map[uint64]*lruNode), - } - c.table[id] = p - return p -} - -// Purge purge entire cache. -func (c *lruCache) Purge(fin PurgeFin) { - c.Lock() - for _, ns := range c.table { - ns.purgeNB(fin) - } - c.Unlock() -} - -func (c *lruCache) Zap(closed bool) { - c.Lock() - for _, ns := range c.table { - ns.zapNB(closed) - } - c.table = make(map[uint64]*lruNs) - c.Unlock() -} - -func (c *lruCache) evict() { - top := &c.recent - for n := c.recent.rPrev; c.size > c.capacity && n != top; { - n.state = nodeEvicted - n.rRemove() - n.evictNB() - c.size -= n.charge - n = c.recent.rPrev - } -} - -type lruNs struct { - lru *lruCache - id uint64 - table map[uint64]*lruNode - state nsState -} - -func (ns *lruNs) Get(key uint64, setf SetFunc) (o Object, ok bool) { - lru := ns.lru - lru.Lock() - - switch ns.state { - case nsZapped: - lru.Unlock() - if setf == nil { - return - } - - var value interface{} - var fin func() - ok, value, _, fin = setf() - if ok { - o = &fakeObject{ - value: value, - fin: fin, - } - } - return - case nsClosed: - lru.Unlock() - return - } - - n, ok := ns.table[key] - if ok { - switch n.state { - case nodeEvicted: - // Insert to recent list. - n.state = nodeEffective - n.ref++ - lru.size += n.charge - lru.evict() - fallthrough - case nodeEffective: - // Bump to front - n.rRemove() - n.rInsert(&lru.recent) - } - n.ref++ - } else { - if setf == nil { - lru.Unlock() - return - } - - var value interface{} - var charge int - var fin func() - ok, value, charge, fin = setf() - if !ok { - lru.Unlock() - return - } - - n = &lruNode{ - ns: ns, - key: key, - value: value, - charge: charge, - setfin: fin, - ref: 2, - } - ns.table[key] = n - n.rInsert(&lru.recent) - - lru.size += charge - lru.evict() - } - - lru.Unlock() - o = &lruObject{node: n} - return -} - -func (ns *lruNs) Delete(key uint64, fin DelFin) bool { - lru := ns.lru - lru.Lock() - - if ns.state != nsEffective { - lru.Unlock() - if fin != nil { - fin(false) - } - return false - } - - n, ok := ns.table[key] - if !ok { - lru.Unlock() - if fin != nil { - fin(false) - } - return false - } - - n.delfin = fin - switch n.state { - case nodeRemoved: - lru.Unlock() - return false - case nodeEffective: - lru.size -= n.charge - n.rRemove() - n.evictNB() - } - n.state = nodeRemoved - - lru.Unlock() - return true -} - -func (ns *lruNs) purgeNB(fin PurgeFin) { - lru := ns.lru - if ns.state != nsEffective { - return - } - - for _, n := range ns.table { - n.purgefin = fin - if n.state == nodeEffective { - lru.size -= n.charge - n.rRemove() - n.evictNB() - } - n.state = nodeRemoved - } -} - -func (ns *lruNs) Purge(fin PurgeFin) { - ns.lru.Lock() - ns.purgeNB(fin) - ns.lru.Unlock() -} - -func (ns *lruNs) zapNB(closed bool) { - lru := ns.lru - if ns.state != nsEffective { - return - } - - if closed { - ns.state = nsClosed - } else { - ns.state = nsZapped - } - for _, n := range ns.table { - if n.state == nodeEffective { - lru.size -= n.charge - n.rRemove() - } - n.state = nodeRemoved - n.execFin() - } - ns.table = nil -} - -func (ns *lruNs) Zap(closed bool) { - ns.lru.Lock() - ns.zapNB(closed) - delete(ns.lru.table, ns.id) - ns.lru.Unlock() -} - -type lruNode struct { - ns *lruNs - - rNext, rPrev *lruNode - - key uint64 - value interface{} - charge int - ref int - state nodeState - setfin SetFin - delfin DelFin - purgefin PurgeFin -} - -func (n *lruNode) rInsert(at *lruNode) { - x := at.rNext - at.rNext = n - n.rPrev = at - n.rNext = x - x.rPrev = n -} - -func (n *lruNode) rRemove() bool { - // only remove if not already removed - if n.rPrev == nil { - return false - } - - n.rPrev.rNext = n.rNext - n.rNext.rPrev = n.rPrev - n.rPrev = nil - n.rNext = nil - - return true -} - -func (n *lruNode) execFin() { - if n.setfin != nil { - n.setfin() - n.setfin = nil - } - if n.purgefin != nil { - n.purgefin(n.ns.id, n.key, n.delfin) - n.delfin = nil - n.purgefin = nil - } else if n.delfin != nil { - n.delfin(true) - n.delfin = nil - } -} - -func (n *lruNode) evictNB() { - n.ref-- - if n.ref == 0 { - if n.ns.state == nsEffective { - // remove elem - delete(n.ns.table, n.key) - // execute finalizer - n.execFin() - } - } else if n.ref < 0 { - panic("leveldb/cache: lruCache: negative node reference") - } -} - -func (n *lruNode) evict() { - n.ns.lru.Lock() - n.evictNB() - n.ns.lru.Unlock() -} - -type lruObject struct { - node *lruNode - once uint32 -} - -func (o *lruObject) Value() interface{} { - if atomic.LoadUint32(&o.once) == 0 { - return o.node.value - } - return nil -} - -func (o *lruObject) Release() { - if !atomic.CompareAndSwapUint32(&o.once, 0, 1) { - return - } - - o.node.evict() - o.node = nil -} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/config.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/config.go deleted file mode 100644 index 511058897..000000000 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/config.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com> -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -const ( - kNumLevels = 7 - - // Level-0 compaction is started when we hit this many files. - kL0_CompactionTrigger float64 = 4 - - // Soft limit on number of level-0 files. We slow down writes at this point. - kL0_SlowdownWritesTrigger = 8 - - // Maximum number of level-0 files. We stop writes at this point. - kL0_StopWritesTrigger = 12 - - // Maximum level to which a new compacted memdb is pushed if it - // does not create overlap. We try to push to level 2 to avoid the - // relatively expensive level 0=>1 compactions and to avoid some - // expensive manifest file operations. We do not push all the way to - // the largest level since that can generate a lot of wasted disk - // space if the same key space is being repeatedly overwritten. - kMaxMemCompactLevel = 2 - - // Maximum size of a table. - kMaxTableSize = 2 * 1048576 - - // Maximum bytes of overlaps in grandparent (i.e., level+2) before we - // stop building a single file in a level->level+1 compaction. - kMaxGrandParentOverlapBytes = 10 * kMaxTableSize - - // Maximum number of bytes in all compacted files. We avoid expanding - // the lower level file set of a compaction if it would make the - // total compaction cover more than this many bytes. - kExpCompactionMaxBytes = 25 * kMaxTableSize -) diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go index a036e0893..a351874ed 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go @@ -9,13 +9,12 @@ package leveldb import ( "bytes" "fmt" + "github.com/syndtr/goleveldb/leveldb/filter" + "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/storage" "io" "math/rand" "testing" - - "github.com/syndtr/goleveldb/leveldb/cache" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" ) const ctValSize = 1000 @@ -32,8 +31,8 @@ func newDbCorruptHarnessWopt(t *testing.T, o *opt.Options) *dbCorruptHarness { func newDbCorruptHarness(t *testing.T) *dbCorruptHarness { return newDbCorruptHarnessWopt(t, &opt.Options{ - BlockCache: cache.NewLRUCache(100), - Strict: opt.StrictJournalChecksum, + BlockCacheCapacity: 100, + Strict: opt.StrictJournalChecksum, }) } @@ -96,21 +95,22 @@ func (h *dbCorruptHarness) deleteRand(n, max int, rnd *rand.Rand) { } } -func (h *dbCorruptHarness) corrupt(ft storage.FileType, offset, n int) { +func (h *dbCorruptHarness) corrupt(ft storage.FileType, fi, offset, n int) { p := &h.dbHarness t := p.t - var file storage.File ff, _ := p.stor.GetFiles(ft) - for _, f := range ff { - if file == nil || f.Num() > file.Num() { - file = f - } + sff := files(ff) + sff.sort() + if fi < 0 { + fi = len(sff) - 1 } - if file == nil { - t.Fatalf("no such file with type %q", ft) + if fi >= len(sff) { + t.Fatalf("no such file with type %q with index %d", ft, fi) } + file := sff[fi] + r, err := file.Open() if err != nil { t.Fatal("cannot open file: ", err) @@ -225,8 +225,8 @@ func TestCorruptDB_Journal(t *testing.T) { h.build(100) h.check(100, 100) h.closeDB() - h.corrupt(storage.TypeJournal, 19, 1) - h.corrupt(storage.TypeJournal, 32*1024+1000, 1) + h.corrupt(storage.TypeJournal, -1, 19, 1) + h.corrupt(storage.TypeJournal, -1, 32*1024+1000, 1) h.openDB() h.check(36, 36) @@ -242,7 +242,7 @@ func TestCorruptDB_Table(t *testing.T) { h.compactRangeAt(0, "", "") h.compactRangeAt(1, "", "") h.closeDB() - h.corrupt(storage.TypeTable, 100, 1) + h.corrupt(storage.TypeTable, -1, 100, 1) h.openDB() h.check(99, 99) @@ -256,7 +256,7 @@ func TestCorruptDB_TableIndex(t *testing.T) { h.build(10000) h.compactMem() h.closeDB() - h.corrupt(storage.TypeTable, -2000, 500) + h.corrupt(storage.TypeTable, -1, -2000, 500) h.openDB() h.check(5000, 9999) @@ -267,9 +267,9 @@ func TestCorruptDB_TableIndex(t *testing.T) { func TestCorruptDB_MissingManifest(t *testing.T) { rnd := rand.New(rand.NewSource(0x0badda7a)) h := newDbCorruptHarnessWopt(t, &opt.Options{ - BlockCache: cache.NewLRUCache(100), - Strict: opt.StrictJournalChecksum, - WriteBuffer: 1000 * 60, + BlockCacheCapacity: 100, + Strict: opt.StrictJournalChecksum, + WriteBuffer: 1000 * 60, }) h.build(1000) @@ -355,7 +355,7 @@ func TestCorruptDB_CorruptedManifest(t *testing.T) { h.compactMem() h.compactRange("", "") h.closeDB() - h.corrupt(storage.TypeManifest, 0, 1000) + h.corrupt(storage.TypeManifest, -1, 0, 1000) h.openAssert(false) h.recover() @@ -370,7 +370,7 @@ func TestCorruptDB_CompactionInputError(t *testing.T) { h.build(10) h.compactMem() h.closeDB() - h.corrupt(storage.TypeTable, 100, 1) + h.corrupt(storage.TypeTable, -1, 100, 1) h.openDB() h.check(9, 9) @@ -387,7 +387,7 @@ func TestCorruptDB_UnrelatedKeys(t *testing.T) { h.build(10) h.compactMem() h.closeDB() - h.corrupt(storage.TypeTable, 100, 1) + h.corrupt(storage.TypeTable, -1, 100, 1) h.openDB() h.put(string(tkey(1000)), string(tval(1000, ctValSize))) @@ -470,3 +470,31 @@ func TestCorruptDB_MissingTableFiles(t *testing.T) { h.close() } + +func TestCorruptDB_RecoverTable(t *testing.T) { + h := newDbCorruptHarnessWopt(t, &opt.Options{ + WriteBuffer: 112 * opt.KiB, + CompactionTableSize: 90 * opt.KiB, + Filter: filter.NewBloomFilter(10), + }) + + h.build(1000) + h.compactMem() + h.compactRangeAt(0, "", "") + h.compactRangeAt(1, "", "") + seq := h.db.seq + h.closeDB() + h.corrupt(storage.TypeTable, 0, 1000, 1) + h.corrupt(storage.TypeTable, 3, 10000, 1) + // Corrupted filter shouldn't affect recovery. + h.corrupt(storage.TypeTable, 3, 113888, 10) + h.corrupt(storage.TypeTable, -1, 20000, 1) + + h.recover() + if h.db.seq != seq { + t.Errorf("invalid seq, want=%d got=%d", seq, h.db.seq) + } + h.check(985, 985) + + h.close() +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go index 8e975dc63..323353b2a 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go @@ -7,15 +7,17 @@ package leveldb import ( - "errors" + "container/list" "fmt" "io" "os" "runtime" "strings" "sync" + "sync/atomic" "time" + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/journal" "github.com/syndtr/goleveldb/leveldb/memdb" @@ -30,41 +32,46 @@ type DB struct { // Need 64-bit alignment. seq uint64 + // Session. s *session - // MemDB + // MemDB. memMu sync.RWMutex - mem *memdb.DB - frozenMem *memdb.DB + memPool chan *memdb.DB + mem, frozenMem *memDB journal *journal.Writer journalWriter storage.Writer journalFile storage.File frozenJournalFile storage.File frozenSeq uint64 - // Snapshot + // Snapshot. snapsMu sync.Mutex - snapsRoot snapshotElement + snapsList *list.List - // Write + // Stats. + aliveSnaps, aliveIters int32 + + // Write. writeC chan *Batch writeMergedC chan bool writeLockC chan struct{} writeAckC chan error + writeDelay time.Duration + writeDelayN int journalC chan *Batch journalAckC chan error - // Compaction - tcompCmdC chan cCmd - tcompPauseC chan chan<- struct{} - tcompTriggerC chan struct{} - mcompCmdC chan cCmd - mcompTriggerC chan struct{} - compErrC chan error - compErrSetC chan error - compStats [kNumLevels]cStats - - // Close + // Compaction. + tcompCmdC chan cCmd + tcompPauseC chan chan<- struct{} + mcompCmdC chan cCmd + compErrC chan error + compPerErrC chan error + compErrSetC chan error + compStats []cStats + + // Close. closeW sync.WaitGroup closeC chan struct{} closed uint32 @@ -77,7 +84,11 @@ func openDB(s *session) (*DB, error) { db := &DB{ s: s, // Initial sequence - seq: s.stSeq, + seq: s.stSeqNum, + // MemDB + memPool: make(chan *memdb.DB, 1), + // Snapshot + snapsList: list.New(), // Write writeC: make(chan *Batch), writeMergedC: make(chan bool), @@ -86,17 +97,16 @@ func openDB(s *session) (*DB, error) { journalC: make(chan *Batch), journalAckC: make(chan error), // Compaction - tcompCmdC: make(chan cCmd), - tcompPauseC: make(chan chan<- struct{}), - tcompTriggerC: make(chan struct{}, 1), - mcompCmdC: make(chan cCmd), - mcompTriggerC: make(chan struct{}, 1), - compErrC: make(chan error), - compErrSetC: make(chan error), + tcompCmdC: make(chan cCmd), + tcompPauseC: make(chan chan<- struct{}), + mcompCmdC: make(chan cCmd), + compErrC: make(chan error), + compPerErrC: make(chan error), + compErrSetC: make(chan error), + compStats: make([]cStats, s.o.GetNumLevel()), // Close closeC: make(chan struct{}), } - db.initSnapshot() if err := db.recoverJournal(); err != nil { return nil, err @@ -112,8 +122,9 @@ func openDB(s *session) (*DB, error) { return nil, err } - // Don't include compaction error goroutine into wait group. + // Doesn't need to be included in the wait group. go db.compactionError() + go db.mpoolDrain() db.closeW.Add(3) go db.tCompaction() @@ -135,9 +146,10 @@ func openDB(s *session) (*DB, error) { // detected in the DB. Corrupted DB can be recovered with Recover // function. // +// The returned DB instance is goroutine-safe. // The DB must be closed after use, by calling Close method. -func Open(p storage.Storage, o *opt.Options) (db *DB, err error) { - s, err := newSession(p, o) +func Open(stor storage.Storage, o *opt.Options) (db *DB, err error) { + s, err := newSession(stor, o) if err != nil { return } @@ -177,6 +189,7 @@ func Open(p storage.Storage, o *opt.Options) (db *DB, err error) { // detected in the DB. Corrupted DB can be recovered with Recover // function. // +// The returned DB instance is goroutine-safe. // The DB must be closed after use, by calling Close method. func OpenFile(path string, o *opt.Options) (db *DB, err error) { stor, err := storage.OpenFile(path) @@ -197,9 +210,10 @@ func OpenFile(path string, o *opt.Options) (db *DB, err error) { // The DB must already exist or it will returns an error. // Also, Recover will ignore ErrorIfMissing and ErrorIfExist options. // +// The returned DB instance is goroutine-safe. // The DB must be closed after use, by calling Close method. -func Recover(p storage.Storage, o *opt.Options) (db *DB, err error) { - s, err := newSession(p, o) +func Recover(stor storage.Storage, o *opt.Options) (db *DB, err error) { + s, err := newSession(stor, o) if err != nil { return } @@ -225,6 +239,7 @@ func Recover(p storage.Storage, o *opt.Options) (db *DB, err error) { // RecoverFile uses standard file-system backed storage implementation as desribed // in the leveldb/storage package. // +// The returned DB instance is goroutine-safe. // The DB must be closed after use, by calling Close method. func RecoverFile(path string, o *opt.Options) (db *DB, err error) { stor, err := storage.OpenFile(path) @@ -241,16 +256,28 @@ func RecoverFile(path string, o *opt.Options) (db *DB, err error) { } func recoverTable(s *session, o *opt.Options) error { - ff0, err := s.getFiles(storage.TypeTable) + o = dupOptions(o) + // Mask StrictReader, lets StrictRecovery doing its job. + o.Strict &= ^opt.StrictReader + + // Get all tables and sort it by file number. + tableFiles_, err := s.getFiles(storage.TypeTable) if err != nil { return err } - ff1 := files(ff0) - ff1.sort() + tableFiles := files(tableFiles_) + tableFiles.sort() - var mSeq uint64 - var good, corrupted int - rec := new(sessionRecord) + var ( + maxSeq uint64 + recoveredKey, goodKey, corruptedKey, corruptedBlock, droppedTable int + + // We will drop corrupted table. + strict = o.GetStrict(opt.StrictRecovery) + + rec = &sessionRecord{numLevel: o.GetNumLevel()} + bpool = util.NewBufferPool(o.GetBlockSize() + 5) + ) buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) { tmp = s.newTemp() writer, err := tmp.Create() @@ -264,8 +291,9 @@ func recoverTable(s *session, o *opt.Options) error { tmp = nil } }() + + // Copy entries. tw := table.NewWriter(writer, o) - // Copy records. for iter.Next() { key := iter.Key() if validIkey(key) { @@ -296,45 +324,73 @@ func recoverTable(s *session, o *opt.Options) error { if err != nil { return err } - defer reader.Close() + var closed bool + defer func() { + if !closed { + reader.Close() + } + }() + // Get file size. size, err := reader.Seek(0, 2) if err != nil { return err } - var tSeq uint64 - var tgood, tcorrupted, blockerr int - var min, max []byte - tr := table.NewReader(reader, size, nil, o) + + var ( + tSeq uint64 + tgoodKey, tcorruptedKey, tcorruptedBlock int + imin, imax []byte + ) + tr, err := table.NewReader(reader, size, storage.NewFileInfo(file), nil, bpool, o) + if err != nil { + return err + } iter := tr.NewIterator(nil, nil) - iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) { - s.logf("table@recovery found error @%d %q", file.Num(), err) - blockerr++ - }) + if itererr, ok := iter.(iterator.ErrorCallbackSetter); ok { + itererr.SetErrorCallback(func(err error) { + if errors.IsCorrupted(err) { + s.logf("table@recovery block corruption @%d %q", file.Num(), err) + tcorruptedBlock++ + } + }) + } + // Scan the table. for iter.Next() { key := iter.Key() - _, seq, _, ok := parseIkey(key) - if !ok { - tcorrupted++ + _, seq, _, kerr := parseIkey(key) + if kerr != nil { + tcorruptedKey++ continue } - tgood++ + tgoodKey++ if seq > tSeq { tSeq = seq } - if min == nil { - min = append([]byte{}, key...) + if imin == nil { + imin = append([]byte{}, key...) } - max = append(max[:0], key...) + imax = append(imax[:0], key...) } if err := iter.Error(); err != nil { iter.Release() return err } iter.Release() - if tgood > 0 { - if tcorrupted > 0 || blockerr > 0 { + + goodKey += tgoodKey + corruptedKey += tcorruptedKey + corruptedBlock += tcorruptedBlock + + if strict && (tcorruptedKey > 0 || tcorruptedBlock > 0) { + droppedTable++ + s.logf("table@recovery dropped @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq) + return nil + } + + if tgoodKey > 0 { + if tcorruptedKey > 0 || tcorruptedBlock > 0 { // Rebuild the table. s.logf("table@recovery rebuilding @%d", file.Num()) iter := tr.NewIterator(nil, nil) @@ -343,62 +399,77 @@ func recoverTable(s *session, o *opt.Options) error { if err != nil { return err } + closed = true reader.Close() if err := file.Replace(tmp); err != nil { return err } size = newSize } - if tSeq > mSeq { - mSeq = tSeq + if tSeq > maxSeq { + maxSeq = tSeq } + recoveredKey += tgoodKey // Add table to level 0. - rec.addTable(0, file.Num(), uint64(size), min, max) - s.logf("table@recovery recovered @%d N·%d C·%d B·%d S·%d Q·%d", file.Num(), tgood, tcorrupted, blockerr, size, tSeq) + rec.addTable(0, file.Num(), uint64(size), imin, imax) + s.logf("table@recovery recovered @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq) } else { - s.logf("table@recovery unrecoverable @%d C·%d B·%d S·%d", file.Num(), tcorrupted, blockerr, size) + droppedTable++ + s.logf("table@recovery unrecoverable @%d Ck·%d Cb·%d S·%d", file.Num(), tcorruptedKey, tcorruptedBlock, size) } - good += tgood - corrupted += tcorrupted - return nil } + // Recover all tables. - if len(ff1) > 0 { - s.logf("table@recovery F·%d", len(ff1)) - s.markFileNum(ff1[len(ff1)-1].Num()) - for _, file := range ff1 { + if len(tableFiles) > 0 { + s.logf("table@recovery F·%d", len(tableFiles)) + + // Mark file number as used. + s.markFileNum(tableFiles[len(tableFiles)-1].Num()) + + for _, file := range tableFiles { if err := recoverTable(file); err != nil { return err } } - s.logf("table@recovery recovered F·%d N·%d C·%d Q·%d", len(ff1), good, corrupted, mSeq) + + s.logf("table@recovery recovered F·%d N·%d Gk·%d Ck·%d Q·%d", len(tableFiles), recoveredKey, goodKey, corruptedKey, maxSeq) } + // Set sequence number. - rec.setSeq(mSeq + 1) + rec.setSeqNum(maxSeq) + // Create new manifest. if err := s.create(); err != nil { return err } + // Commit. return s.commit(rec) } -func (d *DB) recoverJournal() error { - s := d.s - - ff0, err := s.getFiles(storage.TypeJournal) +func (db *DB) recoverJournal() error { + // Get all tables and sort it by file number. + journalFiles_, err := db.s.getFiles(storage.TypeJournal) if err != nil { return err } - ff1 := files(ff0) - ff1.sort() - ff2 := make([]storage.File, 0, len(ff1)) - for _, file := range ff1 { - if file.Num() >= s.stJournalNum || file.Num() == s.stPrevJournalNum { - s.markFileNum(file.Num()) - ff2 = append(ff2, file) + journalFiles := files(journalFiles_) + journalFiles.sort() + + // Discard older journal. + prev := -1 + for i, file := range journalFiles { + if file.Num() >= db.s.stJournalNum { + if prev >= 0 { + i-- + journalFiles[i] = journalFiles[prev] + } + journalFiles = journalFiles[i:] + break + } else if file.Num() == db.s.stPrevJournalNum { + prev = i } } @@ -406,38 +477,43 @@ func (d *DB) recoverJournal() error { var of storage.File var mem *memdb.DB batch := new(Batch) - cm := newCMem(s) + cm := newCMem(db.s) buf := new(util.Buffer) // Options. - strict := s.o.GetStrict(opt.StrictJournal) - checksum := s.o.GetStrict(opt.StrictJournalChecksum) - writeBuffer := s.o.GetWriteBuffer() + strict := db.s.o.GetStrict(opt.StrictJournal) + checksum := db.s.o.GetStrict(opt.StrictJournalChecksum) + writeBuffer := db.s.o.GetWriteBuffer() recoverJournal := func(file storage.File) error { - s.logf("journal@recovery recovering @%d", file.Num()) + db.logf("journal@recovery recovering @%d", file.Num()) reader, err := file.Open() if err != nil { return err } defer reader.Close() + + // Create/reset journal reader instance. if jr == nil { - jr = journal.NewReader(reader, dropper{s, file}, strict, checksum) + jr = journal.NewReader(reader, dropper{db.s, file}, strict, checksum) } else { - jr.Reset(reader, dropper{s, file}, strict, checksum) + jr.Reset(reader, dropper{db.s, file}, strict, checksum) } + + // Flush memdb and remove obsolete journal file. if of != nil { if mem.Len() > 0 { if err := cm.flush(mem, 0); err != nil { return err } } - if err := cm.commit(file.Num(), d.seq); err != nil { + if err := cm.commit(file.Num(), db.seq); err != nil { return err } cm.reset() of.Remove() of = nil } - // Reset memdb. + + // Replay journal to memdb. mem.Reset() for { r, err := jr.Next() @@ -445,43 +521,58 @@ func (d *DB) recoverJournal() error { if err == io.EOF { break } - return err + return errors.SetFile(err, file) } + buf.Reset() if _, err := buf.ReadFrom(r); err != nil { - if strict { - return err + if err == io.ErrUnexpectedEOF { + // This is error returned due to corruption, with strict == false. + continue + } else { + return errors.SetFile(err, file) } - continue } - if err := batch.decode(buf.Bytes()); err != nil { - return err - } - if err := batch.memReplay(mem); err != nil { - return err + if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mem); err != nil { + if strict || !errors.IsCorrupted(err) { + return errors.SetFile(err, file) + } else { + db.s.logf("journal error: %v (skipped)", err) + // We won't apply sequence number as it might be corrupted. + continue + } } - d.seq = batch.seq + uint64(batch.len()) + + // Save sequence number. + db.seq = batch.seq + uint64(batch.Len()) + + // Flush it if large enough. if mem.Size() >= writeBuffer { - // Large enough, flush it. if err := cm.flush(mem, 0); err != nil { return err } - // Reset memdb. mem.Reset() } } + of = file return nil } + // Recover all journals. - if len(ff2) > 0 { - s.logf("journal@recovery F·%d", len(ff2)) - mem = memdb.New(s.icmp, writeBuffer) - for _, file := range ff2 { + if len(journalFiles) > 0 { + db.logf("journal@recovery F·%d", len(journalFiles)) + + // Mark file number as used. + db.s.markFileNum(journalFiles[len(journalFiles)-1].Num()) + + mem = memdb.New(db.s.icmp, writeBuffer) + for _, file := range journalFiles { if err := recoverJournal(file); err != nil { return err } } + // Flush the last journal. if mem.Len() > 0 { if err := cm.flush(mem, 0); err != nil { @@ -489,72 +580,140 @@ func (d *DB) recoverJournal() error { } } } + // Create a new journal. - if _, err := d.newMem(0); err != nil { + if _, err := db.newMem(0); err != nil { return err } + // Commit. - if err := cm.commit(d.journalFile.Num(), d.seq); err != nil { + if err := cm.commit(db.journalFile.Num(), db.seq); err != nil { // Close journal. - if d.journal != nil { - d.journal.Close() - d.journalWriter.Close() + if db.journal != nil { + db.journal.Close() + db.journalWriter.Close() } return err } - // Remove the last journal. + + // Remove the last obsolete journal file. if of != nil { of.Remove() } + return nil } -func (d *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) { - s := d.s +func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) { + ikey := newIkey(key, seq, ktSeek) - ikey := newIKey(key, seq, tSeek) - - em, fm := d.getMems() - for _, m := range [...]*memdb.DB{em, fm} { + em, fm := db.getMems() + for _, m := range [...]*memDB{em, fm} { if m == nil { continue } - mk, mv, me := m.Find(ikey) + defer m.decref() + + mk, mv, me := m.mdb.Find(ikey) if me == nil { - ukey, _, t, ok := parseIkey(mk) - if ok && s.icmp.uCompare(ukey, key) == 0 { - if t == tDel { + ukey, _, kt, kerr := parseIkey(mk) + if kerr != nil { + // Shouldn't have had happen. + panic(kerr) + } + if db.s.icmp.uCompare(ukey, key) == 0 { + if kt == ktDel { return nil, ErrNotFound } - return mv, nil + return append([]byte{}, mv...), nil } } else if me != ErrNotFound { return nil, me } } - v := s.version() - value, cSched, err := v.get(ikey, ro) + v := db.s.version() + value, cSched, err := v.get(ikey, ro, false) v.release() if cSched { // Trigger table compaction. - d.compTrigger(d.tcompTriggerC) + db.compSendTrigger(db.tcompCmdC) + } + return +} + +func (db *DB) has(key []byte, seq uint64, ro *opt.ReadOptions) (ret bool, err error) { + ikey := newIkey(key, seq, ktSeek) + + em, fm := db.getMems() + for _, m := range [...]*memDB{em, fm} { + if m == nil { + continue + } + defer m.decref() + + mk, _, me := m.mdb.Find(ikey) + if me == nil { + ukey, _, kt, kerr := parseIkey(mk) + if kerr != nil { + // Shouldn't have had happen. + panic(kerr) + } + if db.s.icmp.uCompare(ukey, key) == 0 { + if kt == ktDel { + return false, nil + } + return true, nil + } + } else if me != ErrNotFound { + return false, me + } + } + + v := db.s.version() + _, cSched, err := v.get(ikey, ro, true) + v.release() + if cSched { + // Trigger table compaction. + db.compSendTrigger(db.tcompCmdC) + } + if err == nil { + ret = true + } else if err == ErrNotFound { + err = nil } return } // Get gets the value for the given key. It returns ErrNotFound if the -// DB does not contain the key. +// DB does not contains the key. // -// The caller should not modify the contents of the returned slice, but -// it is safe to modify the contents of the argument after Get returns. -func (d *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { - err = d.ok() +// The returned slice is its own copy, it is safe to modify the contents +// of the returned slice. +// It is safe to modify the contents of the argument after Get returns. +func (db *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { + err = db.ok() if err != nil { return } - return d.get(key, d.getSeq(), ro) + se := db.acquireSnapshot() + defer db.releaseSnapshot(se) + return db.get(key, se.seq, ro) +} + +// Has returns true if the DB does contains the given key. +// +// It is safe to modify the contents of the argument after Get returns. +func (db *DB) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) { + err = db.ok() + if err != nil { + return + } + + se := db.acquireSnapshot() + defer db.releaseSnapshot(se) + return db.has(key, se.seq, ro) } // NewIterator returns an iterator for the latest snapshot of the @@ -573,14 +732,16 @@ func (d *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { // The iterator must be released after use, by calling Release method. // // Also read Iterator documentation of the leveldb/iterator package. -func (d *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { - if err := d.ok(); err != nil { +func (db *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { + if err := db.ok(); err != nil { return iterator.NewEmptyIterator(err) } - p := d.newSnapshot() - defer p.Release() - return p.NewIterator(slice, ro) + se := db.acquireSnapshot() + defer db.releaseSnapshot(se) + // Iterator holds 'version' lock, 'version' is immutable so snapshot + // can be released after iterator created. + return db.newIterator(se.seq, slice, ro) } // GetSnapshot returns a latest snapshot of the underlying DB. A snapshot @@ -588,25 +749,35 @@ func (d *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterat // content of snapshot are guaranteed to be consistent. // // The snapshot must be released after use, by calling Release method. -func (d *DB) GetSnapshot() (*Snapshot, error) { - if err := d.ok(); err != nil { +func (db *DB) GetSnapshot() (*Snapshot, error) { + if err := db.ok(); err != nil { return nil, err } - return d.newSnapshot(), nil + return db.newSnapshot(), nil } // GetProperty returns value of the given property name. // // Property names: // leveldb.num-files-at-level{n} -// Returns the number of filer at level 'n'. +// Returns the number of files at level 'n'. // leveldb.stats // Returns statistics of the underlying DB. // leveldb.sstables // Returns sstables list for each level. -func (d *DB) GetProperty(name string) (value string, err error) { - err = d.ok() +// leveldb.blockpool +// Returns block pool stats. +// leveldb.cachedblock +// Returns size of cached block. +// leveldb.openedtables +// Returns number of opened tables. +// leveldb.alivesnaps +// Returns number of alive snapshots. +// leveldb.aliveiters +// Returns number of alive iterators. +func (db *DB) GetProperty(name string) (value string, err error) { + err = db.ok() if err != nil { return } @@ -615,19 +786,18 @@ func (d *DB) GetProperty(name string) (value string, err error) { if !strings.HasPrefix(name, prefix) { return "", errors.New("leveldb: GetProperty: unknown property: " + name) } - p := name[len(prefix):] - s := d.s - v := s.version() + v := db.s.version() defer v.release() + numFilesPrefix := "num-files-at-level" switch { - case strings.HasPrefix(p, "num-files-at-level"): + case strings.HasPrefix(p, numFilesPrefix): var level uint var rest string - n, _ := fmt.Scanf("%d%s", &level, &rest) - if n != 1 || level >= kNumLevels { + n, _ := fmt.Sscanf(p[len(numFilesPrefix):], "%d%s", &level, &rest) + if n != 1 || int(level) >= db.s.o.GetNumLevel() { err = errors.New("leveldb: GetProperty: invalid property: " + name) } else { value = fmt.Sprint(v.tLen(int(level))) @@ -636,22 +806,36 @@ func (d *DB) GetProperty(name string) (value string, err error) { value = "Compactions\n" + " Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB)\n" + "-------+------------+---------------+---------------+---------------+---------------\n" - for level, tt := range v.tables { - duration, read, write := d.compStats[level].get() - if len(tt) == 0 && duration == 0 { + for level, tables := range v.tables { + duration, read, write := db.compStats[level].get() + if len(tables) == 0 && duration == 0 { continue } value += fmt.Sprintf(" %3d | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n", - level, len(tt), float64(tt.size())/1048576.0, duration.Seconds(), + level, len(tables), float64(tables.size())/1048576.0, duration.Seconds(), float64(read)/1048576.0, float64(write)/1048576.0) } case p == "sstables": - for level, tt := range v.tables { + for level, tables := range v.tables { value += fmt.Sprintf("--- level %d ---\n", level) - for _, t := range tt { - value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.file.Num(), t.size, t.min, t.max) + for _, t := range tables { + value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.file.Num(), t.size, t.imin, t.imax) } } + case p == "blockpool": + value = fmt.Sprintf("%v", db.s.tops.bpool) + case p == "cachedblock": + if db.s.tops.bcache != nil { + value = fmt.Sprintf("%d", db.s.tops.bcache.Size()) + } else { + value = "<nil>" + } + case p == "openedtables": + value = fmt.Sprintf("%d", db.s.tops.cache.Size()) + case p == "alivesnaps": + value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveSnaps)) + case p == "aliveiters": + value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveIters)) default: err = errors.New("leveldb: GetProperty: unknown property: " + name) } @@ -665,23 +849,23 @@ func (d *DB) GetProperty(name string) (value string, err error) { // data compresses by a factor of ten, the returned sizes will be one-tenth // the size of the corresponding user data size. // The results may not include the sizes of recently written data. -func (d *DB) SizeOf(ranges []util.Range) (Sizes, error) { - if err := d.ok(); err != nil { +func (db *DB) SizeOf(ranges []util.Range) (Sizes, error) { + if err := db.ok(); err != nil { return nil, err } - v := d.s.version() + v := db.s.version() defer v.release() sizes := make(Sizes, 0, len(ranges)) for _, r := range ranges { - min := newIKey(r.Start, kMaxSeq, tSeek) - max := newIKey(r.Limit, kMaxSeq, tSeek) - start, err := v.offsetOf(min) + imin := newIkey(r.Start, kMaxSeq, ktSeek) + imax := newIkey(r.Limit, kMaxSeq, ktSeek) + start, err := v.offsetOf(imin) if err != nil { return nil, err } - limit, err := v.offsetOf(max) + limit, err := v.offsetOf(imax) if err != nil { return nil, err } @@ -695,61 +879,67 @@ func (d *DB) SizeOf(ranges []util.Range) (Sizes, error) { return sizes, nil } -// Close closes the DB. This will also releases any outstanding snapshot. +// Close closes the DB. This will also releases any outstanding snapshot and +// abort any in-flight compaction. // // It is not safe to close a DB until all outstanding iterators are released. // It is valid to call Close multiple times. Other methods should not be // called after the DB has been closed. -func (d *DB) Close() error { - if !d.setClosed() { +func (db *DB) Close() error { + if !db.setClosed() { return ErrClosed } - s := d.s start := time.Now() - s.log("db@close closing") + db.log("db@close closing") // Clear the finalizer. - runtime.SetFinalizer(d, nil) + runtime.SetFinalizer(db, nil) // Get compaction error. var err error select { - case err = <-d.compErrC: + case err = <-db.compErrC: default: } - close(d.closeC) + // Signal all goroutines. + close(db.closeC) - // Wait for the close WaitGroup. - d.closeW.Wait() + // Wait for all gorotines to exit. + db.closeW.Wait() + + // Lock writer and closes journal. + db.writeLockC <- struct{}{} + if db.journal != nil { + db.journal.Close() + db.journalWriter.Close() + } - // Close journal. - if d.journal != nil { - d.journal.Close() - d.journalWriter.Close() + if db.writeDelayN > 0 { + db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay) } // Close session. - s.close() - s.logf("db@close done T·%v", time.Since(start)) - s.release() + db.s.close() + db.logf("db@close done T·%v", time.Since(start)) + db.s.release() - if d.closer != nil { - if err1 := d.closer.Close(); err == nil { + if db.closer != nil { + if err1 := db.closer.Close(); err == nil { err = err1 } } - d.s = nil - d.mem = nil - d.frozenMem = nil - d.journal = nil - d.journalWriter = nil - d.journalFile = nil - d.frozenJournalFile = nil - d.snapsRoot = snapshotElement{} - d.closer = nil + // NIL'ing pointers. + db.s = nil + db.mem = nil + db.frozenMem = nil + db.journal = nil + db.journalWriter = nil + db.journalFile = nil + db.frozenJournalFile = nil + db.closer = nil return err } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go index c82bd9f28..447407aba 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go @@ -7,11 +7,12 @@ package leveldb import ( - "errors" "sync" "time" + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/memdb" + "github.com/syndtr/goleveldb/leveldb/opt" ) var ( @@ -68,13 +69,13 @@ type cMem struct { } func newCMem(s *session) *cMem { - return &cMem{s: s, rec: new(sessionRecord)} + return &cMem{s: s, rec: &sessionRecord{numLevel: s.o.GetNumLevel()}} } func (c *cMem) flush(mem *memdb.DB, level int) error { s := c.s - // Write memdb to table + // Write memdb to table. iter := mem.NewIterator(nil) defer iter.Release() t, n, err := s.tops.createFrom(iter) @@ -82,51 +83,85 @@ func (c *cMem) flush(mem *memdb.DB, level int) error { return err } + // Pick level. if level < 0 { - level = s.version_NB().pickLevel(t.min.ukey(), t.max.ukey()) + v := s.version() + level = v.pickLevel(t.imin.ukey(), t.imax.ukey()) + v.release() } c.rec.addTableFile(level, t) - s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.min, t.max) + s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.imin, t.imax) c.level = level return nil } func (c *cMem) reset() { - c.rec = new(sessionRecord) + c.rec = &sessionRecord{numLevel: c.s.o.GetNumLevel()} } func (c *cMem) commit(journal, seq uint64) error { c.rec.setJournalNum(journal) - c.rec.setSeq(seq) - // Commit changes + c.rec.setSeqNum(seq) + + // Commit changes. return c.s.commit(c.rec) } -func (d *DB) compactionError() { - var err error +func (db *DB) compactionError() { + var ( + err error + wlocked bool + ) noerr: + // No error. for { select { - case _, _ = <-d.closeC: - return - case err = <-d.compErrSetC: - if err != nil { + case err = <-db.compErrSetC: + switch { + case err == nil: + case errors.IsCorrupted(err): + goto hasperr + default: goto haserr } + case _, _ = <-db.closeC: + return } } haserr: + // Transient error. for { select { - case _, _ = <-d.closeC: - return - case err = <-d.compErrSetC: - if err == nil { + case db.compErrC <- err: + case err = <-db.compErrSetC: + switch { + case err == nil: goto noerr + case errors.IsCorrupted(err): + goto hasperr + default: } - case d.compErrC <- err: + case _, _ = <-db.closeC: + return + } + } +hasperr: + // Persistent error. + for { + select { + case db.compErrC <- err: + case db.compPerErrC <- err: + case db.writeLockC <- struct{}{}: + // Hold write lock, so that write won't pass-through. + wlocked = true + case _, _ = <-db.closeC: + if wlocked { + // We should release the lock or Close will hang. + <-db.writeLockC + } + return } } } @@ -137,114 +172,159 @@ func (cnt *compactionTransactCounter) incr() { *cnt++ } -func (d *DB) compactionTransact(name string, exec func(cnt *compactionTransactCounter) error, rollback func() error) { - s := d.s +type compactionTransactInterface interface { + run(cnt *compactionTransactCounter) error + revert() error +} + +func (db *DB) compactionTransact(name string, t compactionTransactInterface) { defer func() { if x := recover(); x != nil { - if x == errCompactionTransactExiting && rollback != nil { - if err := rollback(); err != nil { - s.logf("%s rollback error %q", name, err) + if x == errCompactionTransactExiting { + if err := t.revert(); err != nil { + db.logf("%s revert error %q", name, err) } } panic(x) } }() + const ( backoffMin = 1 * time.Second backoffMax = 8 * time.Second backoffMul = 2 * time.Second ) - backoff := backoffMin - backoffT := time.NewTimer(backoff) - lastCnt := compactionTransactCounter(0) + var ( + backoff = backoffMin + backoffT = time.NewTimer(backoff) + lastCnt = compactionTransactCounter(0) + + disableBackoff = db.s.o.GetDisableCompactionBackoff() + ) for n := 0; ; n++ { // Check wether the DB is closed. - if d.isClosed() { - s.logf("%s exiting", name) - d.compactionExitTransact() + if db.isClosed() { + db.logf("%s exiting", name) + db.compactionExitTransact() } else if n > 0 { - s.logf("%s retrying N·%d", name, n) + db.logf("%s retrying N·%d", name, n) } // Execute. cnt := compactionTransactCounter(0) - err := exec(&cnt) + err := t.run(&cnt) + if err != nil { + db.logf("%s error I·%d %q", name, cnt, err) + } // Set compaction error status. select { - case d.compErrSetC <- err: - case _, _ = <-d.closeC: - s.logf("%s exiting", name) - d.compactionExitTransact() + case db.compErrSetC <- err: + case perr := <-db.compPerErrC: + if err != nil { + db.logf("%s exiting (persistent error %q)", name, perr) + db.compactionExitTransact() + } + case _, _ = <-db.closeC: + db.logf("%s exiting", name) + db.compactionExitTransact() } if err == nil { return } - s.logf("%s error I·%d %q", name, cnt, err) - - // Reset backoff duration if counter is advancing. - if cnt > lastCnt { - backoff = backoffMin - lastCnt = cnt + if errors.IsCorrupted(err) { + db.logf("%s exiting (corruption detected)", name) + db.compactionExitTransact() } - // Backoff. - backoffT.Reset(backoff) - if backoff < backoffMax { - backoff *= backoffMul - if backoff > backoffMax { - backoff = backoffMax + if !disableBackoff { + // Reset backoff duration if counter is advancing. + if cnt > lastCnt { + backoff = backoffMin + lastCnt = cnt + } + + // Backoff. + backoffT.Reset(backoff) + if backoff < backoffMax { + backoff *= backoffMul + if backoff > backoffMax { + backoff = backoffMax + } + } + select { + case <-backoffT.C: + case _, _ = <-db.closeC: + db.logf("%s exiting", name) + db.compactionExitTransact() } } - select { - case <-backoffT.C: - case _, _ = <-d.closeC: - s.logf("%s exiting", name) - d.compactionExitTransact() - } } } -func (d *DB) compactionExitTransact() { +type compactionTransactFunc struct { + runFunc func(cnt *compactionTransactCounter) error + revertFunc func() error +} + +func (t *compactionTransactFunc) run(cnt *compactionTransactCounter) error { + return t.runFunc(cnt) +} + +func (t *compactionTransactFunc) revert() error { + if t.revertFunc != nil { + return t.revertFunc() + } + return nil +} + +func (db *DB) compactionTransactFunc(name string, run func(cnt *compactionTransactCounter) error, revert func() error) { + db.compactionTransact(name, &compactionTransactFunc{run, revert}) +} + +func (db *DB) compactionExitTransact() { panic(errCompactionTransactExiting) } -func (d *DB) memCompaction() { - mem := d.getFrozenMem() +func (db *DB) memCompaction() { + mem := db.getFrozenMem() if mem == nil { return } + defer mem.decref() - s := d.s - c := newCMem(s) + c := newCMem(db.s) stats := new(cStatsStaging) - s.logf("mem@flush N·%d S·%s", mem.Len(), shortenb(mem.Size())) + db.logf("mem@flush N·%d S·%s", mem.mdb.Len(), shortenb(mem.mdb.Size())) // Don't compact empty memdb. - if mem.Len() == 0 { - s.logf("mem@flush skipping") + if mem.mdb.Len() == 0 { + db.logf("mem@flush skipping") // drop frozen mem - d.dropFrozenMem() + db.dropFrozenMem() return } // Pause table compaction. - ch := make(chan struct{}) + resumeC := make(chan struct{}) select { - case d.tcompPauseC <- (chan<- struct{})(ch): - case _, _ = <-d.closeC: + case db.tcompPauseC <- (chan<- struct{})(resumeC): + case <-db.compPerErrC: + close(resumeC) + resumeC = nil + case _, _ = <-db.closeC: return } - d.compactionTransact("mem@flush", func(cnt *compactionTransactCounter) (err error) { + db.compactionTransactFunc("mem@flush", func(cnt *compactionTransactCounter) (err error) { stats.startTimer() defer stats.stopTimer() - return c.flush(mem, -1) + return c.flush(mem.mdb, -1) }, func() error { for _, r := range c.rec.addedTables { - s.logf("mem@flush rollback @%d", r.num) - f := s.getTableFile(r.num) + db.logf("mem@flush revert @%d", r.num) + f := db.s.getTableFile(r.num) if err := f.Remove(); err != nil { return err } @@ -252,279 +332,327 @@ func (d *DB) memCompaction() { return nil }) - d.compactionTransact("mem@commit", func(cnt *compactionTransactCounter) (err error) { + db.compactionTransactFunc("mem@commit", func(cnt *compactionTransactCounter) (err error) { stats.startTimer() defer stats.stopTimer() - return c.commit(d.journalFile.Num(), d.frozenSeq) + return c.commit(db.journalFile.Num(), db.frozenSeq) }, nil) - s.logf("mem@flush commited F·%d T·%v", len(c.rec.addedTables), stats.duration) + db.logf("mem@flush committed F·%d T·%v", len(c.rec.addedTables), stats.duration) for _, r := range c.rec.addedTables { stats.write += r.size } - d.compStats[c.level].add(stats) + db.compStats[c.level].add(stats) // Drop frozen mem. - d.dropFrozenMem() + db.dropFrozenMem() // Resume table compaction. - select { - case <-ch: - case _, _ = <-d.closeC: - return + if resumeC != nil { + select { + case <-resumeC: + close(resumeC) + case _, _ = <-db.closeC: + return + } } // Trigger table compaction. - d.compTrigger(d.mcompTriggerC) + db.compSendTrigger(db.tcompCmdC) } -func (d *DB) tableCompaction(c *compaction, noTrivial bool) { - s := d.s +type tableCompactionBuilder struct { + db *DB + s *session + c *compaction + rec *sessionRecord + stat0, stat1 *cStatsStaging - rec := new(sessionRecord) - rec.addCompactionPointer(c.level, c.max) + snapHasLastUkey bool + snapLastUkey []byte + snapLastSeq uint64 + snapIter int + snapKerrCnt int + snapDropCnt int - if !noTrivial && c.trivial() { - t := c.tables[0][0] - s.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1) - rec.deleteTable(c.level, t.file.Num()) - rec.addTableFile(c.level+1, t) - d.compactionTransact("table@move", func(cnt *compactionTransactCounter) (err error) { - return s.commit(rec) - }, nil) - return - } + kerrCnt int + dropCnt int - var stats [2]cStatsStaging - for i, tt := range c.tables { - for _, t := range tt { - stats[i].read += t.size - // Insert deleted tables into record - rec.deleteTable(c.level+i, t.file.Num()) - } - } - sourceSize := int(stats[0].read + stats[1].read) - minSeq := d.minSeq() - s.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq) - - var snapUkey []byte - var snapHasUkey bool - var snapSeq uint64 - var snapIter int - var snapDropCnt int - var dropCnt int - d.compactionTransact("table@build", func(cnt *compactionTransactCounter) (err error) { - ukey := append([]byte{}, snapUkey...) - hasUkey := snapHasUkey - lseq := snapSeq - dropCnt = snapDropCnt - snapSched := snapIter == 0 - - var tw *tWriter - finish := func() error { - t, err := tw.finish() - if err != nil { - return err + minSeq uint64 + strict bool + tableSize int + + tw *tWriter +} + +func (b *tableCompactionBuilder) appendKV(key, value []byte) error { + // Create new table if not already. + if b.tw == nil { + // Check for pause event. + if b.db != nil { + select { + case ch := <-b.db.tcompPauseC: + b.db.pauseCompaction(ch) + case _, _ = <-b.db.closeC: + b.db.compactionExitTransact() + default: } - rec.addTableFile(c.level+1, t) - stats[1].write += t.size - s.logf("table@build created L%d@%d N·%d S·%s %q:%q", c.level+1, t.file.Num(), tw.tw.EntriesLen(), shortenb(int(t.size)), t.min, t.max) - return nil } - defer func() { - stats[1].stopTimer() - if tw != nil { - tw.drop() - tw = nil - } - }() + // Create new table. + var err error + b.tw, err = b.s.tops.create() + if err != nil { + return err + } + } - stats[1].startTimer() - iter := c.newIterator() - defer iter.Release() - for i := 0; iter.Next(); i++ { - // Incr transact counter. - cnt.incr() - - // Skip until last state. - if i < snapIter { - continue - } + // Write key/value into table. + return b.tw.append(key, value) +} - key := iKey(iter.Key()) +func (b *tableCompactionBuilder) needFlush() bool { + return b.tw.tw.BytesLen() >= b.tableSize +} - if c.shouldStopBefore(key) && tw != nil { - err = finish() - if err != nil { - return - } - snapSched = true - tw = nil - } +func (b *tableCompactionBuilder) flush() error { + t, err := b.tw.finish() + if err != nil { + return err + } + b.rec.addTableFile(b.c.level+1, t) + b.stat1.write += t.size + b.s.logf("table@build created L%d@%d N·%d S·%s %q:%q", b.c.level+1, t.file.Num(), b.tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax) + b.tw = nil + return nil +} - // Scheduled for snapshot, snapshot will used to retry compaction - // if error occured. - if snapSched { - snapUkey = append(snapUkey[:0], ukey...) - snapHasUkey = hasUkey - snapSeq = lseq - snapIter = i - snapDropCnt = dropCnt - snapSched = false - } +func (b *tableCompactionBuilder) cleanup() { + if b.tw != nil { + b.tw.drop() + b.tw = nil + } +} - if seq, t, ok := key.parseNum(); !ok { - // Don't drop error keys - ukey = ukey[:0] - hasUkey = false - lseq = kMaxSeq - } else { - if !hasUkey || s.icmp.uCompare(key.ukey(), ukey) != 0 { - // First occurrence of this user key - ukey = append(ukey[:0], key.ukey()...) - hasUkey = true - lseq = kMaxSeq - } +func (b *tableCompactionBuilder) run(cnt *compactionTransactCounter) error { + snapResumed := b.snapIter > 0 + hasLastUkey := b.snapHasLastUkey // The key might has zero length, so this is necessary. + lastUkey := append([]byte{}, b.snapLastUkey...) + lastSeq := b.snapLastSeq + b.kerrCnt = b.snapKerrCnt + b.dropCnt = b.snapDropCnt + // Restore compaction state. + b.c.restore() - drop := false - if lseq <= minSeq { - // Dropped because newer entry for same user key exist - drop = true // (A) - } else if t == tDel && seq <= minSeq && c.isBaseLevelForKey(ukey) { - // For this user key: - // (1) there is no data in higher levels - // (2) data in lower levels will have larger seq numbers - // (3) data in layers that are being compacted here and have - // smaller seq numbers will be dropped in the next - // few iterations of this loop (by rule (A) above). - // Therefore this deletion marker is obsolete and can be dropped. - drop = true - } + defer b.cleanup() - lseq = seq - if drop { - dropCnt++ - continue - } - } + b.stat1.startTimer() + defer b.stat1.stopTimer() - // Create new table if not already - if tw == nil { - // Check for pause event. - select { - case ch := <-d.tcompPauseC: - d.pauseCompaction(ch) - case _, _ = <-d.closeC: - d.compactionExitTransact() - default: - } + iter := b.c.newIterator() + defer iter.Release() + for i := 0; iter.Next(); i++ { + // Incr transact counter. + cnt.incr() + + // Skip until last state. + if i < b.snapIter { + continue + } - // Create new table. - tw, err = s.tops.create() - if err != nil { - return + resumed := false + if snapResumed { + resumed = true + snapResumed = false + } + + ikey := iter.Key() + ukey, seq, kt, kerr := parseIkey(ikey) + + if kerr == nil { + shouldStop := !resumed && b.c.shouldStopBefore(ikey) + + if !hasLastUkey || b.s.icmp.uCompare(lastUkey, ukey) != 0 { + // First occurrence of this user key. + + // Only rotate tables if ukey doesn't hop across. + if b.tw != nil && (shouldStop || b.needFlush()) { + if err := b.flush(); err != nil { + return err + } + + // Creates snapshot of the state. + b.c.save() + b.snapHasLastUkey = hasLastUkey + b.snapLastUkey = append(b.snapLastUkey[:0], lastUkey...) + b.snapLastSeq = lastSeq + b.snapIter = i + b.snapKerrCnt = b.kerrCnt + b.snapDropCnt = b.dropCnt } - } - // Write key/value into table - err = tw.add(key, iter.Value()) - if err != nil { - return + hasLastUkey = true + lastUkey = append(lastUkey[:0], ukey...) + lastSeq = kMaxSeq } - // Finish table if it is big enough - if tw.tw.BytesLen() >= kMaxTableSize { - err = finish() - if err != nil { - return - } - snapSched = true - tw = nil + switch { + case lastSeq <= b.minSeq: + // Dropped because newer entry for same user key exist + fallthrough // (A) + case kt == ktDel && seq <= b.minSeq && b.c.baseLevelForKey(lastUkey): + // For this user key: + // (1) there is no data in higher levels + // (2) data in lower levels will have larger seq numbers + // (3) data in layers that are being compacted here and have + // smaller seq numbers will be dropped in the next + // few iterations of this loop (by rule (A) above). + // Therefore this deletion marker is obsolete and can be dropped. + lastSeq = seq + b.dropCnt++ + continue + default: + lastSeq = seq + } + } else { + if b.strict { + return kerr } + + // Don't drop corrupted keys. + hasLastUkey = false + lastUkey = lastUkey[:0] + lastSeq = kMaxSeq + b.kerrCnt++ } - err = iter.Error() - if err != nil { - return + if err := b.appendKV(ikey, iter.Value()); err != nil { + return err } + } - // Finish last table - if tw != nil && !tw.empty() { - err = finish() - if err != nil { - return - } - tw = nil + if err := iter.Error(); err != nil { + return err + } + + // Finish last table. + if b.tw != nil && !b.tw.empty() { + return b.flush() + } + return nil +} + +func (b *tableCompactionBuilder) revert() error { + for _, at := range b.rec.addedTables { + b.s.logf("table@build revert @%d", at.num) + f := b.s.getTableFile(at.num) + if err := f.Remove(); err != nil { + return err } + } + return nil +} + +func (db *DB) tableCompaction(c *compaction, noTrivial bool) { + defer c.release() + + rec := &sessionRecord{numLevel: db.s.o.GetNumLevel()} + rec.addCompPtr(c.level, c.imax) + + if !noTrivial && c.trivial() { + t := c.tables[0][0] + db.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1) + rec.delTable(c.level, t.file.Num()) + rec.addTableFile(c.level+1, t) + db.compactionTransactFunc("table@move", func(cnt *compactionTransactCounter) (err error) { + return db.s.commit(rec) + }, nil) return - }, func() error { - for _, r := range rec.addedTables { - s.logf("table@build rollback @%d", r.num) - f := s.getTableFile(r.num) - if err := f.Remove(); err != nil { - return err - } + } + + var stats [2]cStatsStaging + for i, tables := range c.tables { + for _, t := range tables { + stats[i].read += t.size + // Insert deleted tables into record + rec.delTable(c.level+i, t.file.Num()) } - return nil - }) + } + sourceSize := int(stats[0].read + stats[1].read) + minSeq := db.minSeq() + db.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq) + + b := &tableCompactionBuilder{ + db: db, + s: db.s, + c: c, + rec: rec, + stat1: &stats[1], + minSeq: minSeq, + strict: db.s.o.GetStrict(opt.StrictCompaction), + tableSize: db.s.o.GetCompactionTableSize(c.level + 1), + } + db.compactionTransact("table@build", b) // Commit changes - d.compactionTransact("table@commit", func(cnt *compactionTransactCounter) (err error) { + db.compactionTransactFunc("table@commit", func(cnt *compactionTransactCounter) (err error) { stats[1].startTimer() defer stats[1].stopTimer() - return s.commit(rec) + return db.s.commit(rec) }, nil) - resultSize := int(int(stats[1].write)) - s.logf("table@compaction commited F%s S%s D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), dropCnt, stats[1].duration) + resultSize := int(stats[1].write) + db.logf("table@compaction committed F%s S%s Ke·%d D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), b.kerrCnt, b.dropCnt, stats[1].duration) // Save compaction stats for i := range stats { - d.compStats[c.level+1].add(&stats[i]) + db.compStats[c.level+1].add(&stats[i]) } } -func (d *DB) tableRangeCompaction(level int, min, max []byte) { - s := d.s - s.logf("table@compaction range L%d %q:%q", level, min, max) +func (db *DB) tableRangeCompaction(level int, umin, umax []byte) { + db.logf("table@compaction range L%d %q:%q", level, umin, umax) if level >= 0 { - if c := s.getCompactionRange(level, min, max); c != nil { - d.tableCompaction(c, true) + if c := db.s.getCompactionRange(level, umin, umax); c != nil { + db.tableCompaction(c, true) } } else { - v := s.version_NB() + v := db.s.version() m := 1 for i, t := range v.tables[1:] { - if t.isOverlaps(min, max, true, s.icmp) { + if t.overlaps(db.s.icmp, umin, umax, false) { m = i + 1 } } + v.release() + for level := 0; level < m; level++ { - if c := s.getCompactionRange(level, min, max); c != nil { - d.tableCompaction(c, true) + if c := db.s.getCompactionRange(level, umin, umax); c != nil { + db.tableCompaction(c, true) } } } } -func (d *DB) tableAutoCompaction() { - if c := d.s.pickCompaction(); c != nil { - d.tableCompaction(c, false) +func (db *DB) tableAutoCompaction() { + if c := db.s.pickCompaction(); c != nil { + db.tableCompaction(c, false) } } -func (d *DB) tableNeedCompaction() bool { - return d.s.version_NB().needCompaction() +func (db *DB) tableNeedCompaction() bool { + v := db.s.version() + defer v.release() + return v.needCompaction() } -func (d *DB) pauseCompaction(ch chan<- struct{}) { +func (db *DB) pauseCompaction(ch chan<- struct{}) { select { case ch <- struct{}{}: - case _, _ = <-d.closeC: - d.compactionExitTransact() + case _, _ = <-db.closeC: + db.compactionExitTransact() } } @@ -537,7 +665,12 @@ type cIdle struct { } func (r cIdle) ack(err error) { - r.ackC <- err + if r.ackC != nil { + defer func() { + recover() + }() + r.ackC <- err + } } type cRange struct { @@ -547,56 +680,67 @@ type cRange struct { } func (r cRange) ack(err error) { - defer func() { - recover() - }() if r.ackC != nil { + defer func() { + recover() + }() r.ackC <- err } } -func (d *DB) compSendIdle(compC chan<- cCmd) error { +// This will trigger auto compation and/or wait for all compaction to be done. +func (db *DB) compSendIdle(compC chan<- cCmd) (err error) { ch := make(chan error) defer close(ch) // Send cmd. select { case compC <- cIdle{ch}: - case err := <-d.compErrC: - return err - case _, _ = <-d.closeC: + case err = <-db.compErrC: + return + case _, _ = <-db.closeC: return ErrClosed } // Wait cmd. - return <-ch + select { + case err = <-ch: + case err = <-db.compErrC: + case _, _ = <-db.closeC: + return ErrClosed + } + return err } -func (d *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) { +// This will trigger auto compaction but will not wait for it. +func (db *DB) compSendTrigger(compC chan<- cCmd) { + select { + case compC <- cIdle{}: + default: + } +} + +// Send range compaction request. +func (db *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) { ch := make(chan error) defer close(ch) // Send cmd. select { case compC <- cRange{level, min, max, ch}: - case err := <-d.compErrC: + case err := <-db.compErrC: return err - case _, _ = <-d.closeC: + case _, _ = <-db.closeC: return ErrClosed } // Wait cmd. select { - case err = <-d.compErrC: case err = <-ch: + case err = <-db.compErrC: + case _, _ = <-db.closeC: + return ErrClosed } return err } -func (d *DB) compTrigger(compTriggerC chan struct{}) { - select { - case compTriggerC <- struct{}{}: - default: - } -} - -func (d *DB) mCompaction() { +func (db *DB) mCompaction() { var x cCmd defer func() { @@ -608,24 +752,27 @@ func (d *DB) mCompaction() { if x != nil { x.ack(ErrClosed) } - d.closeW.Done() + db.closeW.Done() }() for { select { - case _, _ = <-d.closeC: + case x = <-db.mcompCmdC: + switch x.(type) { + case cIdle: + db.memCompaction() + x.ack(nil) + x = nil + default: + panic("leveldb: unknown command") + } + case _, _ = <-db.closeC: return - case x = <-d.mcompCmdC: - d.memCompaction() - x.ack(nil) - x = nil - case <-d.mcompTriggerC: - d.memCompaction() } } } -func (d *DB) tCompaction() { +func (db *DB) tCompaction() { var x cCmd var ackQ []cCmd @@ -642,19 +789,18 @@ func (d *DB) tCompaction() { if x != nil { x.ack(ErrClosed) } - d.closeW.Done() + db.closeW.Done() }() for { - if d.tableNeedCompaction() { + if db.tableNeedCompaction() { select { - case x = <-d.tcompCmdC: - case <-d.tcompTriggerC: - case _, _ = <-d.closeC: - return - case ch := <-d.tcompPauseC: - d.pauseCompaction(ch) + case x = <-db.tcompCmdC: + case ch := <-db.tcompPauseC: + db.pauseCompaction(ch) continue + case _, _ = <-db.closeC: + return default: } } else { @@ -664,12 +810,11 @@ func (d *DB) tCompaction() { } ackQ = ackQ[:0] select { - case x = <-d.tcompCmdC: - case <-d.tcompTriggerC: - case ch := <-d.tcompPauseC: - d.pauseCompaction(ch) + case x = <-db.tcompCmdC: + case ch := <-db.tcompPauseC: + db.pauseCompaction(ch) continue - case _, _ = <-d.closeC: + case _, _ = <-db.closeC: return } } @@ -678,11 +823,13 @@ func (d *DB) tCompaction() { case cIdle: ackQ = append(ackQ, x) case cRange: - d.tableRangeCompaction(cmd.level, cmd.min, cmd.max) + db.tableRangeCompaction(cmd.level, cmd.min, cmd.max) x.ack(nil) + default: + panic("leveldb: unknown command") } x = nil } - d.tableAutoCompaction() + db.tableAutoCompaction() } } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go index 9973a8fef..011a94a35 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go @@ -8,7 +8,10 @@ package leveldb import ( "errors" + "math/rand" "runtime" + "sync" + "sync/atomic" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" @@ -19,50 +22,69 @@ var ( errInvalidIkey = errors.New("leveldb: Iterator: invalid internal key") ) -func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { - s := db.s +type memdbReleaser struct { + once sync.Once + m *memDB +} + +func (mr *memdbReleaser) Release() { + mr.once.Do(func() { + mr.m.decref() + }) +} +func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { em, fm := db.getMems() - v := s.version() + v := db.s.version() ti := v.getIterators(slice, ro) n := len(ti) + 2 i := make([]iterator.Iterator, 0, n) - i = append(i, em.NewIterator(slice)) + emi := em.mdb.NewIterator(slice) + emi.SetReleaser(&memdbReleaser{m: em}) + i = append(i, emi) if fm != nil { - i = append(i, fm.NewIterator(slice)) + fmi := fm.mdb.NewIterator(slice) + fmi.SetReleaser(&memdbReleaser{m: fm}) + i = append(i, fmi) } i = append(i, ti...) - strict := s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator) - mi := iterator.NewMergedIterator(i, s.icmp, strict) + strict := opt.GetStrict(db.s.o.Options, ro, opt.StrictReader) + mi := iterator.NewMergedIterator(i, db.s.icmp, strict) mi.SetReleaser(&versionReleaser{v: v}) return mi } func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *dbIter { - var slice_ *util.Range + var islice *util.Range if slice != nil { - slice_ = &util.Range{} + islice = &util.Range{} if slice.Start != nil { - slice_.Start = newIKey(slice.Start, kMaxSeq, tSeek) + islice.Start = newIkey(slice.Start, kMaxSeq, ktSeek) } if slice.Limit != nil { - slice_.Limit = newIKey(slice.Limit, kMaxSeq, tSeek) + islice.Limit = newIkey(slice.Limit, kMaxSeq, ktSeek) } } - rawIter := db.newRawIterator(slice_, ro) + rawIter := db.newRawIterator(islice, ro) iter := &dbIter{ + db: db, icmp: db.s.icmp, iter: rawIter, seq: seq, - strict: db.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator), + strict: opt.GetStrict(db.s.o.Options, ro, opt.StrictReader), key: make([]byte, 0), value: make([]byte, 0), } + atomic.AddInt32(&db.aliveIters, 1) runtime.SetFinalizer(iter, (*dbIter).Release) return iter } +func (db *DB) iterSamplingRate() int { + return rand.Intn(2 * db.s.o.GetIteratorSamplingRate()) +} + type dir int const ( @@ -75,16 +97,27 @@ const ( // dbIter represent an interator states over a database session. type dbIter struct { + db *DB icmp *iComparer iter iterator.Iterator seq uint64 strict bool - dir dir - key []byte - value []byte - err error - releaser util.Releaser + smaplingGap int + dir dir + key []byte + value []byte + err error + releaser util.Releaser +} + +func (i *dbIter) sampleSeek() { + ikey := i.iter.Key() + i.smaplingGap -= len(ikey) + len(i.iter.Value()) + for i.smaplingGap < 0 { + i.smaplingGap += i.db.iterSamplingRate() + i.db.sampleSeek(ikey) + } } func (i *dbIter) setErr(err error) { @@ -144,7 +177,7 @@ func (i *dbIter) Seek(key []byte) bool { return false } - ikey := newIKey(key, i.seq, tSeek) + ikey := newIkey(key, i.seq, ktSeek) if i.iter.Seek(ikey) { i.dir = dirSOI return i.next() @@ -156,15 +189,15 @@ func (i *dbIter) Seek(key []byte) bool { func (i *dbIter) next() bool { for { - ukey, seq, t, ok := parseIkey(i.iter.Key()) - if ok { + if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil { + i.sampleSeek() if seq <= i.seq { - switch t { - case tDel: + switch kt { + case ktDel: // Skip deleted key. i.key = append(i.key[:0], ukey...) i.dir = dirForward - case tVal: + case ktVal: if i.dir == dirSOI || i.icmp.uCompare(ukey, i.key) > 0 { i.key = append(i.key[:0], ukey...) i.value = append(i.value[:0], i.iter.Value()...) @@ -174,7 +207,7 @@ func (i *dbIter) next() bool { } } } else if i.strict { - i.setErr(errInvalidIkey) + i.setErr(kerr) break } if !i.iter.Next() { @@ -207,20 +240,20 @@ func (i *dbIter) prev() bool { del := true if i.iter.Valid() { for { - ukey, seq, t, ok := parseIkey(i.iter.Key()) - if ok { + if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil { + i.sampleSeek() if seq <= i.seq { if !del && i.icmp.uCompare(ukey, i.key) < 0 { return true } - del = (t == tDel) + del = (kt == ktDel) if !del { i.key = append(i.key[:0], ukey...) i.value = append(i.value[:0], i.iter.Value()...) } } } else if i.strict { - i.setErr(errInvalidIkey) + i.setErr(kerr) return false } if !i.iter.Prev() { @@ -249,13 +282,13 @@ func (i *dbIter) Prev() bool { return i.Last() case dirForward: for i.iter.Prev() { - ukey, _, _, ok := parseIkey(i.iter.Key()) - if ok { + if ukey, _, _, kerr := parseIkey(i.iter.Key()); kerr == nil { + i.sampleSeek() if i.icmp.uCompare(ukey, i.key) < 0 { goto cont } } else if i.strict { - i.setErr(errInvalidIkey) + i.setErr(kerr) return false } } @@ -289,6 +322,7 @@ func (i *dbIter) Release() { if i.releaser != nil { i.releaser.Release() + i.releaser = nil } i.dir = dirReleased @@ -296,13 +330,19 @@ func (i *dbIter) Release() { i.value = nil i.iter.Release() i.iter = nil + atomic.AddInt32(&i.db.aliveIters, -1) + i.db = nil } } func (i *dbIter) SetReleaser(releaser util.Releaser) { - if i.dir != dirReleased { - i.releaser = releaser + if i.dir == dirReleased { + panic(util.ErrReleased) + } + if i.releaser != nil && releaser != nil { + panic(util.ErrHasReleaser) } + i.releaser = releaser } func (i *dbIter) Error() error { diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_snapshot.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_snapshot.go index 225b7cd5e..0372848ff 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_snapshot.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_snapshot.go @@ -7,8 +7,11 @@ package leveldb import ( + "container/list" + "fmt" "runtime" "sync" + "sync/atomic" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" @@ -18,51 +21,41 @@ import ( type snapshotElement struct { seq uint64 ref int - // Next and previous pointers in the doubly-linked list of elements. - next, prev *snapshotElement -} - -// Initialize the snapshot. -func (db *DB) initSnapshot() { - db.snapsRoot.next = &db.snapsRoot - db.snapsRoot.prev = &db.snapsRoot + e *list.Element } // Acquires a snapshot, based on latest sequence. func (db *DB) acquireSnapshot() *snapshotElement { db.snapsMu.Lock() + defer db.snapsMu.Unlock() + seq := db.getSeq() - elem := db.snapsRoot.prev - if elem == &db.snapsRoot || elem.seq != seq { - at := db.snapsRoot.prev - next := at.next - elem = &snapshotElement{ - seq: seq, - prev: at, - next: next, + + if e := db.snapsList.Back(); e != nil { + se := e.Value.(*snapshotElement) + if se.seq == seq { + se.ref++ + return se + } else if seq < se.seq { + panic("leveldb: sequence number is not increasing") } - at.next = elem - next.prev = elem } - elem.ref++ - db.snapsMu.Unlock() - return elem + se := &snapshotElement{seq: seq, ref: 1} + se.e = db.snapsList.PushBack(se) + return se } // Releases given snapshot element. -func (db *DB) releaseSnapshot(elem *snapshotElement) { - if !db.isClosed() { - db.snapsMu.Lock() - elem.ref-- - if elem.ref == 0 { - elem.prev.next = elem.next - elem.next.prev = elem.prev - elem.next = nil - elem.prev = nil - } else if elem.ref < 0 { - panic("leveldb: Snapshot: negative element reference") - } - db.snapsMu.Unlock() +func (db *DB) releaseSnapshot(se *snapshotElement) { + db.snapsMu.Lock() + defer db.snapsMu.Unlock() + + se.ref-- + if se.ref == 0 { + db.snapsList.Remove(se.e) + se.e = nil + } else if se.ref < 0 { + panic("leveldb: Snapshot: negative element reference") } } @@ -70,10 +63,11 @@ func (db *DB) releaseSnapshot(elem *snapshotElement) { func (db *DB) minSeq() uint64 { db.snapsMu.Lock() defer db.snapsMu.Unlock() - elem := db.snapsRoot.prev - if elem != &db.snapsRoot { - return elem.seq + + if e := db.snapsList.Front(); e != nil { + return e.Value.(*snapshotElement).seq } + return db.getSeq() } @@ -81,38 +75,59 @@ func (db *DB) minSeq() uint64 { type Snapshot struct { db *DB elem *snapshotElement - mu sync.Mutex + mu sync.RWMutex released bool } // Creates new snapshot object. func (db *DB) newSnapshot() *Snapshot { - p := &Snapshot{ + snap := &Snapshot{ db: db, elem: db.acquireSnapshot(), } - runtime.SetFinalizer(p, (*Snapshot).Release) - return p + atomic.AddInt32(&db.aliveSnaps, 1) + runtime.SetFinalizer(snap, (*Snapshot).Release) + return snap +} + +func (snap *Snapshot) String() string { + return fmt.Sprintf("leveldb.Snapshot{%d}", snap.elem.seq) } // Get gets the value for the given key. It returns ErrNotFound if -// the DB does not contain the key. +// the DB does not contains the key. // // The caller should not modify the contents of the returned slice, but // it is safe to modify the contents of the argument after Get returns. -func (p *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { - db := p.db - err = db.ok() +func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { + err = snap.db.ok() if err != nil { return } - p.mu.Lock() - defer p.mu.Unlock() - if p.released { + snap.mu.RLock() + defer snap.mu.RUnlock() + if snap.released { err = ErrSnapshotReleased return } - return db.get(key, p.elem.seq, ro) + return snap.db.get(key, snap.elem.seq, ro) +} + +// Has returns true if the DB does contains the given key. +// +// It is safe to modify the contents of the argument after Get returns. +func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) { + err = snap.db.ok() + if err != nil { + return + } + snap.mu.RLock() + defer snap.mu.RUnlock() + if snap.released { + err = ErrSnapshotReleased + return + } + return snap.db.has(key, snap.elem.seq, ro) } // NewIterator returns an iterator for the snapshot of the uderlying DB. @@ -132,17 +147,18 @@ func (p *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error // iterator would be still valid until released. // // Also read Iterator documentation of the leveldb/iterator package. -func (p *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { - db := p.db - if err := db.ok(); err != nil { +func (snap *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { + if err := snap.db.ok(); err != nil { return iterator.NewEmptyIterator(err) } - p.mu.Lock() - defer p.mu.Unlock() - if p.released { + snap.mu.Lock() + defer snap.mu.Unlock() + if snap.released { return iterator.NewEmptyIterator(ErrSnapshotReleased) } - return db.newIterator(p.elem.seq, slice, ro) + // Since iterator already hold version ref, it doesn't need to + // hold snapshot ref. + return snap.db.newIterator(snap.elem.seq, slice, ro) } // Release releases the snapshot. This will not release any returned @@ -150,16 +166,18 @@ func (p *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator. // underlying DB is closed. // // Other methods should not be called after the snapshot has been released. -func (p *Snapshot) Release() { - p.mu.Lock() - if !p.released { +func (snap *Snapshot) Release() { + snap.mu.Lock() + defer snap.mu.Unlock() + + if !snap.released { // Clear the finalizer. - runtime.SetFinalizer(p, nil) + runtime.SetFinalizer(snap, nil) - p.released = true - p.db.releaseSnapshot(p.elem) - p.db = nil - p.elem = nil + snap.released = true + snap.db.releaseSnapshot(snap.elem) + atomic.AddInt32(&snap.db.aliveSnaps, -1) + snap.db = nil + snap.elem = nil } - p.mu.Unlock() } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go index a13706142..d4db9d6dd 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go @@ -8,106 +8,203 @@ package leveldb import ( "sync/atomic" + "time" "github.com/syndtr/goleveldb/leveldb/journal" "github.com/syndtr/goleveldb/leveldb/memdb" ) +type memDB struct { + db *DB + mdb *memdb.DB + ref int32 +} + +func (m *memDB) incref() { + atomic.AddInt32(&m.ref, 1) +} + +func (m *memDB) decref() { + if ref := atomic.AddInt32(&m.ref, -1); ref == 0 { + // Only put back memdb with std capacity. + if m.mdb.Capacity() == m.db.s.o.GetWriteBuffer() { + m.mdb.Reset() + m.db.mpoolPut(m.mdb) + } + m.db = nil + m.mdb = nil + } else if ref < 0 { + panic("negative memdb ref") + } +} + // Get latest sequence number. -func (d *DB) getSeq() uint64 { - return atomic.LoadUint64(&d.seq) +func (db *DB) getSeq() uint64 { + return atomic.LoadUint64(&db.seq) } // Atomically adds delta to seq. -func (d *DB) addSeq(delta uint64) { - atomic.AddUint64(&d.seq, delta) +func (db *DB) addSeq(delta uint64) { + atomic.AddUint64(&db.seq, delta) +} + +func (db *DB) sampleSeek(ikey iKey) { + v := db.s.version() + if v.sampleSeek(ikey) { + // Trigger table compaction. + db.compSendTrigger(db.tcompCmdC) + } + v.release() +} + +func (db *DB) mpoolPut(mem *memdb.DB) { + defer func() { + recover() + }() + select { + case db.memPool <- mem: + default: + } +} + +func (db *DB) mpoolGet() *memdb.DB { + select { + case mem := <-db.memPool: + return mem + default: + return nil + } +} + +func (db *DB) mpoolDrain() { + ticker := time.NewTicker(30 * time.Second) + for { + select { + case <-ticker.C: + select { + case <-db.memPool: + default: + } + case _, _ = <-db.closeC: + close(db.memPool) + return + } + } } // Create new memdb and froze the old one; need external synchronization. // newMem only called synchronously by the writer. -func (d *DB) newMem(n int) (mem *memdb.DB, err error) { - s := d.s - - num := s.allocFileNum() - file := s.getJournalFile(num) +func (db *DB) newMem(n int) (mem *memDB, err error) { + num := db.s.allocFileNum() + file := db.s.getJournalFile(num) w, err := file.Create() if err != nil { - s.reuseFileNum(num) + db.s.reuseFileNum(num) return } - d.memMu.Lock() - if d.journal == nil { - d.journal = journal.NewWriter(w) + + db.memMu.Lock() + defer db.memMu.Unlock() + + if db.frozenMem != nil { + panic("still has frozen mem") + } + + if db.journal == nil { + db.journal = journal.NewWriter(w) } else { - d.journal.Reset(w) - d.journalWriter.Close() - d.frozenJournalFile = d.journalFile - } - d.journalWriter = w - d.journalFile = file - d.frozenMem = d.mem - d.mem = memdb.New(s.icmp, maxInt(d.s.o.GetWriteBuffer(), n)) - mem = d.mem - // The seq only incremented by the writer. - d.frozenSeq = d.seq - d.memMu.Unlock() + db.journal.Reset(w) + db.journalWriter.Close() + db.frozenJournalFile = db.journalFile + } + db.journalWriter = w + db.journalFile = file + db.frozenMem = db.mem + mdb := db.mpoolGet() + if mdb == nil || mdb.Capacity() < n { + mdb = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n)) + } + mem = &memDB{ + db: db, + mdb: mdb, + ref: 2, + } + db.mem = mem + // The seq only incremented by the writer. And whoever called newMem + // should hold write lock, so no need additional synchronization here. + db.frozenSeq = db.seq return } // Get all memdbs. -func (d *DB) getMems() (e *memdb.DB, f *memdb.DB) { - d.memMu.RLock() - defer d.memMu.RUnlock() - return d.mem, d.frozenMem +func (db *DB) getMems() (e, f *memDB) { + db.memMu.RLock() + defer db.memMu.RUnlock() + if db.mem == nil { + panic("nil effective mem") + } + db.mem.incref() + if db.frozenMem != nil { + db.frozenMem.incref() + } + return db.mem, db.frozenMem } // Get frozen memdb. -func (d *DB) getEffectiveMem() *memdb.DB { - d.memMu.RLock() - defer d.memMu.RUnlock() - return d.mem +func (db *DB) getEffectiveMem() *memDB { + db.memMu.RLock() + defer db.memMu.RUnlock() + if db.mem == nil { + panic("nil effective mem") + } + db.mem.incref() + return db.mem } // Check whether we has frozen memdb. -func (d *DB) hasFrozenMem() bool { - d.memMu.RLock() - defer d.memMu.RUnlock() - return d.frozenMem != nil +func (db *DB) hasFrozenMem() bool { + db.memMu.RLock() + defer db.memMu.RUnlock() + return db.frozenMem != nil } // Get frozen memdb. -func (d *DB) getFrozenMem() *memdb.DB { - d.memMu.RLock() - defer d.memMu.RUnlock() - return d.frozenMem +func (db *DB) getFrozenMem() *memDB { + db.memMu.RLock() + defer db.memMu.RUnlock() + if db.frozenMem != nil { + db.frozenMem.incref() + } + return db.frozenMem } // Drop frozen memdb; assume that frozen memdb isn't nil. -func (d *DB) dropFrozenMem() { - d.memMu.Lock() - if err := d.frozenJournalFile.Remove(); err != nil { - d.s.logf("journal@remove removing @%d %q", d.frozenJournalFile.Num(), err) +func (db *DB) dropFrozenMem() { + db.memMu.Lock() + if err := db.frozenJournalFile.Remove(); err != nil { + db.logf("journal@remove removing @%d %q", db.frozenJournalFile.Num(), err) } else { - d.s.logf("journal@remove removed @%d", d.frozenJournalFile.Num()) + db.logf("journal@remove removed @%d", db.frozenJournalFile.Num()) } - d.frozenJournalFile = nil - d.frozenMem = nil - d.memMu.Unlock() + db.frozenJournalFile = nil + db.frozenMem.decref() + db.frozenMem = nil + db.memMu.Unlock() } // Set closed flag; return true if not already closed. -func (d *DB) setClosed() bool { - return atomic.CompareAndSwapUint32(&d.closed, 0, 1) +func (db *DB) setClosed() bool { + return atomic.CompareAndSwapUint32(&db.closed, 0, 1) } // Check whether DB was closed. -func (d *DB) isClosed() bool { - return atomic.LoadUint32(&d.closed) != 0 +func (db *DB) isClosed() bool { + return atomic.LoadUint32(&db.closed) != 0 } // Check read ok status. -func (d *DB) ok() error { - if d.isClosed() { +func (db *DB) ok() error { + if db.isClosed() { return ErrClosed } return nil diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go index 5de7d9723..38bfbf1ea 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go @@ -7,6 +7,10 @@ package leveldb import ( + "bytes" + "container/list" + crand "crypto/rand" + "encoding/binary" "fmt" "math/rand" "os" @@ -20,6 +24,7 @@ import ( "unsafe" "github.com/syndtr/goleveldb/leveldb/comparer" + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" @@ -148,25 +153,29 @@ func (h *dbHarness) maxNextLevelOverlappingBytes(want uint64) { t := h.t db := h.db - var res uint64 + var ( + maxOverlaps uint64 + maxLevel int + ) v := db.s.version() for i, tt := range v.tables[1 : len(v.tables)-1] { level := i + 1 next := v.tables[level+1] for _, t := range tt { - var r tFiles - min, max := t.min.ukey(), t.max.ukey() - next.getOverlaps(min, max, &r, true, db.s.icmp.ucmp) + r := next.getOverlaps(nil, db.s.icmp, t.imin.ukey(), t.imax.ukey(), false) sum := r.size() - if sum > res { - res = sum + if sum > maxOverlaps { + maxOverlaps = sum + maxLevel = level } } } v.release() - if res > want { - t.Errorf("next level overlapping bytes is more than %d, got=%d", want, res) + if maxOverlaps > want { + t.Errorf("next level most overlapping bytes is more than %d, got=%d level=%d", want, maxOverlaps, maxLevel) + } else { + t.Logf("next level most overlapping bytes is %d, level=%d want=%d", maxOverlaps, maxLevel, want) } } @@ -239,7 +248,7 @@ func (h *dbHarness) allEntriesFor(key, want string) { db := h.db s := db.s - ikey := newIKey([]byte(key), kMaxSeq, tVal) + ikey := newIkey([]byte(key), kMaxSeq, ktVal) iter := db.newRawIterator(nil, nil) if !iter.Seek(ikey) && iter.Error() != nil { t.Error("AllEntries: error during seek, err: ", iter.Error()) @@ -248,19 +257,18 @@ func (h *dbHarness) allEntriesFor(key, want string) { res := "[ " first := true for iter.Valid() { - rkey := iKey(iter.Key()) - if _, t, ok := rkey.parseNum(); ok { - if s.icmp.uCompare(ikey.ukey(), rkey.ukey()) != 0 { + if ukey, _, kt, kerr := parseIkey(iter.Key()); kerr == nil { + if s.icmp.uCompare(ikey.ukey(), ukey) != 0 { break } if !first { res += ", " } first = false - switch t { - case tVal: + switch kt { + case ktVal: res += string(iter.Value()) - case tDel: + case ktDel: res += "DEL" } } else { @@ -325,6 +333,8 @@ func (h *dbHarness) compactMem() { t := h.t db := h.db + t.Log("starting memdb compaction") + db.writeLockC <- struct{}{} defer func() { <-db.writeLockC @@ -340,6 +350,8 @@ func (h *dbHarness) compactMem() { if h.totalTables() == 0 { t.Error("zero tables after mem compaction") } + + t.Log("memdb compaction done") } func (h *dbHarness) compactRangeAtErr(level int, min, max string, wanterr bool) { @@ -354,6 +366,8 @@ func (h *dbHarness) compactRangeAtErr(level int, min, max string, wanterr bool) _max = []byte(max) } + t.Logf("starting table range compaction: level=%d, min=%q, max=%q", level, min, max) + if err := db.compSendRange(db.tcompCmdC, level, _min, _max); err != nil { if wanterr { t.Log("CompactRangeAt: got error (expected): ", err) @@ -363,6 +377,8 @@ func (h *dbHarness) compactRangeAtErr(level int, min, max string, wanterr bool) } else if wanterr { t.Error("CompactRangeAt: expect error") } + + t.Log("table range compaction done") } func (h *dbHarness) compactRangeAt(level int, min, max string) { @@ -373,6 +389,8 @@ func (h *dbHarness) compactRange(min, max string) { t := h.t db := h.db + t.Logf("starting DB range compaction: min=%q, max=%q", min, max) + var r util.Range if min != "" { r.Start = []byte(min) @@ -383,21 +401,25 @@ func (h *dbHarness) compactRange(min, max string) { if err := db.CompactRange(r); err != nil { t.Error("CompactRange: got error: ", err) } -} -func (h *dbHarness) sizeAssert(start, limit string, low, hi uint64) { - t := h.t - db := h.db + t.Log("DB range compaction done") +} - s, err := db.SizeOf([]util.Range{ +func (h *dbHarness) sizeOf(start, limit string) uint64 { + sz, err := h.db.SizeOf([]util.Range{ {[]byte(start), []byte(limit)}, }) if err != nil { - t.Error("SizeOf: got error: ", err) + h.t.Error("SizeOf: got error: ", err) } - if s.Sum() < low || s.Sum() > hi { - t.Errorf("sizeof %q to %q not in range, want %d - %d, got %d", - shorten(start), shorten(limit), low, hi, s.Sum()) + return sz.Sum() +} + +func (h *dbHarness) sizeAssert(start, limit string, low, hi uint64) { + sz := h.sizeOf(start, limit) + if sz < low || sz > hi { + h.t.Errorf("sizeOf %q to %q not in range, want %d - %d, got %d", + shorten(start), shorten(limit), low, hi, sz) } } @@ -504,13 +526,13 @@ func Test_FieldsAligned(t *testing.T) { p1 := new(DB) testAligned(t, "DB.seq", unsafe.Offsetof(p1.seq)) p2 := new(session) - testAligned(t, "session.stFileNum", unsafe.Offsetof(p2.stFileNum)) + testAligned(t, "session.stNextFileNum", unsafe.Offsetof(p2.stNextFileNum)) testAligned(t, "session.stJournalNum", unsafe.Offsetof(p2.stJournalNum)) testAligned(t, "session.stPrevJournalNum", unsafe.Offsetof(p2.stPrevJournalNum)) - testAligned(t, "session.stSeq", unsafe.Offsetof(p2.stSeq)) + testAligned(t, "session.stSeqNum", unsafe.Offsetof(p2.stSeqNum)) } -func TestDb_Locking(t *testing.T) { +func TestDB_Locking(t *testing.T) { h := newDbHarness(t) defer h.stor.Close() h.openAssert(false) @@ -518,7 +540,7 @@ func TestDb_Locking(t *testing.T) { h.openAssert(true) } -func TestDb_Empty(t *testing.T) { +func TestDB_Empty(t *testing.T) { trun(t, func(h *dbHarness) { h.get("foo", false) @@ -527,7 +549,7 @@ func TestDb_Empty(t *testing.T) { }) } -func TestDb_ReadWrite(t *testing.T) { +func TestDB_ReadWrite(t *testing.T) { trun(t, func(h *dbHarness) { h.put("foo", "v1") h.getVal("foo", "v1") @@ -542,7 +564,7 @@ func TestDb_ReadWrite(t *testing.T) { }) } -func TestDb_PutDeleteGet(t *testing.T) { +func TestDB_PutDeleteGet(t *testing.T) { trun(t, func(h *dbHarness) { h.put("foo", "v1") h.getVal("foo", "v1") @@ -556,7 +578,7 @@ func TestDb_PutDeleteGet(t *testing.T) { }) } -func TestDb_EmptyBatch(t *testing.T) { +func TestDB_EmptyBatch(t *testing.T) { h := newDbHarness(t) defer h.close() @@ -568,7 +590,7 @@ func TestDb_EmptyBatch(t *testing.T) { h.get("foo", false) } -func TestDb_GetFromFrozen(t *testing.T) { +func TestDB_GetFromFrozen(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 100100}) defer h.close() @@ -594,7 +616,7 @@ func TestDb_GetFromFrozen(t *testing.T) { h.get("k2", true) } -func TestDb_GetFromTable(t *testing.T) { +func TestDB_GetFromTable(t *testing.T) { trun(t, func(h *dbHarness) { h.put("foo", "v1") h.compactMem() @@ -602,7 +624,7 @@ func TestDb_GetFromTable(t *testing.T) { }) } -func TestDb_GetSnapshot(t *testing.T) { +func TestDB_GetSnapshot(t *testing.T) { trun(t, func(h *dbHarness) { bar := strings.Repeat("b", 200) h.put("foo", "v1") @@ -636,7 +658,7 @@ func TestDb_GetSnapshot(t *testing.T) { }) } -func TestDb_GetLevel0Ordering(t *testing.T) { +func TestDB_GetLevel0Ordering(t *testing.T) { trun(t, func(h *dbHarness) { for i := 0; i < 4; i++ { h.put("bar", fmt.Sprintf("b%d", i)) @@ -659,7 +681,7 @@ func TestDb_GetLevel0Ordering(t *testing.T) { }) } -func TestDb_GetOrderedByLevels(t *testing.T) { +func TestDB_GetOrderedByLevels(t *testing.T) { trun(t, func(h *dbHarness) { h.put("foo", "v1") h.compactMem() @@ -671,7 +693,7 @@ func TestDb_GetOrderedByLevels(t *testing.T) { }) } -func TestDb_GetPicksCorrectFile(t *testing.T) { +func TestDB_GetPicksCorrectFile(t *testing.T) { trun(t, func(h *dbHarness) { // Arrange to have multiple files in a non-level-0 level. h.put("a", "va") @@ -695,7 +717,7 @@ func TestDb_GetPicksCorrectFile(t *testing.T) { }) } -func TestDb_GetEncountersEmptyLevel(t *testing.T) { +func TestDB_GetEncountersEmptyLevel(t *testing.T) { trun(t, func(h *dbHarness) { // Arrange for the following to happen: // * sstable A in level 0 @@ -750,7 +772,7 @@ func TestDb_GetEncountersEmptyLevel(t *testing.T) { }) } -func TestDb_IterMultiWithDelete(t *testing.T) { +func TestDB_IterMultiWithDelete(t *testing.T) { trun(t, func(h *dbHarness) { h.put("a", "va") h.put("b", "vb") @@ -776,7 +798,7 @@ func TestDb_IterMultiWithDelete(t *testing.T) { }) } -func TestDb_IteratorPinsRef(t *testing.T) { +func TestDB_IteratorPinsRef(t *testing.T) { h := newDbHarness(t) defer h.close() @@ -800,7 +822,7 @@ func TestDb_IteratorPinsRef(t *testing.T) { iter.Release() } -func TestDb_Recover(t *testing.T) { +func TestDB_Recover(t *testing.T) { trun(t, func(h *dbHarness) { h.put("foo", "v1") h.put("baz", "v5") @@ -822,7 +844,7 @@ func TestDb_Recover(t *testing.T) { }) } -func TestDb_RecoverWithEmptyJournal(t *testing.T) { +func TestDB_RecoverWithEmptyJournal(t *testing.T) { trun(t, func(h *dbHarness) { h.put("foo", "v1") h.put("foo", "v2") @@ -836,7 +858,7 @@ func TestDb_RecoverWithEmptyJournal(t *testing.T) { }) } -func TestDb_RecoverDuringMemtableCompaction(t *testing.T) { +func TestDB_RecoverDuringMemtableCompaction(t *testing.T) { truno(t, &opt.Options{WriteBuffer: 1000000}, func(h *dbHarness) { h.stor.DelaySync(storage.TypeTable) @@ -852,7 +874,7 @@ func TestDb_RecoverDuringMemtableCompaction(t *testing.T) { }) } -func TestDb_MinorCompactionsHappen(t *testing.T) { +func TestDB_MinorCompactionsHappen(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 10000}) defer h.close() @@ -876,7 +898,7 @@ func TestDb_MinorCompactionsHappen(t *testing.T) { } } -func TestDb_RecoverWithLargeJournal(t *testing.T) { +func TestDB_RecoverWithLargeJournal(t *testing.T) { h := newDbHarness(t) defer h.close() @@ -901,7 +923,7 @@ func TestDb_RecoverWithLargeJournal(t *testing.T) { v.release() } -func TestDb_CompactionsGenerateMultipleFiles(t *testing.T) { +func TestDB_CompactionsGenerateMultipleFiles(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{ WriteBuffer: 10000000, Compression: opt.NoCompression, @@ -939,11 +961,11 @@ func TestDb_CompactionsGenerateMultipleFiles(t *testing.T) { } } -func TestDb_RepeatedWritesToSameKey(t *testing.T) { +func TestDB_RepeatedWritesToSameKey(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 100000}) defer h.close() - maxTables := kNumLevels + kL0_StopWritesTrigger + maxTables := h.o.GetNumLevel() + h.o.GetWriteL0PauseTrigger() value := strings.Repeat("v", 2*h.o.GetWriteBuffer()) for i := 0; i < 5*maxTables; i++ { @@ -955,13 +977,13 @@ func TestDb_RepeatedWritesToSameKey(t *testing.T) { } } -func TestDb_RepeatedWritesToSameKeyAfterReopen(t *testing.T) { +func TestDB_RepeatedWritesToSameKeyAfterReopen(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 100000}) defer h.close() h.reopenDB() - maxTables := kNumLevels + kL0_StopWritesTrigger + maxTables := h.o.GetNumLevel() + h.o.GetWriteL0PauseTrigger() value := strings.Repeat("v", 2*h.o.GetWriteBuffer()) for i := 0; i < 5*maxTables; i++ { @@ -973,11 +995,11 @@ func TestDb_RepeatedWritesToSameKeyAfterReopen(t *testing.T) { } } -func TestDb_SparseMerge(t *testing.T) { +func TestDB_SparseMerge(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{Compression: opt.NoCompression}) defer h.close() - h.putMulti(kNumLevels, "A", "Z") + h.putMulti(h.o.GetNumLevel(), "A", "Z") // Suppose there is: // small amount of data with prefix A @@ -1001,6 +1023,7 @@ func TestDb_SparseMerge(t *testing.T) { h.put("C", "vc2") h.compactMem() + h.waitCompaction() h.maxNextLevelOverlappingBytes(20 * 1048576) h.compactRangeAt(0, "", "") h.waitCompaction() @@ -1010,7 +1033,7 @@ func TestDb_SparseMerge(t *testing.T) { h.maxNextLevelOverlappingBytes(20 * 1048576) } -func TestDb_SizeOf(t *testing.T) { +func TestDB_SizeOf(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{ Compression: opt.NoCompression, WriteBuffer: 10000000, @@ -1060,7 +1083,7 @@ func TestDb_SizeOf(t *testing.T) { } } -func TestDb_SizeOf_MixOfSmallAndLarge(t *testing.T) { +func TestDB_SizeOf_MixOfSmallAndLarge(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{Compression: opt.NoCompression}) defer h.close() @@ -1098,7 +1121,7 @@ func TestDb_SizeOf_MixOfSmallAndLarge(t *testing.T) { } } -func TestDb_Snapshot(t *testing.T) { +func TestDB_Snapshot(t *testing.T) { trun(t, func(h *dbHarness) { h.put("foo", "v1") s1 := h.getSnapshot() @@ -1127,13 +1150,51 @@ func TestDb_Snapshot(t *testing.T) { }) } -func TestDb_HiddenValuesAreRemoved(t *testing.T) { +func TestDB_SnapshotList(t *testing.T) { + db := &DB{snapsList: list.New()} + e0a := db.acquireSnapshot() + e0b := db.acquireSnapshot() + db.seq = 1 + e1 := db.acquireSnapshot() + db.seq = 2 + e2 := db.acquireSnapshot() + + if db.minSeq() != 0 { + t.Fatalf("invalid sequence number, got=%d", db.minSeq()) + } + db.releaseSnapshot(e0a) + if db.minSeq() != 0 { + t.Fatalf("invalid sequence number, got=%d", db.minSeq()) + } + db.releaseSnapshot(e2) + if db.minSeq() != 0 { + t.Fatalf("invalid sequence number, got=%d", db.minSeq()) + } + db.releaseSnapshot(e0b) + if db.minSeq() != 1 { + t.Fatalf("invalid sequence number, got=%d", db.minSeq()) + } + e2 = db.acquireSnapshot() + if db.minSeq() != 1 { + t.Fatalf("invalid sequence number, got=%d", db.minSeq()) + } + db.releaseSnapshot(e1) + if db.minSeq() != 2 { + t.Fatalf("invalid sequence number, got=%d", db.minSeq()) + } + db.releaseSnapshot(e2) + if db.minSeq() != 2 { + t.Fatalf("invalid sequence number, got=%d", db.minSeq()) + } +} + +func TestDB_HiddenValuesAreRemoved(t *testing.T) { trun(t, func(h *dbHarness) { s := h.db.s h.put("foo", "v1") h.compactMem() - m := kMaxMemCompactLevel + m := h.o.GetMaxMemCompationLevel() v := s.version() num := v.tLen(m) v.release() @@ -1170,14 +1231,14 @@ func TestDb_HiddenValuesAreRemoved(t *testing.T) { }) } -func TestDb_DeletionMarkers2(t *testing.T) { +func TestDB_DeletionMarkers2(t *testing.T) { h := newDbHarness(t) defer h.close() s := h.db.s h.put("foo", "v1") h.compactMem() - m := kMaxMemCompactLevel + m := h.o.GetMaxMemCompationLevel() v := s.version() num := v.tLen(m) v.release() @@ -1211,8 +1272,8 @@ func TestDb_DeletionMarkers2(t *testing.T) { h.allEntriesFor("foo", "[ ]") } -func TestDb_CompactionTableOpenError(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{MaxOpenFiles: 0}) +func TestDB_CompactionTableOpenError(t *testing.T) { + h := newDbHarnessWopt(t, &opt.Options{OpenFilesCacheCapacity: -1}) defer h.close() im := 10 @@ -1230,14 +1291,14 @@ func TestDb_CompactionTableOpenError(t *testing.T) { t.Errorf("total tables is %d, want %d", n, im) } - h.stor.SetOpenErr(storage.TypeTable) + h.stor.SetEmuErr(storage.TypeTable, tsOpOpen) go h.db.CompactRange(util.Range{}) if err := h.db.compSendIdle(h.db.tcompCmdC); err != nil { t.Log("compaction error: ", err) } h.closeDB0() h.openDB() - h.stor.SetOpenErr(0) + h.stor.SetEmuErr(0, tsOpOpen) for i := 0; i < im; i++ { for j := 0; j < jm; j++ { @@ -1246,9 +1307,9 @@ func TestDb_CompactionTableOpenError(t *testing.T) { } } -func TestDb_OverlapInLevel0(t *testing.T) { +func TestDB_OverlapInLevel0(t *testing.T) { trun(t, func(h *dbHarness) { - if kMaxMemCompactLevel != 2 { + if h.o.GetMaxMemCompationLevel() != 2 { t.Fatal("fix test to reflect the config") } @@ -1289,7 +1350,7 @@ func TestDb_OverlapInLevel0(t *testing.T) { }) } -func TestDb_L0_CompactionBug_Issue44_a(t *testing.T) { +func TestDB_L0_CompactionBug_Issue44_a(t *testing.T) { h := newDbHarness(t) defer h.close() @@ -1309,7 +1370,7 @@ func TestDb_L0_CompactionBug_Issue44_a(t *testing.T) { h.getKeyVal("(a->v)") } -func TestDb_L0_CompactionBug_Issue44_b(t *testing.T) { +func TestDB_L0_CompactionBug_Issue44_b(t *testing.T) { h := newDbHarness(t) defer h.close() @@ -1338,7 +1399,7 @@ func TestDb_L0_CompactionBug_Issue44_b(t *testing.T) { h.getKeyVal("(->)(c->cv)") } -func TestDb_SingleEntryMemCompaction(t *testing.T) { +func TestDB_SingleEntryMemCompaction(t *testing.T) { trun(t, func(h *dbHarness) { for i := 0; i < 10; i++ { h.put("big", strings.Repeat("v", opt.DefaultWriteBuffer)) @@ -1355,7 +1416,7 @@ func TestDb_SingleEntryMemCompaction(t *testing.T) { }) } -func TestDb_ManifestWriteError(t *testing.T) { +func TestDB_ManifestWriteError(t *testing.T) { for i := 0; i < 2; i++ { func() { h := newDbHarness(t) @@ -1368,23 +1429,23 @@ func TestDb_ManifestWriteError(t *testing.T) { h.compactMem() h.getVal("foo", "bar") v := h.db.s.version() - if n := v.tLen(kMaxMemCompactLevel); n != 1 { + if n := v.tLen(h.o.GetMaxMemCompationLevel()); n != 1 { t.Errorf("invalid total tables, want=1 got=%d", n) } v.release() if i == 0 { - h.stor.SetWriteErr(storage.TypeManifest) + h.stor.SetEmuErr(storage.TypeManifest, tsOpWrite) } else { - h.stor.SetSyncErr(storage.TypeManifest) + h.stor.SetEmuErr(storage.TypeManifest, tsOpSync) } // Merging compaction (will fail) - h.compactRangeAtErr(kMaxMemCompactLevel, "", "", true) + h.compactRangeAtErr(h.o.GetMaxMemCompationLevel(), "", "", true) h.db.Close() - h.stor.SetWriteErr(0) - h.stor.SetSyncErr(0) + h.stor.SetEmuErr(0, tsOpWrite) + h.stor.SetEmuErr(0, tsOpSync) // Should not lose data h.openDB() @@ -1405,7 +1466,7 @@ func assertErr(t *testing.T, err error, wanterr bool) { } } -func TestDb_ClosedIsClosed(t *testing.T) { +func TestDB_ClosedIsClosed(t *testing.T) { h := newDbHarness(t) db := h.db @@ -1500,7 +1561,7 @@ func (p numberComparer) Compare(a, b []byte) int { func (numberComparer) Separator(dst, a, b []byte) []byte { return nil } func (numberComparer) Successor(dst, b []byte) []byte { return nil } -func TestDb_CustomComparer(t *testing.T) { +func TestDB_CustomComparer(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{ Comparer: numberComparer{}, WriteBuffer: 1000, @@ -1530,11 +1591,11 @@ func TestDb_CustomComparer(t *testing.T) { } } -func TestDb_ManualCompaction(t *testing.T) { +func TestDB_ManualCompaction(t *testing.T) { h := newDbHarness(t) defer h.close() - if kMaxMemCompactLevel != 2 { + if h.o.GetMaxMemCompationLevel() != 2 { t.Fatal("fix test to reflect the config") } @@ -1568,10 +1629,10 @@ func TestDb_ManualCompaction(t *testing.T) { h.tablesPerLevel("0,0,1") } -func TestDb_BloomFilter(t *testing.T) { +func TestDB_BloomFilter(t *testing.T) { h := newDbHarnessWopt(t, &opt.Options{ - BlockCache: opt.NoCache, - Filter: filter.NewBloomFilter(10), + DisableBlockCache: true, + Filter: filter.NewBloomFilter(10), }) defer h.close() @@ -1579,7 +1640,7 @@ func TestDb_BloomFilter(t *testing.T) { return fmt.Sprintf("key%06d", i) } - n := 10000 + const n = 10000 // Populate multiple layers for i := 0; i < n; i++ { @@ -1621,7 +1682,7 @@ func TestDb_BloomFilter(t *testing.T) { h.stor.ReleaseSync(storage.TypeTable) } -func TestDb_Concurrent(t *testing.T) { +func TestDB_Concurrent(t *testing.T) { const n, secs, maxkey = 4, 2, 1000 runtime.GOMAXPROCS(n) @@ -1686,7 +1747,7 @@ func TestDb_Concurrent(t *testing.T) { runtime.GOMAXPROCS(1) } -func TestDb_Concurrent2(t *testing.T) { +func TestDB_Concurrent2(t *testing.T) { const n, n2 = 4, 4000 runtime.GOMAXPROCS(n*2 + 2) @@ -1757,7 +1818,7 @@ func TestDb_Concurrent2(t *testing.T) { runtime.GOMAXPROCS(1) } -func TestDb_CreateReopenDbOnFile(t *testing.T) { +func TestDB_CreateReopenDbOnFile(t *testing.T) { dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile-%d", os.Getuid())) if err := os.RemoveAll(dbpath); err != nil { t.Fatal("cannot remove old db: ", err) @@ -1785,7 +1846,7 @@ func TestDb_CreateReopenDbOnFile(t *testing.T) { } } -func TestDb_CreateReopenDbOnFile2(t *testing.T) { +func TestDB_CreateReopenDbOnFile2(t *testing.T) { dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile2-%d", os.Getuid())) if err := os.RemoveAll(dbpath); err != nil { t.Fatal("cannot remove old db: ", err) @@ -1806,7 +1867,7 @@ func TestDb_CreateReopenDbOnFile2(t *testing.T) { } } -func TestDb_DeletionMarkersOnMemdb(t *testing.T) { +func TestDB_DeletionMarkersOnMemdb(t *testing.T) { h := newDbHarness(t) defer h.close() @@ -1817,8 +1878,8 @@ func TestDb_DeletionMarkersOnMemdb(t *testing.T) { h.getKeyVal("") } -func TestDb_LeveldbIssue178(t *testing.T) { - nKeys := (kMaxTableSize / 30) * 5 +func TestDB_LeveldbIssue178(t *testing.T) { + nKeys := (opt.DefaultCompactionTableSize / 30) * 5 key1 := func(i int) string { return fmt.Sprintf("my_key_%d", i) } @@ -1860,7 +1921,7 @@ func TestDb_LeveldbIssue178(t *testing.T) { h.assertNumKeys(nKeys) } -func TestDb_LeveldbIssue200(t *testing.T) { +func TestDB_LeveldbIssue200(t *testing.T) { h := newDbHarness(t) defer h.close() @@ -1886,3 +1947,719 @@ func TestDb_LeveldbIssue200(t *testing.T) { iter.Next() assertBytes(t, []byte("5"), iter.Key()) } + +func TestDB_GoleveldbIssue74(t *testing.T) { + h := newDbHarnessWopt(t, &opt.Options{ + WriteBuffer: 1 * opt.MiB, + }) + defer h.close() + + const n, dur = 10000, 5 * time.Second + + runtime.GOMAXPROCS(runtime.NumCPU()) + + until := time.Now().Add(dur) + wg := new(sync.WaitGroup) + wg.Add(2) + var done uint32 + go func() { + var i int + defer func() { + t.Logf("WRITER DONE #%d", i) + atomic.StoreUint32(&done, 1) + wg.Done() + }() + + b := new(Batch) + for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ { + iv := fmt.Sprintf("VAL%010d", i) + for k := 0; k < n; k++ { + key := fmt.Sprintf("KEY%06d", k) + b.Put([]byte(key), []byte(key+iv)) + b.Put([]byte(fmt.Sprintf("PTR%06d", k)), []byte(key)) + } + h.write(b) + + b.Reset() + snap := h.getSnapshot() + iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil) + var k int + for ; iter.Next(); k++ { + ptrKey := iter.Key() + key := iter.Value() + + if _, err := snap.Get(ptrKey, nil); err != nil { + t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, ptrKey, err) + } + if value, err := snap.Get(key, nil); err != nil { + t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, key, err) + } else if string(value) != string(key)+iv { + t.Fatalf("WRITER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+iv, value) + } + + b.Delete(key) + b.Delete(ptrKey) + } + h.write(b) + iter.Release() + snap.Release() + if k != n { + t.Fatalf("#%d %d != %d", i, k, n) + } + } + }() + go func() { + var i int + defer func() { + t.Logf("READER DONE #%d", i) + atomic.StoreUint32(&done, 1) + wg.Done() + }() + for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ { + snap := h.getSnapshot() + iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil) + var prevValue string + var k int + for ; iter.Next(); k++ { + ptrKey := iter.Key() + key := iter.Value() + + if _, err := snap.Get(ptrKey, nil); err != nil { + t.Fatalf("READER #%d snapshot.Get %q: %v", i, ptrKey, err) + } + + if value, err := snap.Get(key, nil); err != nil { + t.Fatalf("READER #%d snapshot.Get %q: %v", i, key, err) + } else if prevValue != "" && string(value) != string(key)+prevValue { + t.Fatalf("READER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+prevValue, value) + } else { + prevValue = string(value[len(key):]) + } + } + iter.Release() + snap.Release() + if k > 0 && k != n { + t.Fatalf("#%d %d != %d", i, k, n) + } + } + }() + wg.Wait() +} + +func TestDB_GetProperties(t *testing.T) { + h := newDbHarness(t) + defer h.close() + + _, err := h.db.GetProperty("leveldb.num-files-at-level") + if err == nil { + t.Error("GetProperty() failed to detect missing level") + } + + _, err = h.db.GetProperty("leveldb.num-files-at-level0") + if err != nil { + t.Error("got unexpected error", err) + } + + _, err = h.db.GetProperty("leveldb.num-files-at-level0x") + if err == nil { + t.Error("GetProperty() failed to detect invalid level") + } +} + +func TestDB_GoleveldbIssue72and83(t *testing.T) { + h := newDbHarnessWopt(t, &opt.Options{ + WriteBuffer: 1 * opt.MiB, + OpenFilesCacheCapacity: 3, + }) + defer h.close() + + const n, wn, dur = 10000, 100, 30 * time.Second + + runtime.GOMAXPROCS(runtime.NumCPU()) + + randomData := func(prefix byte, i int) []byte { + data := make([]byte, 1+4+32+64+32) + _, err := crand.Reader.Read(data[1 : len(data)-8]) + if err != nil { + panic(err) + } + data[0] = prefix + binary.LittleEndian.PutUint32(data[len(data)-8:], uint32(i)) + binary.LittleEndian.PutUint32(data[len(data)-4:], util.NewCRC(data[:len(data)-4]).Value()) + return data + } + + keys := make([][]byte, n) + for i := range keys { + keys[i] = randomData(1, 0) + } + + until := time.Now().Add(dur) + wg := new(sync.WaitGroup) + wg.Add(3) + var done uint32 + go func() { + i := 0 + defer func() { + t.Logf("WRITER DONE #%d", i) + wg.Done() + }() + + b := new(Batch) + for ; i < wn && atomic.LoadUint32(&done) == 0; i++ { + b.Reset() + for _, k1 := range keys { + k2 := randomData(2, i) + b.Put(k2, randomData(42, i)) + b.Put(k1, k2) + } + if err := h.db.Write(b, h.wo); err != nil { + atomic.StoreUint32(&done, 1) + t.Fatalf("WRITER #%d db.Write: %v", i, err) + } + } + }() + go func() { + var i int + defer func() { + t.Logf("READER0 DONE #%d", i) + atomic.StoreUint32(&done, 1) + wg.Done() + }() + for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ { + snap := h.getSnapshot() + seq := snap.elem.seq + if seq == 0 { + snap.Release() + continue + } + iter := snap.NewIterator(util.BytesPrefix([]byte{1}), nil) + writei := int(seq/(n*2) - 1) + var k int + for ; iter.Next(); k++ { + k1 := iter.Key() + k2 := iter.Value() + k1checksum0 := binary.LittleEndian.Uint32(k1[len(k1)-4:]) + k1checksum1 := util.NewCRC(k1[:len(k1)-4]).Value() + if k1checksum0 != k1checksum1 { + t.Fatalf("READER0 #%d.%d W#%d invalid K1 checksum: %#x != %#x", i, k, k1checksum0, k1checksum0) + } + k2checksum0 := binary.LittleEndian.Uint32(k2[len(k2)-4:]) + k2checksum1 := util.NewCRC(k2[:len(k2)-4]).Value() + if k2checksum0 != k2checksum1 { + t.Fatalf("READER0 #%d.%d W#%d invalid K2 checksum: %#x != %#x", i, k, k2checksum0, k2checksum1) + } + kwritei := int(binary.LittleEndian.Uint32(k2[len(k2)-8:])) + if writei != kwritei { + t.Fatalf("READER0 #%d.%d W#%d invalid write iteration num: %d", i, k, writei, kwritei) + } + if _, err := snap.Get(k2, nil); err != nil { + t.Fatalf("READER0 #%d.%d W#%d snap.Get: %v\nk1: %x\n -> k2: %x", i, k, writei, err, k1, k2) + } + } + if err := iter.Error(); err != nil { + t.Fatalf("READER0 #%d.%d W#%d snap.Iterator: %v", i, k, writei, err) + } + iter.Release() + snap.Release() + if k > 0 && k != n { + t.Fatalf("READER0 #%d W#%d short read, got=%d want=%d", i, writei, k, n) + } + } + }() + go func() { + var i int + defer func() { + t.Logf("READER1 DONE #%d", i) + atomic.StoreUint32(&done, 1) + wg.Done() + }() + for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ { + iter := h.db.NewIterator(nil, nil) + seq := iter.(*dbIter).seq + if seq == 0 { + iter.Release() + continue + } + writei := int(seq/(n*2) - 1) + var k int + for ok := iter.Last(); ok; ok = iter.Prev() { + k++ + } + if err := iter.Error(); err != nil { + t.Fatalf("READER1 #%d.%d W#%d db.Iterator: %v", i, k, writei, err) + } + iter.Release() + if m := (writei+1)*n + n; k != m { + t.Fatalf("READER1 #%d W#%d short read, got=%d want=%d", i, writei, k, m) + } + } + }() + + wg.Wait() +} + +func TestDB_TransientError(t *testing.T) { + h := newDbHarnessWopt(t, &opt.Options{ + WriteBuffer: 128 * opt.KiB, + OpenFilesCacheCapacity: 3, + DisableCompactionBackoff: true, + }) + defer h.close() + + const ( + nSnap = 20 + nKey = 10000 + ) + + var ( + snaps [nSnap]*Snapshot + b = &Batch{} + ) + for i := range snaps { + vtail := fmt.Sprintf("VAL%030d", i) + b.Reset() + for k := 0; k < nKey; k++ { + key := fmt.Sprintf("KEY%8d", k) + b.Put([]byte(key), []byte(key+vtail)) + } + h.stor.SetEmuRandErr(storage.TypeTable, tsOpOpen, tsOpRead, tsOpReadAt) + if err := h.db.Write(b, nil); err != nil { + t.Logf("WRITE #%d error: %v", i, err) + h.stor.SetEmuRandErr(0, tsOpOpen, tsOpRead, tsOpReadAt, tsOpWrite) + for { + if err := h.db.Write(b, nil); err == nil { + break + } else if errors.IsCorrupted(err) { + t.Fatalf("WRITE #%d corrupted: %v", i, err) + } + } + } + + snaps[i] = h.db.newSnapshot() + b.Reset() + for k := 0; k < nKey; k++ { + key := fmt.Sprintf("KEY%8d", k) + b.Delete([]byte(key)) + } + h.stor.SetEmuRandErr(storage.TypeTable, tsOpOpen, tsOpRead, tsOpReadAt) + if err := h.db.Write(b, nil); err != nil { + t.Logf("WRITE #%d error: %v", i, err) + h.stor.SetEmuRandErr(0, tsOpOpen, tsOpRead, tsOpReadAt) + for { + if err := h.db.Write(b, nil); err == nil { + break + } else if errors.IsCorrupted(err) { + t.Fatalf("WRITE #%d corrupted: %v", i, err) + } + } + } + } + h.stor.SetEmuRandErr(0, tsOpOpen, tsOpRead, tsOpReadAt) + + runtime.GOMAXPROCS(runtime.NumCPU()) + + rnd := rand.New(rand.NewSource(0xecafdaed)) + wg := &sync.WaitGroup{} + for i, snap := range snaps { + wg.Add(2) + + go func(i int, snap *Snapshot, sk []int) { + defer wg.Done() + + vtail := fmt.Sprintf("VAL%030d", i) + for _, k := range sk { + key := fmt.Sprintf("KEY%8d", k) + xvalue, err := snap.Get([]byte(key), nil) + if err != nil { + t.Fatalf("READER_GET #%d SEQ=%d K%d error: %v", i, snap.elem.seq, k, err) + } + value := key + vtail + if !bytes.Equal([]byte(value), xvalue) { + t.Fatalf("READER_GET #%d SEQ=%d K%d invalid value: want %q, got %q", i, snap.elem.seq, k, value, xvalue) + } + } + }(i, snap, rnd.Perm(nKey)) + + go func(i int, snap *Snapshot) { + defer wg.Done() + + vtail := fmt.Sprintf("VAL%030d", i) + iter := snap.NewIterator(nil, nil) + defer iter.Release() + for k := 0; k < nKey; k++ { + if !iter.Next() { + if err := iter.Error(); err != nil { + t.Fatalf("READER_ITER #%d K%d error: %v", i, k, err) + } else { + t.Fatalf("READER_ITER #%d K%d eoi", i, k) + } + } + key := fmt.Sprintf("KEY%8d", k) + xkey := iter.Key() + if !bytes.Equal([]byte(key), xkey) { + t.Fatalf("READER_ITER #%d K%d invalid key: want %q, got %q", i, k, key, xkey) + } + value := key + vtail + xvalue := iter.Value() + if !bytes.Equal([]byte(value), xvalue) { + t.Fatalf("READER_ITER #%d K%d invalid value: want %q, got %q", i, k, value, xvalue) + } + } + }(i, snap) + } + + wg.Wait() +} + +func TestDB_UkeyShouldntHopAcrossTable(t *testing.T) { + h := newDbHarnessWopt(t, &opt.Options{ + WriteBuffer: 112 * opt.KiB, + CompactionTableSize: 90 * opt.KiB, + CompactionExpandLimitFactor: 1, + }) + defer h.close() + + const ( + nSnap = 190 + nKey = 140 + ) + + var ( + snaps [nSnap]*Snapshot + b = &Batch{} + ) + for i := range snaps { + vtail := fmt.Sprintf("VAL%030d", i) + b.Reset() + for k := 0; k < nKey; k++ { + key := fmt.Sprintf("KEY%08d", k) + b.Put([]byte(key), []byte(key+vtail)) + } + if err := h.db.Write(b, nil); err != nil { + t.Fatalf("WRITE #%d error: %v", i, err) + } + + snaps[i] = h.db.newSnapshot() + b.Reset() + for k := 0; k < nKey; k++ { + key := fmt.Sprintf("KEY%08d", k) + b.Delete([]byte(key)) + } + if err := h.db.Write(b, nil); err != nil { + t.Fatalf("WRITE #%d error: %v", i, err) + } + } + + h.compactMem() + + h.waitCompaction() + for level, tables := range h.db.s.stVersion.tables { + for _, table := range tables { + t.Logf("L%d@%d %q:%q", level, table.file.Num(), table.imin, table.imax) + } + } + + h.compactRangeAt(0, "", "") + h.waitCompaction() + for level, tables := range h.db.s.stVersion.tables { + for _, table := range tables { + t.Logf("L%d@%d %q:%q", level, table.file.Num(), table.imin, table.imax) + } + } + h.compactRangeAt(1, "", "") + h.waitCompaction() + for level, tables := range h.db.s.stVersion.tables { + for _, table := range tables { + t.Logf("L%d@%d %q:%q", level, table.file.Num(), table.imin, table.imax) + } + } + runtime.GOMAXPROCS(runtime.NumCPU()) + + wg := &sync.WaitGroup{} + for i, snap := range snaps { + wg.Add(1) + + go func(i int, snap *Snapshot) { + defer wg.Done() + + vtail := fmt.Sprintf("VAL%030d", i) + for k := 0; k < nKey; k++ { + key := fmt.Sprintf("KEY%08d", k) + xvalue, err := snap.Get([]byte(key), nil) + if err != nil { + t.Fatalf("READER_GET #%d SEQ=%d K%d error: %v", i, snap.elem.seq, k, err) + } + value := key + vtail + if !bytes.Equal([]byte(value), xvalue) { + t.Fatalf("READER_GET #%d SEQ=%d K%d invalid value: want %q, got %q", i, snap.elem.seq, k, value, xvalue) + } + } + }(i, snap) + } + + wg.Wait() +} + +func TestDB_TableCompactionBuilder(t *testing.T) { + stor := newTestStorage(t) + defer stor.Close() + + const nSeq = 99 + + o := &opt.Options{ + WriteBuffer: 112 * opt.KiB, + CompactionTableSize: 43 * opt.KiB, + CompactionExpandLimitFactor: 1, + CompactionGPOverlapsFactor: 1, + DisableBlockCache: true, + } + s, err := newSession(stor, o) + if err != nil { + t.Fatal(err) + } + if err := s.create(); err != nil { + t.Fatal(err) + } + defer s.close() + var ( + seq uint64 + targetSize = 5 * o.CompactionTableSize + value = bytes.Repeat([]byte{'0'}, 100) + ) + for i := 0; i < 2; i++ { + tw, err := s.tops.create() + if err != nil { + t.Fatal(err) + } + for k := 0; tw.tw.BytesLen() < targetSize; k++ { + key := []byte(fmt.Sprintf("%09d", k)) + seq += nSeq - 1 + for x := uint64(0); x < nSeq; x++ { + if err := tw.append(newIkey(key, seq-x, ktVal), value); err != nil { + t.Fatal(err) + } + } + } + tf, err := tw.finish() + if err != nil { + t.Fatal(err) + } + rec := &sessionRecord{numLevel: s.o.GetNumLevel()} + rec.addTableFile(i, tf) + if err := s.commit(rec); err != nil { + t.Fatal(err) + } + } + + // Build grandparent. + v := s.version() + c := newCompaction(s, v, 1, append(tFiles{}, v.tables[1]...)) + rec := &sessionRecord{numLevel: s.o.GetNumLevel()} + b := &tableCompactionBuilder{ + s: s, + c: c, + rec: rec, + stat1: new(cStatsStaging), + minSeq: 0, + strict: true, + tableSize: o.CompactionTableSize/3 + 961, + } + if err := b.run(new(compactionTransactCounter)); err != nil { + t.Fatal(err) + } + for _, t := range c.tables[0] { + rec.delTable(c.level, t.file.Num()) + } + if err := s.commit(rec); err != nil { + t.Fatal(err) + } + c.release() + + // Build level-1. + v = s.version() + c = newCompaction(s, v, 0, append(tFiles{}, v.tables[0]...)) + rec = &sessionRecord{numLevel: s.o.GetNumLevel()} + b = &tableCompactionBuilder{ + s: s, + c: c, + rec: rec, + stat1: new(cStatsStaging), + minSeq: 0, + strict: true, + tableSize: o.CompactionTableSize, + } + if err := b.run(new(compactionTransactCounter)); err != nil { + t.Fatal(err) + } + for _, t := range c.tables[0] { + rec.delTable(c.level, t.file.Num()) + } + // Move grandparent to level-3 + for _, t := range v.tables[2] { + rec.delTable(2, t.file.Num()) + rec.addTableFile(3, t) + } + if err := s.commit(rec); err != nil { + t.Fatal(err) + } + c.release() + + v = s.version() + for level, want := range []bool{false, true, false, true, false} { + got := len(v.tables[level]) > 0 + if want != got { + t.Fatalf("invalid level-%d tables len: want %v, got %v", level, want, got) + } + } + for i, f := range v.tables[1][:len(v.tables[1])-1] { + nf := v.tables[1][i+1] + if bytes.Equal(f.imax.ukey(), nf.imin.ukey()) { + t.Fatalf("KEY %q hop across table %d .. %d", f.imax.ukey(), f.file.Num(), nf.file.Num()) + } + } + v.release() + + // Compaction with transient error. + v = s.version() + c = newCompaction(s, v, 1, append(tFiles{}, v.tables[1]...)) + rec = &sessionRecord{numLevel: s.o.GetNumLevel()} + b = &tableCompactionBuilder{ + s: s, + c: c, + rec: rec, + stat1: new(cStatsStaging), + minSeq: 0, + strict: true, + tableSize: o.CompactionTableSize, + } + stor.SetEmuErrOnce(storage.TypeTable, tsOpSync) + stor.SetEmuRandErr(storage.TypeTable, tsOpRead, tsOpReadAt, tsOpWrite) + stor.SetEmuRandErrProb(0xf0) + for { + if err := b.run(new(compactionTransactCounter)); err != nil { + t.Logf("(expected) b.run: %v", err) + } else { + break + } + } + if err := s.commit(rec); err != nil { + t.Fatal(err) + } + c.release() + + stor.SetEmuErrOnce(0, tsOpSync) + stor.SetEmuRandErr(0, tsOpRead, tsOpReadAt, tsOpWrite) + + v = s.version() + if len(v.tables[1]) != len(v.tables[2]) { + t.Fatalf("invalid tables length, want %d, got %d", len(v.tables[1]), len(v.tables[2])) + } + for i, f0 := range v.tables[1] { + f1 := v.tables[2][i] + iter0 := s.tops.newIterator(f0, nil, nil) + iter1 := s.tops.newIterator(f1, nil, nil) + for j := 0; true; j++ { + next0 := iter0.Next() + next1 := iter1.Next() + if next0 != next1 { + t.Fatalf("#%d.%d invalid eoi: want %v, got %v", i, j, next0, next1) + } + key0 := iter0.Key() + key1 := iter1.Key() + if !bytes.Equal(key0, key1) { + t.Fatalf("#%d.%d invalid key: want %q, got %q", i, j, key0, key1) + } + if next0 == false { + break + } + } + iter0.Release() + iter1.Release() + } + v.release() +} + +func testDB_IterTriggeredCompaction(t *testing.T, limitDiv int) { + const ( + vSize = 200 * opt.KiB + tSize = 100 * opt.MiB + mIter = 100 + n = tSize / vSize + ) + + h := newDbHarnessWopt(t, &opt.Options{ + Compression: opt.NoCompression, + DisableBlockCache: true, + }) + defer h.close() + + key := func(x int) string { + return fmt.Sprintf("v%06d", x) + } + + // Fill. + value := strings.Repeat("x", vSize) + for i := 0; i < n; i++ { + h.put(key(i), value) + } + h.compactMem() + + // Delete all. + for i := 0; i < n; i++ { + h.delete(key(i)) + } + h.compactMem() + + var ( + limit = n / limitDiv + + startKey = key(0) + limitKey = key(limit) + maxKey = key(n) + slice = &util.Range{Limit: []byte(limitKey)} + + initialSize0 = h.sizeOf(startKey, limitKey) + initialSize1 = h.sizeOf(limitKey, maxKey) + ) + + t.Logf("inital size %s [rest %s]", shortenb(int(initialSize0)), shortenb(int(initialSize1))) + + for r := 0; true; r++ { + if r >= mIter { + t.Fatal("taking too long to compact") + } + + // Iterates. + iter := h.db.NewIterator(slice, h.ro) + for iter.Next() { + } + if err := iter.Error(); err != nil { + t.Fatalf("Iter err: %v", err) + } + iter.Release() + + // Wait compaction. + h.waitCompaction() + + // Check size. + size0 := h.sizeOf(startKey, limitKey) + size1 := h.sizeOf(limitKey, maxKey) + t.Logf("#%03d size %s [rest %s]", r, shortenb(int(size0)), shortenb(int(size1))) + if size0 < initialSize0/10 { + break + } + } + + if initialSize1 > 0 { + h.sizeAssert(limitKey, maxKey, initialSize1/4-opt.MiB, initialSize1+opt.MiB) + } +} + +func TestDB_IterTriggeredCompaction(t *testing.T) { + testDB_IterTriggeredCompaction(t, 1) +} + +func TestDB_IterTriggeredCompactionHalf(t *testing.T) { + testDB_IterTriggeredCompaction(t, 2) +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_util.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_util.go index 2db007e27..a8a2bdf72 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_util.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_util.go @@ -7,8 +7,7 @@ package leveldb import ( - "errors" - + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/storage" @@ -32,40 +31,44 @@ func (p Sizes) Sum() (n uint64) { return n } +// Logging. +func (db *DB) log(v ...interface{}) { db.s.log(v...) } +func (db *DB) logf(format string, v ...interface{}) { db.s.logf(format, v...) } + // Check and clean files. -func (d *DB) checkAndCleanFiles() error { - s := d.s +func (db *DB) checkAndCleanFiles() error { + v := db.s.version() + defer v.release() - v := s.version_NB() - tables := make(map[uint64]bool) - for _, tt := range v.tables { - for _, t := range tt { - tables[t.file.Num()] = false + tablesMap := make(map[uint64]bool) + for _, tables := range v.tables { + for _, t := range tables { + tablesMap[t.file.Num()] = false } } - ff, err := s.getFiles(storage.TypeAll) + files, err := db.s.getFiles(storage.TypeAll) if err != nil { return err } var nTables int var rem []storage.File - for _, f := range ff { + for _, f := range files { keep := true switch f.Type() { case storage.TypeManifest: - keep = f.Num() >= s.manifestFile.Num() + keep = f.Num() >= db.s.manifestFile.Num() case storage.TypeJournal: - if d.frozenJournalFile != nil { - keep = f.Num() >= d.frozenJournalFile.Num() + if db.frozenJournalFile != nil { + keep = f.Num() >= db.frozenJournalFile.Num() } else { - keep = f.Num() >= d.journalFile.Num() + keep = f.Num() >= db.journalFile.Num() } case storage.TypeTable: - _, keep = tables[f.Num()] + _, keep = tablesMap[f.Num()] if keep { - tables[f.Num()] = true + tablesMap[f.Num()] = true nTables++ } } @@ -75,18 +78,20 @@ func (d *DB) checkAndCleanFiles() error { } } - if nTables != len(tables) { - for num, present := range tables { + if nTables != len(tablesMap) { + var missing []*storage.FileInfo + for num, present := range tablesMap { if !present { - s.logf("db@janitor table missing @%d", num) + missing = append(missing, &storage.FileInfo{Type: storage.TypeTable, Num: num}) + db.logf("db@janitor table missing @%d", num) } } - return ErrCorrupted{Type: MissingFiles, Err: errors.New("leveldb: table files missing")} + return errors.NewErrCorrupted(nil, &errors.ErrMissingFiles{Files: missing}) } - s.logf("db@janitor F·%d G·%d", len(ff), len(rem)) + db.logf("db@janitor F·%d G·%d", len(files), len(rem)) for _, f := range rem { - s.logf("db@janitor removing %s-%d", f.Type(), f.Num()) + db.logf("db@janitor removing %s-%d", f.Type(), f.Num()) if err := f.Remove(); err != nil { return err } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go index 4660e840c..e1cf30c53 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go @@ -14,84 +14,93 @@ import ( "github.com/syndtr/goleveldb/leveldb/util" ) -func (d *DB) writeJournal(b *Batch) error { - w, err := d.journal.Next() +func (db *DB) writeJournal(b *Batch) error { + w, err := db.journal.Next() if err != nil { return err } if _, err := w.Write(b.encode()); err != nil { return err } - if err := d.journal.Flush(); err != nil { + if err := db.journal.Flush(); err != nil { return err } if b.sync { - return d.journalWriter.Sync() + return db.journalWriter.Sync() } return nil } -func (d *DB) jWriter() { - defer d.closeW.Done() +func (db *DB) jWriter() { + defer db.closeW.Done() for { select { - case b := <-d.journalC: + case b := <-db.journalC: if b != nil { - d.journalAckC <- d.writeJournal(b) + db.journalAckC <- db.writeJournal(b) } - case _, _ = <-d.closeC: + case _, _ = <-db.closeC: return } } } -func (d *DB) rotateMem(n int) (mem *memdb.DB, err error) { +func (db *DB) rotateMem(n int) (mem *memDB, err error) { // Wait for pending memdb compaction. - err = d.compSendIdle(d.mcompCmdC) + err = db.compSendIdle(db.mcompCmdC) if err != nil { return } // Create new memdb and journal. - mem, err = d.newMem(n) + mem, err = db.newMem(n) if err != nil { return } // Schedule memdb compaction. - d.compTrigger(d.mcompTriggerC) + db.compSendTrigger(db.mcompCmdC) return } -func (d *DB) flush(n int) (mem *memdb.DB, nn int, err error) { - s := d.s - +func (db *DB) flush(n int) (mem *memDB, nn int, err error) { delayed := false - flush := func() bool { - v := s.version() + flush := func() (retry bool) { + v := db.s.version() defer v.release() - mem = d.getEffectiveMem() - nn = mem.Free() + mem = db.getEffectiveMem() + defer func() { + if retry { + mem.decref() + mem = nil + } + }() + nn = mem.mdb.Free() switch { - case v.tLen(0) >= kL0_SlowdownWritesTrigger && !delayed: + case v.tLen(0) >= db.s.o.GetWriteL0SlowdownTrigger() && !delayed: delayed = true time.Sleep(time.Millisecond) case nn >= n: return false - case v.tLen(0) >= kL0_StopWritesTrigger: + case v.tLen(0) >= db.s.o.GetWriteL0PauseTrigger(): delayed = true - err = d.compSendIdle(d.tcompCmdC) + err = db.compSendIdle(db.tcompCmdC) if err != nil { return false } default: // Allow memdb to grow if it has no entry. - if mem.Len() == 0 { + if mem.mdb.Len() == 0 { nn = n - return false + } else { + mem.decref() + mem, err = db.rotateMem(n) + if err == nil { + nn = mem.mdb.Free() + } else { + nn = 0 + } } - mem, err = d.rotateMem(n) - nn = mem.Free() return false } return true @@ -100,7 +109,12 @@ func (d *DB) flush(n int) (mem *memdb.DB, nn int, err error) { for flush() { } if delayed { - s.logf("db@write delayed T·%v", time.Since(start)) + db.writeDelay += time.Since(start) + db.writeDelayN++ + } else if db.writeDelayN > 0 { + db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay) + db.writeDelay = 0 + db.writeDelayN = 0 } return } @@ -109,39 +123,45 @@ func (d *DB) flush(n int) (mem *memdb.DB, nn int, err error) { // sequentially. // // It is safe to modify the contents of the arguments after Write returns. -func (d *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) { - err = d.ok() - if err != nil || b == nil || b.len() == 0 { +func (db *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) { + err = db.ok() + if err != nil || b == nil || b.Len() == 0 { return } b.init(wo.GetSync()) // The write happen synchronously. -retry: select { - case d.writeC <- b: - if <-d.writeMergedC { - return <-d.writeAckC + case db.writeC <- b: + if <-db.writeMergedC { + return <-db.writeAckC } - goto retry - case d.writeLockC <- struct{}{}: - case _, _ = <-d.closeC: + case db.writeLockC <- struct{}{}: + case err = <-db.compPerErrC: + return + case _, _ = <-db.closeC: return ErrClosed } merged := 0 + danglingMerge := false defer func() { - <-d.writeLockC + if danglingMerge { + db.writeMergedC <- false + } else { + <-db.writeLockC + } for i := 0; i < merged; i++ { - d.writeAckC <- err + db.writeAckC <- err } }() - mem, memFree, err := d.flush(b.size()) + mem, memFree, err := db.flush(b.size()) if err != nil { return } + defer mem.decref() // Calculate maximum size of the batch. m := 1 << 20 @@ -154,13 +174,13 @@ retry: drain: for b.size() < m && !b.sync { select { - case nb := <-d.writeC: + case nb := <-db.writeC: if b.size()+nb.size() <= m { b.append(nb) - d.writeMergedC <- true + db.writeMergedC <- true merged++ } else { - d.writeMergedC <- false + danglingMerge = true break drain } default: @@ -169,44 +189,52 @@ drain: } // Set batch first seq number relative from last seq. - b.seq = d.seq + 1 + b.seq = db.seq + 1 // Write journal concurrently if it is large enough. if b.size() >= (128 << 10) { // Push the write batch to the journal writer select { - case _, _ = <-d.closeC: + case db.journalC <- b: + // Write into memdb + if berr := b.memReplay(mem.mdb); berr != nil { + panic(berr) + } + case err = <-db.compPerErrC: + return + case _, _ = <-db.closeC: err = ErrClosed return - case d.journalC <- b: - // Write into memdb - b.memReplay(mem) } // Wait for journal writer select { - case _, _ = <-d.closeC: - err = ErrClosed - return - case err = <-d.journalAckC: + case err = <-db.journalAckC: if err != nil { // Revert memdb if error detected - b.revertMemReplay(mem) + if berr := b.revertMemReplay(mem.mdb); berr != nil { + panic(berr) + } return } + case _, _ = <-db.closeC: + err = ErrClosed + return } } else { - err = d.writeJournal(b) + err = db.writeJournal(b) if err != nil { return } - b.memReplay(mem) + if berr := b.memReplay(mem.mdb); berr != nil { + panic(berr) + } } // Set last seq number. - d.addSeq(uint64(b.len())) + db.addSeq(uint64(b.Len())) if b.size() >= memFree { - d.rotateMem(0) + db.rotateMem(0) } return } @@ -215,20 +243,20 @@ drain: // for that key; a DB is not a multi-map. // // It is safe to modify the contents of the arguments after Put returns. -func (d *DB) Put(key, value []byte, wo *opt.WriteOptions) error { +func (db *DB) Put(key, value []byte, wo *opt.WriteOptions) error { b := new(Batch) b.Put(key, value) - return d.Write(b, wo) + return db.Write(b, wo) } // Delete deletes the value for the given key. It returns ErrNotFound if // the DB does not contain the key. // // It is safe to modify the contents of the arguments after Delete returns. -func (d *DB) Delete(key []byte, wo *opt.WriteOptions) error { +func (db *DB) Delete(key []byte, wo *opt.WriteOptions) error { b := new(Batch) b.Delete(key) - return d.Write(b, wo) + return db.Write(b, wo) } func isMemOverlaps(icmp *iComparer, mem *memdb.DB, min, max []byte) bool { @@ -247,33 +275,37 @@ func isMemOverlaps(icmp *iComparer, mem *memdb.DB, min, max []byte) bool { // A nil Range.Start is treated as a key before all keys in the DB. // And a nil Range.Limit is treated as a key after all keys in the DB. // Therefore if both is nil then it will compact entire DB. -func (d *DB) CompactRange(r util.Range) error { - if err := d.ok(); err != nil { +func (db *DB) CompactRange(r util.Range) error { + if err := db.ok(); err != nil { return err } + // Lock writer. select { - case d.writeLockC <- struct{}{}: - case _, _ = <-d.closeC: + case db.writeLockC <- struct{}{}: + case err := <-db.compPerErrC: + return err + case _, _ = <-db.closeC: return ErrClosed } // Check for overlaps in memdb. - mem := d.getEffectiveMem() - if isMemOverlaps(d.s.icmp, mem, r.Start, r.Limit) { + mem := db.getEffectiveMem() + defer mem.decref() + if isMemOverlaps(db.s.icmp, mem.mdb, r.Start, r.Limit) { // Memdb compaction. - if _, err := d.rotateMem(0); err != nil { - <-d.writeLockC + if _, err := db.rotateMem(0); err != nil { + <-db.writeLockC return err } - <-d.writeLockC - if err := d.compSendIdle(d.mcompCmdC); err != nil { + <-db.writeLockC + if err := db.compSendIdle(db.mcompCmdC); err != nil { return err } } else { - <-d.writeLockC + <-db.writeLockC } // Table compaction. - return d.compSendRange(d.tcompCmdC, -1, r.Start, r.Limit) + return db.compSendRange(db.tcompCmdC, -1, r.Start, r.Limit) } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/doc.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/doc.go index ac9ea3d0c..53f13bb24 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/doc.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/doc.go @@ -37,6 +37,16 @@ // err = iter.Error() // ... // +// Iterate over subset of database content with a particular prefix: +// iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil) +// for iter.Next() { +// // Use key/value. +// ... +// } +// iter.Release() +// err = iter.Error() +// ... +// // Seek-then-Iterate: // // iter := db.NewIterator(nil, nil) diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/error.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors.go index 8066bd9a4..29d0d2f27 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/error.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors.go @@ -7,32 +7,12 @@ package leveldb import ( - "errors" - - "github.com/syndtr/goleveldb/leveldb/util" + "github.com/syndtr/goleveldb/leveldb/errors" ) var ( - ErrNotFound = util.ErrNotFound + ErrNotFound = errors.ErrNotFound ErrSnapshotReleased = errors.New("leveldb: snapshot released") ErrIterReleased = errors.New("leveldb: iterator released") ErrClosed = errors.New("leveldb: closed") ) - -type CorruptionType int - -const ( - CorruptedManifest CorruptionType = iota - MissingFiles -) - -// ErrCorrupted is the type that wraps errors that indicate corruption in -// the database. -type ErrCorrupted struct { - Type CorruptionType - Err error -} - -func (e ErrCorrupted) Error() string { - return e.Err.Error() -} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors/errors.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors/errors.go new file mode 100644 index 000000000..84b5d6b7b --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors/errors.go @@ -0,0 +1,76 @@ +// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com> +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Package errors provides common error types used throughout leveldb. +package errors + +import ( + "errors" + "fmt" + + "github.com/syndtr/goleveldb/leveldb/storage" + "github.com/syndtr/goleveldb/leveldb/util" +) + +var ( + ErrNotFound = New("leveldb: not found") + ErrReleased = util.ErrReleased + ErrHasReleaser = util.ErrHasReleaser +) + +// New returns an error that formats as the given text. +func New(text string) error { + return errors.New(text) +} + +// ErrCorrupted is the type that wraps errors that indicate corruption in +// the database. +type ErrCorrupted struct { + File *storage.FileInfo + Err error +} + +func (e *ErrCorrupted) Error() string { + if e.File != nil { + return fmt.Sprintf("%v [file=%v]", e.Err, e.File) + } else { + return e.Err.Error() + } +} + +// NewErrCorrupted creates new ErrCorrupted error. +func NewErrCorrupted(f storage.File, err error) error { + return &ErrCorrupted{storage.NewFileInfo(f), err} +} + +// IsCorrupted returns a boolean indicating whether the error is indicating +// a corruption. +func IsCorrupted(err error) bool { + switch err.(type) { + case *ErrCorrupted: + return true + } + return false +} + +// ErrMissingFiles is the type that indicating a corruption due to missing +// files. +type ErrMissingFiles struct { + Files []*storage.FileInfo +} + +func (e *ErrMissingFiles) Error() string { return "file missing" } + +// SetFile sets 'file info' of the given error with the given file. +// Currently only ErrCorrupted is supported, otherwise will do nothing. +func SetFile(err error, f storage.File) error { + switch x := err.(type) { + case *ErrCorrupted: + x.File = storage.NewFileInfo(f) + return x + } + return err +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go index d7dff04b6..b328ece4e 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go @@ -17,13 +17,14 @@ import ( var _ = testutil.Defer(func() { Describe("Leveldb external", func() { o := &opt.Options{ - BlockCache: opt.NoCache, - BlockRestartInterval: 5, - BlockSize: 50, - Compression: opt.NoCompression, - MaxOpenFiles: 0, - Strict: opt.StrictAll, - WriteBuffer: 1000, + DisableBlockCache: true, + BlockRestartInterval: 5, + BlockSize: 80, + Compression: opt.NoCompression, + OpenFilesCacheCapacity: -1, + Strict: opt.StrictAll, + WriteBuffer: 1000, + CompactionTableSize: 2000, } Describe("write test", func() { @@ -36,22 +37,21 @@ var _ = testutil.Defer(func() { testutil.DoDBTesting(&t) db.TestClose() done <- true - }, 9.0) + }, 20.0) }) Describe("read test", func() { - testutil.AllKeyValueTesting(nil, func(kv testutil.KeyValue) testutil.DB { + testutil.AllKeyValueTesting(nil, nil, func(kv testutil.KeyValue) testutil.DB { // Building the DB. db := newTestingDB(o, nil, nil) kv.IterateShuffled(nil, func(i int, key, value []byte) { err := db.TestPut(key, value) Expect(err).NotTo(HaveOccurred()) }) - testutil.Defer("teardown", func() { - db.TestClose() - }) return db + }, func(db testutil.DB) { + db.(*testingDB).TestClose() }) }) }) diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go index 9b4b72741..a23ab05f7 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go @@ -40,13 +40,19 @@ type basicArrayIterator struct { util.BasicReleaser array BasicArray pos int + err error } func (i *basicArrayIterator) Valid() bool { - return i.pos >= 0 && i.pos < i.array.Len() + return i.pos >= 0 && i.pos < i.array.Len() && !i.Released() } func (i *basicArrayIterator) First() bool { + if i.Released() { + i.err = ErrIterReleased + return false + } + if i.array.Len() == 0 { i.pos = -1 return false @@ -56,6 +62,11 @@ func (i *basicArrayIterator) First() bool { } func (i *basicArrayIterator) Last() bool { + if i.Released() { + i.err = ErrIterReleased + return false + } + n := i.array.Len() if n == 0 { i.pos = 0 @@ -66,6 +77,11 @@ func (i *basicArrayIterator) Last() bool { } func (i *basicArrayIterator) Seek(key []byte) bool { + if i.Released() { + i.err = ErrIterReleased + return false + } + n := i.array.Len() if n == 0 { i.pos = 0 @@ -79,6 +95,11 @@ func (i *basicArrayIterator) Seek(key []byte) bool { } func (i *basicArrayIterator) Next() bool { + if i.Released() { + i.err = ErrIterReleased + return false + } + i.pos++ if n := i.array.Len(); i.pos >= n { i.pos = n @@ -88,6 +109,11 @@ func (i *basicArrayIterator) Next() bool { } func (i *basicArrayIterator) Prev() bool { + if i.Released() { + i.err = ErrIterReleased + return false + } + i.pos-- if i.pos < 0 { i.pos = -1 @@ -96,7 +122,7 @@ func (i *basicArrayIterator) Prev() bool { return true } -func (i *basicArrayIterator) Error() error { return nil } +func (i *basicArrayIterator) Error() error { return i.err } type arrayIterator struct { basicArrayIterator diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go index 1e99a2bf6..939adbb93 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go @@ -7,6 +7,7 @@ package iterator import ( + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/util" ) @@ -22,13 +23,13 @@ type IteratorIndexer interface { type indexedIterator struct { util.BasicReleaser - index IteratorIndexer - strict bool - strictGet bool + index IteratorIndexer + strict bool - data Iterator - err error - errf func(err error) + data Iterator + err error + errf func(err error) + closed bool } func (i *indexedIterator) setData() { @@ -36,11 +37,6 @@ func (i *indexedIterator) setData() { i.data.Release() } i.data = i.index.Get() - if i.strictGet { - if err := i.data.Error(); err != nil { - i.err = err - } - } } func (i *indexedIterator) clearData() { @@ -50,14 +46,21 @@ func (i *indexedIterator) clearData() { i.data = nil } -func (i *indexedIterator) dataErr() bool { - if i.errf != nil { - if err := i.data.Error(); err != nil { +func (i *indexedIterator) indexErr() { + if err := i.index.Error(); err != nil { + if i.errf != nil { i.errf(err) } + i.err = err } - if i.strict { - if err := i.data.Error(); err != nil { +} + +func (i *indexedIterator) dataErr() bool { + if err := i.data.Error(); err != nil { + if i.errf != nil { + i.errf(err) + } + if i.strict || !errors.IsCorrupted(err) { i.err = err return true } @@ -72,9 +75,13 @@ func (i *indexedIterator) Valid() bool { func (i *indexedIterator) First() bool { if i.err != nil { return false + } else if i.Released() { + i.err = ErrIterReleased + return false } if !i.index.First() { + i.indexErr() i.clearData() return false } @@ -85,9 +92,13 @@ func (i *indexedIterator) First() bool { func (i *indexedIterator) Last() bool { if i.err != nil { return false + } else if i.Released() { + i.err = ErrIterReleased + return false } if !i.index.Last() { + i.indexErr() i.clearData() return false } @@ -105,9 +116,13 @@ func (i *indexedIterator) Last() bool { func (i *indexedIterator) Seek(key []byte) bool { if i.err != nil { return false + } else if i.Released() { + i.err = ErrIterReleased + return false } if !i.index.Seek(key) { + i.indexErr() i.clearData() return false } @@ -125,6 +140,9 @@ func (i *indexedIterator) Seek(key []byte) bool { func (i *indexedIterator) Next() bool { if i.err != nil { return false + } else if i.Released() { + i.err = ErrIterReleased + return false } switch { @@ -136,6 +154,7 @@ func (i *indexedIterator) Next() bool { fallthrough case i.data == nil: if !i.index.Next() { + i.indexErr() return false } i.setData() @@ -147,6 +166,9 @@ func (i *indexedIterator) Next() bool { func (i *indexedIterator) Prev() bool { if i.err != nil { return false + } else if i.Released() { + i.err = ErrIterReleased + return false } switch { @@ -158,6 +180,7 @@ func (i *indexedIterator) Prev() bool { fallthrough case i.data == nil: if !i.index.Prev() { + i.indexErr() return false } i.setData() @@ -206,16 +229,14 @@ func (i *indexedIterator) SetErrorCallback(f func(err error)) { i.errf = f } -// NewIndexedIterator returns an indexed iterator. An index is iterator -// that returns another iterator, a data iterator. A data iterator is the +// NewIndexedIterator returns an 'indexed iterator'. An index is iterator +// that returns another iterator, a 'data iterator'. A 'data iterator' is the // iterator that contains actual key/value pairs. // -// If strict is true then error yield by data iterator will halt the indexed -// iterator, on contrary if strict is false then the indexed iterator will -// ignore those error and move on to the next index. If strictGet is true and -// index.Get() yield an 'error iterator' then the indexed iterator will be halted. -// An 'error iterator' is iterator which its Error() method always return non-nil -// even before any 'seeks method' is called. -func NewIndexedIterator(index IteratorIndexer, strict, strictGet bool) Iterator { - return &indexedIterator{index: index, strict: strict, strictGet: strictGet} +// If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true) +// won't be ignored and will halt 'indexed iterator', otherwise the iterator will +// continue to the next 'data iterator'. Corruption on 'index iterator' will not be +// ignored and will halt the iterator. +func NewIndexedIterator(index IteratorIndexer, strict bool) Iterator { + return &indexedIterator{index: index, strict: strict} } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go index 6a89b3830..72a797892 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go @@ -65,7 +65,7 @@ var _ = testutil.Defer(func() { // Test the iterator. t := testutil.IteratorTesting{ KeyValue: kv.Clone(), - Iter: NewIndexedIterator(NewArrayIndexer(index), true, true), + Iter: NewIndexedIterator(NewArrayIndexer(index), true), } testutil.DoIteratorTesting(&t) done <- true diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter.go index 1b80184e8..c2522860b 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter.go @@ -14,6 +14,10 @@ import ( "github.com/syndtr/goleveldb/leveldb/util" ) +var ( + ErrIterReleased = errors.New("leveldb/iterator: iterator released") +) + // IteratorSeeker is the interface that wraps the 'seeks method'. type IteratorSeeker interface { // First moves the iterator to the first key/value pair. If the iterator @@ -100,28 +104,13 @@ type ErrorCallbackSetter interface { } type emptyIterator struct { - releaser util.Releaser - released bool - err error + util.BasicReleaser + err error } func (i *emptyIterator) rErr() { - if i.err == nil && i.released { - i.err = errors.New("leveldb/iterator: iterator released") - } -} - -func (i *emptyIterator) Release() { - if i.releaser != nil { - i.releaser.Release() - i.releaser = nil - } - i.released = true -} - -func (i *emptyIterator) SetReleaser(releaser util.Releaser) { - if !i.released { - i.releaser = releaser + if i.err == nil && i.Released() { + i.err = ErrIterReleased } } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go index 7ec2fc6f2..5ef8d5baf 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go @@ -3,15 +3,9 @@ package iterator_test import ( "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/syndtr/goleveldb/leveldb/testutil" ) func TestIterator(t *testing.T) { - testutil.RunDefer() - - RegisterFailHandler(Fail) - RunSpecs(t, "Iterator Suite") + testutil.RunSuite(t, "Iterator Suite") } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go index c8314c4e5..1a7e29df8 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go @@ -7,16 +7,11 @@ package iterator import ( - "errors" - "github.com/syndtr/goleveldb/leveldb/comparer" + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/util" ) -var ( - ErrIterReleased = errors.New("leveldb/iterator: iterator released") -) - type dir int const ( @@ -48,13 +43,11 @@ func assertKey(key []byte) []byte { } func (i *mergedIterator) iterErr(iter Iterator) bool { - if i.errf != nil { - if err := iter.Error(); err != nil { + if err := iter.Error(); err != nil { + if i.errf != nil { i.errf(err) } - } - if i.strict { - if err := iter.Error(); err != nil { + if i.strict || !errors.IsCorrupted(err) { i.err = err return true } @@ -274,9 +267,13 @@ func (i *mergedIterator) Release() { } func (i *mergedIterator) SetReleaser(releaser util.Releaser) { - if i.dir != dirReleased { - i.releaser = releaser + if i.dir == dirReleased { + panic(util.ErrReleased) + } + if i.releaser != nil && releaser != nil { + panic(util.ErrHasReleaser) } + i.releaser = releaser } func (i *mergedIterator) Error() error { @@ -294,9 +291,9 @@ func (i *mergedIterator) SetErrorCallback(f func(err error)) { // keys: if iters[i] contains a key k then iters[j] will not contain that key k. // None of the iters may be nil. // -// If strict is true then error yield by any iterators will halt the merged -// iterator, on contrary if strict is false then the merged iterator will -// ignore those error and move on to the next iterator. +// If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true) +// won't be ignored and will halt 'merged iterator', otherwise the iterator will +// continue to the next 'input iterator'. func NewMergedIterator(iters []Iterator, cmp comparer.Comparer, strict bool) Iterator { return &mergedIterator{ iters: iters, diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal.go index b522c76e6..6519ec660 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal.go @@ -79,10 +79,10 @@ package journal import ( "encoding/binary" - "errors" "fmt" "io" + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/util" ) @@ -103,18 +103,18 @@ type flusher interface { Flush() error } -// DroppedError is the error type that passed to Dropper.Drop method. -type DroppedError struct { +// ErrCorrupted is the error type that generated by corrupted block or chunk. +type ErrCorrupted struct { Size int Reason string } -func (e DroppedError) Error() string { - return fmt.Sprintf("leveldb/journal: dropped %d bytes: %s", e.Size, e.Reason) +func (e *ErrCorrupted) Error() string { + return fmt.Sprintf("leveldb/journal: block/chunk corrupted: %s (%d bytes)", e.Reason, e.Size) } // Dropper is the interface that wrap simple Drop method. The Drop -// method will be called when the journal reader dropping a chunk. +// method will be called when the journal reader dropping a block or chunk. type Dropper interface { Drop(err error) } @@ -158,76 +158,78 @@ func NewReader(r io.Reader, dropper Dropper, strict, checksum bool) *Reader { } } +var errSkip = errors.New("leveldb/journal: skipped") + +func (r *Reader) corrupt(n int, reason string, skip bool) error { + if r.dropper != nil { + r.dropper.Drop(&ErrCorrupted{n, reason}) + } + if r.strict && !skip { + r.err = errors.NewErrCorrupted(nil, &ErrCorrupted{n, reason}) + return r.err + } + return errSkip +} + // nextChunk sets r.buf[r.i:r.j] to hold the next chunk's payload, reading the // next block into the buffer if necessary. -func (r *Reader) nextChunk(wantFirst, skip bool) error { +func (r *Reader) nextChunk(first bool) error { for { if r.j+headerSize <= r.n { checksum := binary.LittleEndian.Uint32(r.buf[r.j+0 : r.j+4]) length := binary.LittleEndian.Uint16(r.buf[r.j+4 : r.j+6]) chunkType := r.buf[r.j+6] - var err error if checksum == 0 && length == 0 && chunkType == 0 { // Drop entire block. - err = DroppedError{r.n - r.j, "zero header"} + m := r.n - r.j r.i = r.n r.j = r.n + return r.corrupt(m, "zero header", false) } else { m := r.n - r.j r.i = r.j + headerSize r.j = r.j + headerSize + int(length) if r.j > r.n { // Drop entire block. - err = DroppedError{m, "chunk length overflows block"} r.i = r.n r.j = r.n + return r.corrupt(m, "chunk length overflows block", false) } else if r.checksum && checksum != util.NewCRC(r.buf[r.i-1:r.j]).Value() { // Drop entire block. - err = DroppedError{m, "checksum mismatch"} r.i = r.n r.j = r.n + return r.corrupt(m, "checksum mismatch", false) } } - if wantFirst && err == nil && chunkType != fullChunkType && chunkType != firstChunkType { - if skip { - // The chunk are intentionally skipped. - if chunkType == lastChunkType { - skip = false - } - continue - } else { - // Drop the chunk. - err = DroppedError{r.j - r.i + headerSize, "orphan chunk"} - } - } - if err == nil { - r.last = chunkType == fullChunkType || chunkType == lastChunkType - } else { - if r.dropper != nil { - r.dropper.Drop(err) - } - if r.strict { - r.err = err - } + if first && chunkType != fullChunkType && chunkType != firstChunkType { + m := r.j - r.i + r.i = r.j + // Report the error, but skip it. + return r.corrupt(m+headerSize, "orphan chunk", true) } - return err + r.last = chunkType == fullChunkType || chunkType == lastChunkType + return nil } + + // The last block. if r.n < blockSize && r.n > 0 { - // This is the last block. - if r.j != r.n { - r.err = io.ErrUnexpectedEOF - } else { - r.err = io.EOF + if !first { + return r.corrupt(0, "missing chunk part", false) } + r.err = io.EOF return r.err } + + // Read block. n, err := io.ReadFull(r.r, r.buf[:]) - if err != nil && err != io.ErrUnexpectedEOF { - r.err = err - return r.err + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return err } if n == 0 { + if !first { + return r.corrupt(0, "missing chunk part", false) + } r.err = io.EOF return r.err } @@ -237,29 +239,26 @@ func (r *Reader) nextChunk(wantFirst, skip bool) error { // Next returns a reader for the next journal. It returns io.EOF if there are no // more journals. The reader returned becomes stale after the next Next call, -// and should no longer be used. +// and should no longer be used. If strict is false, the reader will returns +// io.ErrUnexpectedEOF error when found corrupted journal. func (r *Reader) Next() (io.Reader, error) { r.seq++ if r.err != nil { return nil, r.err } - skip := !r.last + r.i = r.j for { - r.i = r.j - if r.nextChunk(true, skip) != nil { - // So that 'orphan chunk' drop will be reported. - skip = false - } else { + if err := r.nextChunk(true); err == nil { break - } - if r.err != nil { - return nil, r.err + } else if err != errSkip { + return nil, err } } return &singleReader{r, r.seq, nil}, nil } -// Reset resets the journal reader, allows reuse of the journal reader. +// Reset resets the journal reader, allows reuse of the journal reader. Reset returns +// last accumulated error. func (r *Reader) Reset(reader io.Reader, dropper Dropper, strict, checksum bool) error { r.seq++ err := r.err @@ -296,7 +295,11 @@ func (x *singleReader) Read(p []byte) (int, error) { if r.last { return 0, io.EOF } - if x.err = r.nextChunk(false, false); x.err != nil { + x.err = r.nextChunk(false) + if x.err != nil { + if x.err == errSkip { + x.err = io.ErrUnexpectedEOF + } return 0, x.err } } @@ -320,7 +323,11 @@ func (x *singleReader) ReadByte() (byte, error) { if r.last { return 0, io.EOF } - if x.err = r.nextChunk(false, false); x.err != nil { + x.err = r.nextChunk(false) + if x.err != nil { + if x.err == errSkip { + x.err = io.ErrUnexpectedEOF + } return 0, x.err } } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go index 5e1193ae2..0fcf22599 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go @@ -12,6 +12,7 @@ package journal import ( "bytes" + "encoding/binary" "fmt" "io" "io/ioutil" @@ -326,3 +327,492 @@ func TestStaleWriter(t *testing.T) { t.Fatalf("stale write #1: unexpected error: %v", err) } } + +func TestCorrupt_MissingLastBlock(t *testing.T) { + buf := new(bytes.Buffer) + + w := NewWriter(buf) + + // First record. + ww, err := w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-1024)); err != nil { + t.Fatalf("write #0: unexpected error: %v", err) + } + + // Second record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil { + t.Fatalf("write #1: unexpected error: %v", err) + } + + if err := w.Close(); err != nil { + t.Fatal(err) + } + + // Cut the last block. + b := buf.Bytes()[:blockSize] + r := NewReader(bytes.NewReader(b), dropper{t}, false, true) + + // First read. + rr, err := r.Next() + if err != nil { + t.Fatal(err) + } + n, err := io.Copy(ioutil.Discard, rr) + if err != nil { + t.Fatalf("read #0: %v", err) + } + if n != blockSize-1024 { + t.Fatalf("read #0: got %d bytes want %d", n, blockSize-1024) + } + + // Second read. + rr, err = r.Next() + if err != nil { + t.Fatal(err) + } + n, err = io.Copy(ioutil.Discard, rr) + if err != io.ErrUnexpectedEOF { + t.Fatalf("read #1: unexpected error: %v", err) + } + + if _, err := r.Next(); err != io.EOF { + t.Fatalf("last next: unexpected error: %v", err) + } +} + +func TestCorrupt_CorruptedFirstBlock(t *testing.T) { + buf := new(bytes.Buffer) + + w := NewWriter(buf) + + // First record. + ww, err := w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil { + t.Fatalf("write #0: unexpected error: %v", err) + } + + // Second record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil { + t.Fatalf("write #1: unexpected error: %v", err) + } + + // Third record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil { + t.Fatalf("write #2: unexpected error: %v", err) + } + + // Fourth record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil { + t.Fatalf("write #3: unexpected error: %v", err) + } + + if err := w.Close(); err != nil { + t.Fatal(err) + } + + b := buf.Bytes() + // Corrupting block #0. + for i := 0; i < 1024; i++ { + b[i] = '1' + } + + r := NewReader(bytes.NewReader(b), dropper{t}, false, true) + + // First read (third record). + rr, err := r.Next() + if err != nil { + t.Fatal(err) + } + n, err := io.Copy(ioutil.Discard, rr) + if err != nil { + t.Fatalf("read #0: %v", err) + } + if want := int64(blockSize-headerSize) + 1; n != want { + t.Fatalf("read #0: got %d bytes want %d", n, want) + } + + // Second read (fourth record). + rr, err = r.Next() + if err != nil { + t.Fatal(err) + } + n, err = io.Copy(ioutil.Discard, rr) + if err != nil { + t.Fatalf("read #1: %v", err) + } + if want := int64(blockSize-headerSize) + 2; n != want { + t.Fatalf("read #1: got %d bytes want %d", n, want) + } + + if _, err := r.Next(); err != io.EOF { + t.Fatalf("last next: unexpected error: %v", err) + } +} + +func TestCorrupt_CorruptedMiddleBlock(t *testing.T) { + buf := new(bytes.Buffer) + + w := NewWriter(buf) + + // First record. + ww, err := w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil { + t.Fatalf("write #0: unexpected error: %v", err) + } + + // Second record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil { + t.Fatalf("write #1: unexpected error: %v", err) + } + + // Third record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil { + t.Fatalf("write #2: unexpected error: %v", err) + } + + // Fourth record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil { + t.Fatalf("write #3: unexpected error: %v", err) + } + + if err := w.Close(); err != nil { + t.Fatal(err) + } + + b := buf.Bytes() + // Corrupting block #1. + for i := 0; i < 1024; i++ { + b[blockSize+i] = '1' + } + + r := NewReader(bytes.NewReader(b), dropper{t}, false, true) + + // First read (first record). + rr, err := r.Next() + if err != nil { + t.Fatal(err) + } + n, err := io.Copy(ioutil.Discard, rr) + if err != nil { + t.Fatalf("read #0: %v", err) + } + if want := int64(blockSize / 2); n != want { + t.Fatalf("read #0: got %d bytes want %d", n, want) + } + + // Second read (second record). + rr, err = r.Next() + if err != nil { + t.Fatal(err) + } + n, err = io.Copy(ioutil.Discard, rr) + if err != io.ErrUnexpectedEOF { + t.Fatalf("read #1: unexpected error: %v", err) + } + + // Third read (fourth record). + rr, err = r.Next() + if err != nil { + t.Fatal(err) + } + n, err = io.Copy(ioutil.Discard, rr) + if err != nil { + t.Fatalf("read #2: %v", err) + } + if want := int64(blockSize-headerSize) + 2; n != want { + t.Fatalf("read #2: got %d bytes want %d", n, want) + } + + if _, err := r.Next(); err != io.EOF { + t.Fatalf("last next: unexpected error: %v", err) + } +} + +func TestCorrupt_CorruptedLastBlock(t *testing.T) { + buf := new(bytes.Buffer) + + w := NewWriter(buf) + + // First record. + ww, err := w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil { + t.Fatalf("write #0: unexpected error: %v", err) + } + + // Second record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil { + t.Fatalf("write #1: unexpected error: %v", err) + } + + // Third record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil { + t.Fatalf("write #2: unexpected error: %v", err) + } + + // Fourth record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil { + t.Fatalf("write #3: unexpected error: %v", err) + } + + if err := w.Close(); err != nil { + t.Fatal(err) + } + + b := buf.Bytes() + // Corrupting block #3. + for i := len(b) - 1; i > len(b)-1024; i-- { + b[i] = '1' + } + + r := NewReader(bytes.NewReader(b), dropper{t}, false, true) + + // First read (first record). + rr, err := r.Next() + if err != nil { + t.Fatal(err) + } + n, err := io.Copy(ioutil.Discard, rr) + if err != nil { + t.Fatalf("read #0: %v", err) + } + if want := int64(blockSize / 2); n != want { + t.Fatalf("read #0: got %d bytes want %d", n, want) + } + + // Second read (second record). + rr, err = r.Next() + if err != nil { + t.Fatal(err) + } + n, err = io.Copy(ioutil.Discard, rr) + if err != nil { + t.Fatalf("read #1: %v", err) + } + if want := int64(blockSize - headerSize); n != want { + t.Fatalf("read #1: got %d bytes want %d", n, want) + } + + // Third read (third record). + rr, err = r.Next() + if err != nil { + t.Fatal(err) + } + n, err = io.Copy(ioutil.Discard, rr) + if err != nil { + t.Fatalf("read #2: %v", err) + } + if want := int64(blockSize-headerSize) + 1; n != want { + t.Fatalf("read #2: got %d bytes want %d", n, want) + } + + // Fourth read (fourth record). + rr, err = r.Next() + if err != nil { + t.Fatal(err) + } + n, err = io.Copy(ioutil.Discard, rr) + if err != io.ErrUnexpectedEOF { + t.Fatalf("read #3: unexpected error: %v", err) + } + + if _, err := r.Next(); err != io.EOF { + t.Fatalf("last next: unexpected error: %v", err) + } +} + +func TestCorrupt_FirstChuckLengthOverflow(t *testing.T) { + buf := new(bytes.Buffer) + + w := NewWriter(buf) + + // First record. + ww, err := w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil { + t.Fatalf("write #0: unexpected error: %v", err) + } + + // Second record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil { + t.Fatalf("write #1: unexpected error: %v", err) + } + + // Third record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil { + t.Fatalf("write #2: unexpected error: %v", err) + } + + if err := w.Close(); err != nil { + t.Fatal(err) + } + + b := buf.Bytes() + // Corrupting record #1. + x := blockSize + binary.LittleEndian.PutUint16(b[x+4:], 0xffff) + + r := NewReader(bytes.NewReader(b), dropper{t}, false, true) + + // First read (first record). + rr, err := r.Next() + if err != nil { + t.Fatal(err) + } + n, err := io.Copy(ioutil.Discard, rr) + if err != nil { + t.Fatalf("read #0: %v", err) + } + if want := int64(blockSize / 2); n != want { + t.Fatalf("read #0: got %d bytes want %d", n, want) + } + + // Second read (second record). + rr, err = r.Next() + if err != nil { + t.Fatal(err) + } + n, err = io.Copy(ioutil.Discard, rr) + if err != io.ErrUnexpectedEOF { + t.Fatalf("read #1: unexpected error: %v", err) + } + + if _, err := r.Next(); err != io.EOF { + t.Fatalf("last next: unexpected error: %v", err) + } +} + +func TestCorrupt_MiddleChuckLengthOverflow(t *testing.T) { + buf := new(bytes.Buffer) + + w := NewWriter(buf) + + // First record. + ww, err := w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil { + t.Fatalf("write #0: unexpected error: %v", err) + } + + // Second record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil { + t.Fatalf("write #1: unexpected error: %v", err) + } + + // Third record. + ww, err = w.Next() + if err != nil { + t.Fatal(err) + } + if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil { + t.Fatalf("write #2: unexpected error: %v", err) + } + + if err := w.Close(); err != nil { + t.Fatal(err) + } + + b := buf.Bytes() + // Corrupting record #1. + x := blockSize/2 + headerSize + binary.LittleEndian.PutUint16(b[x+4:], 0xffff) + + r := NewReader(bytes.NewReader(b), dropper{t}, false, true) + + // First read (first record). + rr, err := r.Next() + if err != nil { + t.Fatal(err) + } + n, err := io.Copy(ioutil.Discard, rr) + if err != nil { + t.Fatalf("read #0: %v", err) + } + if want := int64(blockSize / 2); n != want { + t.Fatalf("read #0: got %d bytes want %d", n, want) + } + + // Second read (third record). + rr, err = r.Next() + if err != nil { + t.Fatal(err) + } + n, err = io.Copy(ioutil.Discard, rr) + if err != nil { + t.Fatalf("read #1: %v", err) + } + if want := int64(blockSize-headerSize) + 1; n != want { + t.Fatalf("read #1: got %d bytes want %d", n, want) + } + + if _, err := r.Next(); err != io.EOF { + t.Fatalf("last next: unexpected error: %v", err) + } +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key.go index b9acf932d..572ae8150 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key.go @@ -9,15 +9,30 @@ package leveldb import ( "encoding/binary" "fmt" + + "github.com/syndtr/goleveldb/leveldb/errors" ) -type vType int +type ErrIkeyCorrupted struct { + Ikey []byte + Reason string +} + +func (e *ErrIkeyCorrupted) Error() string { + return fmt.Sprintf("leveldb: iKey %q corrupted: %s", e.Ikey, e.Reason) +} + +func newErrIkeyCorrupted(ikey []byte, reason string) error { + return errors.NewErrCorrupted(nil, &ErrIkeyCorrupted{append([]byte{}, ikey...), reason}) +} + +type kType int -func (t vType) String() string { - switch t { - case tDel: +func (kt kType) String() string { + switch kt { + case ktDel: return "d" - case tVal: + case ktVal: return "v" } return "x" @@ -26,16 +41,16 @@ func (t vType) String() string { // Value types encoded as the last component of internal keys. // Don't modify; this value are saved to disk. const ( - tDel vType = iota - tVal + ktDel kType = iota + ktVal ) -// tSeek defines the vType that should be passed when constructing an +// ktSeek defines the kType that should be passed when constructing an // internal key for seeking to a particular sequence number (since we // sort sequence numbers in decreasing order and the value type is // embedded as the low 8 bits in the sequence number in internal keys, // we need to use the highest-numbered ValueType, not the lowest). -const tSeek = tVal +const ktSeek = ktVal const ( // Maximum value possible for sequence number; the 8-bits are @@ -43,7 +58,7 @@ const ( // 64-bit integer. kMaxSeq uint64 = (uint64(1) << 56) - 1 // Maximum value possible for packed sequence number and type. - kMaxNum uint64 = (kMaxSeq << 8) | uint64(tSeek) + kMaxNum uint64 = (kMaxSeq << 8) | uint64(ktSeek) ) // Maximum number encoded in bytes. @@ -55,85 +70,73 @@ func init() { type iKey []byte -func newIKey(ukey []byte, seq uint64, t vType) iKey { - if seq > kMaxSeq || t > tVal { - panic("invalid seq number or value type") +func newIkey(ukey []byte, seq uint64, kt kType) iKey { + if seq > kMaxSeq { + panic("leveldb: invalid sequence number") + } else if kt > ktVal { + panic("leveldb: invalid type") } - b := make(iKey, len(ukey)+8) - copy(b, ukey) - binary.LittleEndian.PutUint64(b[len(ukey):], (seq<<8)|uint64(t)) - return b + ik := make(iKey, len(ukey)+8) + copy(ik, ukey) + binary.LittleEndian.PutUint64(ik[len(ukey):], (seq<<8)|uint64(kt)) + return ik } -func parseIkey(p []byte) (ukey []byte, seq uint64, t vType, ok bool) { - if len(p) < 8 { - return +func parseIkey(ik []byte) (ukey []byte, seq uint64, kt kType, err error) { + if len(ik) < 8 { + return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid length") } - num := binary.LittleEndian.Uint64(p[len(p)-8:]) - seq, t = uint64(num>>8), vType(num&0xff) - if t > tVal { - return + num := binary.LittleEndian.Uint64(ik[len(ik)-8:]) + seq, kt = uint64(num>>8), kType(num&0xff) + if kt > ktVal { + return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid type") } - ukey = p[:len(p)-8] - ok = true + ukey = ik[:len(ik)-8] return } -func validIkey(p []byte) bool { - _, _, _, ok := parseIkey(p) - return ok +func validIkey(ik []byte) bool { + _, _, _, err := parseIkey(ik) + return err == nil } -func (p iKey) assert() { - if p == nil { - panic("nil iKey") +func (ik iKey) assert() { + if ik == nil { + panic("leveldb: nil iKey") } - if len(p) < 8 { - panic(fmt.Sprintf("invalid iKey %q, len=%d", []byte(p), len(p))) + if len(ik) < 8 { + panic(fmt.Sprintf("leveldb: iKey %q, len=%d: invalid length", []byte(ik), len(ik))) } } -func (p iKey) ok() bool { - if len(p) < 8 { - return false - } - _, _, ok := p.parseNum() - return ok -} - -func (p iKey) ukey() []byte { - p.assert() - return p[:len(p)-8] +func (ik iKey) ukey() []byte { + ik.assert() + return ik[:len(ik)-8] } -func (p iKey) num() uint64 { - p.assert() - return binary.LittleEndian.Uint64(p[len(p)-8:]) +func (ik iKey) num() uint64 { + ik.assert() + return binary.LittleEndian.Uint64(ik[len(ik)-8:]) } -func (p iKey) parseNum() (seq uint64, t vType, ok bool) { - if p == nil { - panic("nil iKey") +func (ik iKey) parseNum() (seq uint64, kt kType) { + num := ik.num() + seq, kt = uint64(num>>8), kType(num&0xff) + if kt > ktVal { + panic(fmt.Sprintf("leveldb: iKey %q, len=%d: invalid type %#x", []byte(ik), len(ik), kt)) } - if len(p) < 8 { - return - } - num := p.num() - seq, t = uint64(num>>8), vType(num&0xff) - if t > tVal { - return 0, 0, false - } - ok = true return } -func (p iKey) String() string { - if len(p) == 0 { +func (ik iKey) String() string { + if ik == nil { return "<nil>" } - if seq, t, ok := p.parseNum(); ok { - return fmt.Sprintf("%s,%s%d", shorten(string(p.ukey())), t, seq) + + if ukey, seq, kt, err := parseIkey(ik); err == nil { + return fmt.Sprintf("%s,%s%d", shorten(string(ukey)), kt, seq) + } else { + return "<invalid>" } - return "<invalid>" } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key_test.go index e307cfc1d..30eadf784 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key_test.go @@ -15,8 +15,8 @@ import ( var defaultIComparer = &iComparer{comparer.DefaultComparer} -func ikey(key string, seq uint64, t vType) iKey { - return newIKey([]byte(key), uint64(seq), t) +func ikey(key string, seq uint64, kt kType) iKey { + return newIkey([]byte(key), uint64(seq), kt) } func shortSep(a, b []byte) []byte { @@ -37,27 +37,37 @@ func shortSuccessor(b []byte) []byte { return dst } -func testSingleKey(t *testing.T, key string, seq uint64, vt vType) { - ik := ikey(key, seq, vt) +func testSingleKey(t *testing.T, key string, seq uint64, kt kType) { + ik := ikey(key, seq, kt) if !bytes.Equal(ik.ukey(), []byte(key)) { t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key) } - if rseq, rt, ok := ik.parseNum(); ok { + rseq, rt := ik.parseNum() + if rseq != seq { + t.Errorf("seq number does not equal, got %v, want %v", rseq, seq) + } + if rt != kt { + t.Errorf("type does not equal, got %v, want %v", rt, kt) + } + + if rukey, rseq, rt, kerr := parseIkey(ik); kerr == nil { + if !bytes.Equal(rukey, []byte(key)) { + t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key) + } if rseq != seq { t.Errorf("seq number does not equal, got %v, want %v", rseq, seq) } - - if rt != vt { - t.Errorf("type does not equal, got %v, want %v", rt, vt) + if rt != kt { + t.Errorf("type does not equal, got %v, want %v", rt, kt) } } else { - t.Error("cannot parse seq and type") + t.Errorf("key error: %v", kerr) } } -func TestIKey_EncodeDecode(t *testing.T) { +func TestIkey_EncodeDecode(t *testing.T) { keys := []string{"", "k", "hello", "longggggggggggggggggggggg"} seqs := []uint64{ 1, 2, 3, @@ -67,8 +77,8 @@ func TestIKey_EncodeDecode(t *testing.T) { } for _, key := range keys { for _, seq := range seqs { - testSingleKey(t, key, seq, tVal) - testSingleKey(t, "hello", 1, tDel) + testSingleKey(t, key, seq, ktVal) + testSingleKey(t, "hello", 1, ktDel) } } } @@ -79,45 +89,45 @@ func assertBytes(t *testing.T, want, got []byte) { } } -func TestIKeyShortSeparator(t *testing.T) { +func TestIkeyShortSeparator(t *testing.T) { // When user keys are same - assertBytes(t, ikey("foo", 100, tVal), - shortSep(ikey("foo", 100, tVal), - ikey("foo", 99, tVal))) - assertBytes(t, ikey("foo", 100, tVal), - shortSep(ikey("foo", 100, tVal), - ikey("foo", 101, tVal))) - assertBytes(t, ikey("foo", 100, tVal), - shortSep(ikey("foo", 100, tVal), - ikey("foo", 100, tVal))) - assertBytes(t, ikey("foo", 100, tVal), - shortSep(ikey("foo", 100, tVal), - ikey("foo", 100, tDel))) + assertBytes(t, ikey("foo", 100, ktVal), + shortSep(ikey("foo", 100, ktVal), + ikey("foo", 99, ktVal))) + assertBytes(t, ikey("foo", 100, ktVal), + shortSep(ikey("foo", 100, ktVal), + ikey("foo", 101, ktVal))) + assertBytes(t, ikey("foo", 100, ktVal), + shortSep(ikey("foo", 100, ktVal), + ikey("foo", 100, ktVal))) + assertBytes(t, ikey("foo", 100, ktVal), + shortSep(ikey("foo", 100, ktVal), + ikey("foo", 100, ktDel))) // When user keys are misordered - assertBytes(t, ikey("foo", 100, tVal), - shortSep(ikey("foo", 100, tVal), - ikey("bar", 99, tVal))) + assertBytes(t, ikey("foo", 100, ktVal), + shortSep(ikey("foo", 100, ktVal), + ikey("bar", 99, ktVal))) // When user keys are different, but correctly ordered - assertBytes(t, ikey("g", uint64(kMaxSeq), tSeek), - shortSep(ikey("foo", 100, tVal), - ikey("hello", 200, tVal))) + assertBytes(t, ikey("g", uint64(kMaxSeq), ktSeek), + shortSep(ikey("foo", 100, ktVal), + ikey("hello", 200, ktVal))) // When start user key is prefix of limit user key - assertBytes(t, ikey("foo", 100, tVal), - shortSep(ikey("foo", 100, tVal), - ikey("foobar", 200, tVal))) + assertBytes(t, ikey("foo", 100, ktVal), + shortSep(ikey("foo", 100, ktVal), + ikey("foobar", 200, ktVal))) // When limit user key is prefix of start user key - assertBytes(t, ikey("foobar", 100, tVal), - shortSep(ikey("foobar", 100, tVal), - ikey("foo", 200, tVal))) + assertBytes(t, ikey("foobar", 100, ktVal), + shortSep(ikey("foobar", 100, ktVal), + ikey("foo", 200, ktVal))) } -func TestIKeyShortestSuccessor(t *testing.T) { - assertBytes(t, ikey("g", uint64(kMaxSeq), tSeek), - shortSuccessor(ikey("foo", 100, tVal))) - assertBytes(t, ikey("\xff\xff", 100, tVal), - shortSuccessor(ikey("\xff\xff", 100, tVal))) +func TestIkeyShortestSuccessor(t *testing.T) { + assertBytes(t, ikey("g", uint64(kMaxSeq), ktSeek), + shortSuccessor(ikey("foo", 100, ktVal))) + assertBytes(t, ikey("\xff\xff", 100, ktVal), + shortSuccessor(ikey("\xff\xff", 100, ktVal))) } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go index 245b1fd4d..fefa007a7 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go @@ -3,18 +3,9 @@ package leveldb import ( "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/syndtr/goleveldb/leveldb/testutil" ) -func TestLeveldb(t *testing.T) { - testutil.RunDefer() - - RegisterFailHandler(Fail) - RunSpecs(t, "Leveldb Suite") - - RegisterTestingT(t) - testutil.RunDefer("teardown") +func TestLevelDB(t *testing.T) { + testutil.RunSuite(t, "LevelDB Suite") } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go index 7bcae992a..e5398873b 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go @@ -12,12 +12,14 @@ import ( "sync" "github.com/syndtr/goleveldb/leveldb/comparer" + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/util" ) var ( - ErrNotFound = util.ErrNotFound + ErrNotFound = errors.ErrNotFound + ErrIterReleased = errors.New("leveldb/memdb: iterator released") ) const tMaxHeight = 12 @@ -29,6 +31,7 @@ type dbIter struct { node int forward bool key, value []byte + err error } func (i *dbIter) fill(checkStart, checkLimit bool) bool { @@ -59,6 +62,11 @@ func (i *dbIter) Valid() bool { } func (i *dbIter) First() bool { + if i.Released() { + i.err = ErrIterReleased + return false + } + i.forward = true i.p.mu.RLock() defer i.p.mu.RUnlock() @@ -71,9 +79,11 @@ func (i *dbIter) First() bool { } func (i *dbIter) Last() bool { - if i.p == nil { + if i.Released() { + i.err = ErrIterReleased return false } + i.forward = false i.p.mu.RLock() defer i.p.mu.RUnlock() @@ -86,9 +96,11 @@ func (i *dbIter) Last() bool { } func (i *dbIter) Seek(key []byte) bool { - if i.p == nil { + if i.Released() { + i.err = ErrIterReleased return false } + i.forward = true i.p.mu.RLock() defer i.p.mu.RUnlock() @@ -100,9 +112,11 @@ func (i *dbIter) Seek(key []byte) bool { } func (i *dbIter) Next() bool { - if i.p == nil { + if i.Released() { + i.err = ErrIterReleased return false } + if i.node == 0 { if !i.forward { return i.First() @@ -117,9 +131,11 @@ func (i *dbIter) Next() bool { } func (i *dbIter) Prev() bool { - if i.p == nil { + if i.Released() { + i.err = ErrIterReleased return false } + if i.node == 0 { if i.forward { return i.Last() @@ -141,10 +157,10 @@ func (i *dbIter) Value() []byte { return i.value } -func (i *dbIter) Error() error { return nil } +func (i *dbIter) Error() error { return i.err } func (i *dbIter) Release() { - if i.p != nil { + if !i.Released() { i.p = nil i.node = 0 i.key = nil @@ -437,6 +453,8 @@ func (p *DB) Reset() { // New creates a new initalized in-memory key/value DB. The capacity // is the initial key/value buffer capacity. The capacity is advisory, // not enforced. +// +// The returned DB instance is goroutine-safe. func New(cmp comparer.BasicComparer, capacity int) *DB { p := &DB{ cmp: cmp, diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go index 788539a87..18c304b7f 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go @@ -3,15 +3,9 @@ package memdb import ( "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/syndtr/goleveldb/leveldb/testutil" ) -func TestMemdb(t *testing.T) { - testutil.RunDefer() - - RegisterFailHandler(Fail) - RunSpecs(t, "Memdb Suite") +func TestMemDB(t *testing.T) { + testutil.RunSuite(t, "MemDB Suite") } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go index f96a9d1ea..5dd6dbc7b 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go @@ -129,7 +129,7 @@ var _ = testutil.Defer(func() { } return db - }) + }, nil, nil) }) }) }) diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go index b940ce427..61f0eadf9 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go @@ -11,6 +11,7 @@ import ( "github.com/syndtr/goleveldb/leveldb/cache" "github.com/syndtr/goleveldb/leveldb/comparer" "github.com/syndtr/goleveldb/leveldb/filter" + "math" ) const ( @@ -19,25 +20,57 @@ const ( GiB = MiB * 1024 ) -const ( - DefaultBlockCacheSize = 8 * MiB - DefaultBlockRestartInterval = 16 - DefaultBlockSize = 4 * KiB - DefaultCompressionType = SnappyCompression - DefaultMaxOpenFiles = 1000 - DefaultWriteBuffer = 4 * MiB +var ( + DefaultBlockCacher = LRUCacher + DefaultBlockCacheCapacity = 8 * MiB + DefaultBlockRestartInterval = 16 + DefaultBlockSize = 4 * KiB + DefaultCompactionExpandLimitFactor = 25 + DefaultCompactionGPOverlapsFactor = 10 + DefaultCompactionL0Trigger = 4 + DefaultCompactionSourceLimitFactor = 1 + DefaultCompactionTableSize = 2 * MiB + DefaultCompactionTableSizeMultiplier = 1.0 + DefaultCompactionTotalSize = 10 * MiB + DefaultCompactionTotalSizeMultiplier = 10.0 + DefaultCompressionType = SnappyCompression + DefaultIteratorSamplingRate = 1 * MiB + DefaultMaxMemCompationLevel = 2 + DefaultNumLevel = 7 + DefaultOpenFilesCacher = LRUCacher + DefaultOpenFilesCacheCapacity = 500 + DefaultWriteBuffer = 4 * MiB + DefaultWriteL0PauseTrigger = 12 + DefaultWriteL0SlowdownTrigger = 8 ) -type noCache struct{} +// Cacher is a caching algorithm. +type Cacher interface { + New(capacity int) cache.Cacher +} + +type CacherFunc struct { + NewFunc func(capacity int) cache.Cacher +} + +func (f *CacherFunc) New(capacity int) cache.Cacher { + if f.NewFunc != nil { + return f.NewFunc(capacity) + } + return nil +} -func (noCache) SetCapacity(capacity int) {} -func (noCache) GetNamespace(id uint64) cache.Namespace { return nil } -func (noCache) Purge(fin cache.PurgeFin) {} -func (noCache) Zap(closed bool) {} +func noCacher(int) cache.Cacher { return nil } -var NoCache cache.Cache = noCache{} +var ( + // LRUCacher is the LRU-cache algorithm. + LRUCacher = &CacherFunc{cache.NewLRU} -// Compression is the per-block compression algorithm to use. + // NoCacher is the value to disable caching algorithm. + NoCacher = &CacherFunc{} +) + +// Compression is the 'sorted table' block compression algorithm to use. type Compression uint func (c Compression) String() string { @@ -59,34 +92,47 @@ const ( nCompression ) -// Strict is the DB strict level. +// Strict is the DB 'strict level'. type Strict uint const ( // If present then a corrupted or invalid chunk or block in manifest - // journal will cause an error istead of being dropped. + // journal will cause an error instead of being dropped. + // This will prevent database with corrupted manifest to be opened. StrictManifest Strict = 1 << iota - // If present then a corrupted or invalid chunk or block in journal - // will cause an error istead of being dropped. - StrictJournal - // If present then journal chunk checksum will be verified. StrictJournalChecksum - // If present then an invalid key/value pair will cause an error - // instead of being skipped. - StrictIterator + // If present then a corrupted or invalid chunk or block in journal + // will cause an error instead of being dropped. + // This will prevent database with corrupted journal to be opened. + StrictJournal // If present then 'sorted table' block checksum will be verified. + // This has effect on both 'read operation' and compaction. StrictBlockChecksum + // If present then a corrupted 'sorted table' will fails compaction. + // The database will enter read-only mode. + StrictCompaction + + // If present then a corrupted 'sorted table' will halts 'read operation'. + StrictReader + + // If present then leveldb.Recover will drop corrupted 'sorted table'. + StrictRecovery + + // This only applicable for ReadOptions, if present then this ReadOptions + // 'strict level' will override global ones. + StrictOverride + // StrictAll enables all strict flags. - StrictAll = StrictManifest | StrictJournal | StrictJournalChecksum | StrictIterator | StrictBlockChecksum + StrictAll = StrictManifest | StrictJournalChecksum | StrictJournal | StrictBlockChecksum | StrictCompaction | StrictReader | StrictRecovery // DefaultStrict is the default strict flags. Specify any strict flags // will override default strict flags as whole (i.e. not OR'ed). - DefaultStrict = StrictJournalChecksum | StrictBlockChecksum + DefaultStrict = StrictJournalChecksum | StrictBlockChecksum | StrictCompaction | StrictReader // NoStrict disables all strict flags. Override default strict flags. NoStrict = ^StrictAll @@ -101,11 +147,17 @@ type Options struct { // The default value is nil AltFilters []filter.Filter - // BlockCache provides per-block caching for LevelDB. Specify NoCache to - // disable block caching. + // BlockCacher provides cache algorithm for LevelDB 'sorted table' block caching. + // Specify NoCacher to disable caching algorithm. // - // By default LevelDB will create LRU-cache with capacity of 8MiB. - BlockCache cache.Cache + // The default value is LRUCacher. + BlockCacher Cacher + + // BlockCacheCapacity defines the capacity of the 'sorted table' block caching. + // Use -1 for zero, this has same effect as specifying NoCacher to BlockCacher. + // + // The default value is 8MiB. + BlockCacheCapacity int // BlockRestartInterval is the number of keys between restart points for // delta encoding of keys. @@ -119,6 +171,73 @@ type Options struct { // The default value is 4KiB. BlockSize int + // CompactionExpandLimitFactor limits compaction size after expanded. + // This will be multiplied by table size limit at compaction target level. + // + // The default value is 25. + CompactionExpandLimitFactor int + + // CompactionGPOverlapsFactor limits overlaps in grandparent (Level + 2) that a + // single 'sorted table' generates. + // This will be multiplied by table size limit at grandparent level. + // + // The default value is 10. + CompactionGPOverlapsFactor int + + // CompactionL0Trigger defines number of 'sorted table' at level-0 that will + // trigger compaction. + // + // The default value is 4. + CompactionL0Trigger int + + // CompactionSourceLimitFactor limits compaction source size. This doesn't apply to + // level-0. + // This will be multiplied by table size limit at compaction target level. + // + // The default value is 1. + CompactionSourceLimitFactor int + + // CompactionTableSize limits size of 'sorted table' that compaction generates. + // The limits for each level will be calculated as: + // CompactionTableSize * (CompactionTableSizeMultiplier ^ Level) + // The multiplier for each level can also fine-tuned using CompactionTableSizeMultiplierPerLevel. + // + // The default value is 2MiB. + CompactionTableSize int + + // CompactionTableSizeMultiplier defines multiplier for CompactionTableSize. + // + // The default value is 1. + CompactionTableSizeMultiplier float64 + + // CompactionTableSizeMultiplierPerLevel defines per-level multiplier for + // CompactionTableSize. + // Use zero to skip a level. + // + // The default value is nil. + CompactionTableSizeMultiplierPerLevel []float64 + + // CompactionTotalSize limits total size of 'sorted table' for each level. + // The limits for each level will be calculated as: + // CompactionTotalSize * (CompactionTotalSizeMultiplier ^ Level) + // The multiplier for each level can also fine-tuned using + // CompactionTotalSizeMultiplierPerLevel. + // + // The default value is 10MiB. + CompactionTotalSize int + + // CompactionTotalSizeMultiplier defines multiplier for CompactionTotalSize. + // + // The default value is 10. + CompactionTotalSizeMultiplier float64 + + // CompactionTotalSizeMultiplierPerLevel defines per-level multiplier for + // CompactionTotalSize. + // Use zero to skip a level. + // + // The default value is nil. + CompactionTotalSizeMultiplierPerLevel []float64 + // Comparer defines a total ordering over the space of []byte keys: a 'less // than' relationship. The same comparison algorithm must be used for reads // and writes over the lifetime of the DB. @@ -126,11 +245,22 @@ type Options struct { // The default value uses the same ordering as bytes.Compare. Comparer comparer.Comparer - // Compression defines the per-block compression to use. + // Compression defines the 'sorted table' block compression to use. // // The default value (DefaultCompression) uses snappy compression. Compression Compression + // DisableBlockCache allows disable use of cache.Cache functionality on + // 'sorted table' block. + // + // The default value is false. + DisableBlockCache bool + + // DisableCompactionBackoff allows disable compaction retry backoff. + // + // The default value is false. + DisableCompactionBackoff bool + // ErrorIfExist defines whether an error should returned if the DB already // exist. // @@ -159,12 +289,37 @@ type Options struct { // The default value is nil. Filter filter.Filter - // MaxOpenFiles defines maximum number of open files to kept around - // (cached). This is not an hard limit, actual open files may exceed - // the defined value. + // IteratorSamplingRate defines approximate gap (in bytes) between read + // sampling of an iterator. The samples will be used to determine when + // compaction should be triggered. + // + // The default is 1MiB. + IteratorSamplingRate int + + // MaxMemCompationLevel defines maximum level a newly compacted 'memdb' + // will be pushed into if doesn't creates overlap. This should less than + // NumLevel. Use -1 for level-0. // - // The default value is 1000. - MaxOpenFiles int + // The default is 2. + MaxMemCompationLevel int + + // NumLevel defines number of database level. The level shouldn't changed + // between opens, or the database will panic. + // + // The default is 7. + NumLevel int + + // OpenFilesCacher provides cache algorithm for open files caching. + // Specify NoCacher to disable caching algorithm. + // + // The default value is LRUCacher. + OpenFilesCacher Cacher + + // OpenFilesCacheCapacity defines the capacity of the open files caching. + // Use -1 for zero, this has same effect as specifying NoCacher to OpenFilesCacher. + // + // The default value is 500. + OpenFilesCacheCapacity int // Strict defines the DB strict level. Strict Strict @@ -177,6 +332,18 @@ type Options struct { // // The default value is 4MiB. WriteBuffer int + + // WriteL0StopTrigger defines number of 'sorted table' at level-0 that will + // pause write. + // + // The default value is 12. + WriteL0PauseTrigger int + + // WriteL0SlowdownTrigger defines number of 'sorted table' at level-0 that + // will trigger write slowdown. + // + // The default value is 8. + WriteL0SlowdownTrigger int } func (o *Options) GetAltFilters() []filter.Filter { @@ -186,11 +353,22 @@ func (o *Options) GetAltFilters() []filter.Filter { return o.AltFilters } -func (o *Options) GetBlockCache() cache.Cache { - if o == nil { +func (o *Options) GetBlockCacher() Cacher { + if o == nil || o.BlockCacher == nil { + return DefaultBlockCacher + } else if o.BlockCacher == NoCacher { return nil } - return o.BlockCache + return o.BlockCacher +} + +func (o *Options) GetBlockCacheCapacity() int { + if o == nil || o.BlockCacheCapacity == 0 { + return DefaultBlockCacheCapacity + } else if o.BlockCacheCapacity < 0 { + return 0 + } + return o.BlockCacheCapacity } func (o *Options) GetBlockRestartInterval() int { @@ -207,6 +385,79 @@ func (o *Options) GetBlockSize() int { return o.BlockSize } +func (o *Options) GetCompactionExpandLimit(level int) int { + factor := DefaultCompactionExpandLimitFactor + if o != nil && o.CompactionExpandLimitFactor > 0 { + factor = o.CompactionExpandLimitFactor + } + return o.GetCompactionTableSize(level+1) * factor +} + +func (o *Options) GetCompactionGPOverlaps(level int) int { + factor := DefaultCompactionGPOverlapsFactor + if o != nil && o.CompactionGPOverlapsFactor > 0 { + factor = o.CompactionGPOverlapsFactor + } + return o.GetCompactionTableSize(level+2) * factor +} + +func (o *Options) GetCompactionL0Trigger() int { + if o == nil || o.CompactionL0Trigger == 0 { + return DefaultCompactionL0Trigger + } + return o.CompactionL0Trigger +} + +func (o *Options) GetCompactionSourceLimit(level int) int { + factor := DefaultCompactionSourceLimitFactor + if o != nil && o.CompactionSourceLimitFactor > 0 { + factor = o.CompactionSourceLimitFactor + } + return o.GetCompactionTableSize(level+1) * factor +} + +func (o *Options) GetCompactionTableSize(level int) int { + var ( + base = DefaultCompactionTableSize + mult float64 + ) + if o != nil { + if o.CompactionTableSize > 0 { + base = o.CompactionTableSize + } + if len(o.CompactionTableSizeMultiplierPerLevel) > level && o.CompactionTableSizeMultiplierPerLevel[level] > 0 { + mult = o.CompactionTableSizeMultiplierPerLevel[level] + } else if o.CompactionTableSizeMultiplier > 0 { + mult = math.Pow(o.CompactionTableSizeMultiplier, float64(level)) + } + } + if mult == 0 { + mult = math.Pow(DefaultCompactionTableSizeMultiplier, float64(level)) + } + return int(float64(base) * mult) +} + +func (o *Options) GetCompactionTotalSize(level int) int64 { + var ( + base = DefaultCompactionTotalSize + mult float64 + ) + if o != nil { + if o.CompactionTotalSize > 0 { + base = o.CompactionTotalSize + } + if len(o.CompactionTotalSizeMultiplierPerLevel) > level && o.CompactionTotalSizeMultiplierPerLevel[level] > 0 { + mult = o.CompactionTotalSizeMultiplierPerLevel[level] + } else if o.CompactionTotalSizeMultiplier > 0 { + mult = math.Pow(o.CompactionTotalSizeMultiplier, float64(level)) + } + } + if mult == 0 { + mult = math.Pow(DefaultCompactionTotalSizeMultiplier, float64(level)) + } + return int64(float64(base) * mult) +} + func (o *Options) GetComparer() comparer.Comparer { if o == nil || o.Comparer == nil { return comparer.DefaultComparer @@ -221,6 +472,13 @@ func (o *Options) GetCompression() Compression { return o.Compression } +func (o *Options) GetDisableCompactionBackoff() bool { + if o == nil { + return false + } + return o.DisableCompactionBackoff +} + func (o *Options) GetErrorIfExist() bool { if o == nil { return false @@ -242,11 +500,52 @@ func (o *Options) GetFilter() filter.Filter { return o.Filter } -func (o *Options) GetMaxOpenFiles() int { - if o == nil || o.MaxOpenFiles <= 0 { - return DefaultMaxOpenFiles +func (o *Options) GetIteratorSamplingRate() int { + if o == nil || o.IteratorSamplingRate <= 0 { + return DefaultIteratorSamplingRate + } + return o.IteratorSamplingRate +} + +func (o *Options) GetMaxMemCompationLevel() int { + level := DefaultMaxMemCompationLevel + if o != nil { + if o.MaxMemCompationLevel > 0 { + level = o.MaxMemCompationLevel + } else if o.MaxMemCompationLevel < 0 { + level = 0 + } + } + if level >= o.GetNumLevel() { + return o.GetNumLevel() - 1 + } + return level +} + +func (o *Options) GetNumLevel() int { + if o == nil || o.NumLevel <= 0 { + return DefaultNumLevel + } + return o.NumLevel +} + +func (o *Options) GetOpenFilesCacher() Cacher { + if o == nil || o.OpenFilesCacher == nil { + return DefaultOpenFilesCacher } - return o.MaxOpenFiles + if o.OpenFilesCacher == NoCacher { + return nil + } + return o.OpenFilesCacher +} + +func (o *Options) GetOpenFilesCacheCapacity() int { + if o == nil || o.OpenFilesCacheCapacity == 0 { + return DefaultOpenFilesCacheCapacity + } else if o.OpenFilesCacheCapacity < 0 { + return 0 + } + return o.OpenFilesCacheCapacity } func (o *Options) GetStrict(strict Strict) bool { @@ -263,6 +562,20 @@ func (o *Options) GetWriteBuffer() int { return o.WriteBuffer } +func (o *Options) GetWriteL0PauseTrigger() int { + if o == nil || o.WriteL0PauseTrigger == 0 { + return DefaultWriteL0PauseTrigger + } + return o.WriteL0PauseTrigger +} + +func (o *Options) GetWriteL0SlowdownTrigger() int { + if o == nil || o.WriteL0SlowdownTrigger == 0 { + return DefaultWriteL0SlowdownTrigger + } + return o.WriteL0SlowdownTrigger +} + // ReadOptions holds the optional parameters for 'read operation'. The // 'read operation' includes Get, Find and NewIterator. type ReadOptions struct { @@ -273,8 +586,8 @@ type ReadOptions struct { // The default value is false. DontFillCache bool - // Strict overrides global DB strict level. Only StrictIterator and - // StrictBlockChecksum that does have effects here. + // Strict will be OR'ed with global DB 'strict level' unless StrictOverride + // is present. Currently only StrictReader that has effect here. Strict Strict } @@ -316,3 +629,11 @@ func (wo *WriteOptions) GetSync() bool { } return wo.Sync } + +func GetStrict(o *Options, ro *ReadOptions, strict Strict) bool { + if ro.GetStrict(StrictOverride) { + return ro.GetStrict(strict) + } else { + return o.GetStrict(strict) || ro.GetStrict(strict) + } +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/options.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/options.go index fc6a96965..a3d84ef60 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/options.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/options.go @@ -7,35 +7,86 @@ package leveldb import ( - "github.com/syndtr/goleveldb/leveldb/cache" "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/opt" ) -func (s *session) setOptions(o *opt.Options) { - s.o = &opt.Options{} +func dupOptions(o *opt.Options) *opt.Options { + newo := &opt.Options{} if o != nil { - *s.o = *o + *newo = *o + } + if newo.Strict == 0 { + newo.Strict = opt.DefaultStrict } + return newo +} + +func (s *session) setOptions(o *opt.Options) { + no := dupOptions(o) // Alternative filters. if filters := o.GetAltFilters(); len(filters) > 0 { - s.o.AltFilters = make([]filter.Filter, len(filters)) + no.AltFilters = make([]filter.Filter, len(filters)) for i, filter := range filters { - s.o.AltFilters[i] = &iFilter{filter} + no.AltFilters[i] = &iFilter{filter} } } - // Block cache. - switch o.GetBlockCache() { - case nil: - s.o.BlockCache = cache.NewLRUCache(opt.DefaultBlockCacheSize) - case opt.NoCache: - s.o.BlockCache = nil - } // Comparer. s.icmp = &iComparer{o.GetComparer()} - s.o.Comparer = s.icmp + no.Comparer = s.icmp // Filter. if filter := o.GetFilter(); filter != nil { - s.o.Filter = &iFilter{filter} + no.Filter = &iFilter{filter} } + + s.o = &cachedOptions{Options: no} + s.o.cache() +} + +type cachedOptions struct { + *opt.Options + + compactionExpandLimit []int + compactionGPOverlaps []int + compactionSourceLimit []int + compactionTableSize []int + compactionTotalSize []int64 +} + +func (co *cachedOptions) cache() { + numLevel := co.Options.GetNumLevel() + + co.compactionExpandLimit = make([]int, numLevel) + co.compactionGPOverlaps = make([]int, numLevel) + co.compactionSourceLimit = make([]int, numLevel) + co.compactionTableSize = make([]int, numLevel) + co.compactionTotalSize = make([]int64, numLevel) + + for level := 0; level < numLevel; level++ { + co.compactionExpandLimit[level] = co.Options.GetCompactionExpandLimit(level) + co.compactionGPOverlaps[level] = co.Options.GetCompactionGPOverlaps(level) + co.compactionSourceLimit[level] = co.Options.GetCompactionSourceLimit(level) + co.compactionTableSize[level] = co.Options.GetCompactionTableSize(level) + co.compactionTotalSize[level] = co.Options.GetCompactionTotalSize(level) + } +} + +func (co *cachedOptions) GetCompactionExpandLimit(level int) int { + return co.compactionExpandLimit[level] +} + +func (co *cachedOptions) GetCompactionGPOverlaps(level int) int { + return co.compactionGPOverlaps[level] +} + +func (co *cachedOptions) GetCompactionSourceLimit(level int) int { + return co.compactionSourceLimit[level] +} + +func (co *cachedOptions) GetCompactionTableSize(level int) int { + return co.compactionTableSize[level] +} + +func (co *cachedOptions) GetCompactionTotalSize(level int) int64 { + return co.compactionTotalSize[level] } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go index 6b2a61683..b3906f7fc 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go @@ -7,12 +7,13 @@ package leveldb import ( - "errors" + "fmt" "io" "os" "sync" "sync/atomic" + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/journal" "github.com/syndtr/goleveldb/leveldb/opt" @@ -20,18 +21,31 @@ import ( "github.com/syndtr/goleveldb/leveldb/util" ) +type ErrManifestCorrupted struct { + Field string + Reason string +} + +func (e *ErrManifestCorrupted) Error() string { + return fmt.Sprintf("leveldb: manifest corrupted (field '%s'): %s", e.Field, e.Reason) +} + +func newErrManifestCorrupted(f storage.File, field, reason string) error { + return errors.NewErrCorrupted(f, &ErrManifestCorrupted{field, reason}) +} + // session represent a persistent database session. type session struct { // Need 64-bit alignment. - stFileNum uint64 // current unused file number + stNextFileNum uint64 // current unused file number stJournalNum uint64 // current journal file number; need external synchronization stPrevJournalNum uint64 // prev journal file number; no longer used; for compatibility with older version of leveldb - stSeq uint64 // last mem compacted seq; need external synchronization + stSeqNum uint64 // last mem compacted seq; need external synchronization stTempFileNum uint64 stor storage.Storage storLock util.Releaser - o *opt.Options + o *cachedOptions icmp *iComparer tops *tOps @@ -39,11 +53,12 @@ type session struct { manifestWriter storage.Writer manifestFile storage.File - stCPtrs [kNumLevels]iKey // compact pointers; need external synchronization - stVersion *version // current version - vmu sync.Mutex + stCompPtrs []iKey // compaction pointers; need external synchronization + stVersion *version // current version + vmu sync.Mutex } +// Creates new initialized session instance. func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) { if stor == nil { return nil, os.ErrInvalid @@ -53,22 +68,20 @@ func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) { return } s = &session{ - stor: stor, - storLock: storLock, + stor: stor, + storLock: storLock, + stCompPtrs: make([]iKey, o.GetNumLevel()), } s.setOptions(o) - s.tops = newTableOps(s, s.o.GetMaxOpenFiles()) - s.setVersion(&version{s: s}) - s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock D·DeletedEntry L·Level Q·SeqNum T·TimeElapsed") + s.tops = newTableOps(s) + s.setVersion(newVersion(s)) + s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed") return } // Close session. func (s *session) close() { s.tops.close() - if bc := s.o.GetBlockCache(); bc != nil { - bc.Purge(nil) - } if s.manifest != nil { s.manifest.Close() } @@ -81,6 +94,7 @@ func (s *session) close() { s.stVersion = nil } +// Release session lock. func (s *session) release() { s.storLock.Release() } @@ -98,26 +112,26 @@ func (s *session) recover() (err error) { // Don't return os.ErrNotExist if the underlying storage contains // other files that belong to LevelDB. So the DB won't get trashed. if files, _ := s.stor.GetFiles(storage.TypeAll); len(files) > 0 { - err = ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest file missing")} + err = &errors.ErrCorrupted{File: &storage.FileInfo{Type: storage.TypeManifest}, Err: &errors.ErrMissingFiles{}} } } }() - file, err := s.stor.GetManifest() + m, err := s.stor.GetManifest() if err != nil { return } - reader, err := file.Open() + reader, err := m.Open() if err != nil { return } defer reader.Close() strict := s.o.GetStrict(opt.StrictManifest) - jr := journal.NewReader(reader, dropper{s, file}, strict, true) + jr := journal.NewReader(reader, dropper{s, m}, strict, true) - staging := s.version_NB().newStaging() - rec := &sessionRecord{} + staging := s.stVersion.newStaging() + rec := &sessionRecord{numLevel: s.o.GetNumLevel()} for { var r io.Reader r, err = jr.Next() @@ -126,51 +140,57 @@ func (s *session) recover() (err error) { err = nil break } - return + return errors.SetFile(err, m) } err = rec.decode(r) if err == nil { // save compact pointers - for _, rp := range rec.compactionPointers { - s.stCPtrs[rp.level] = iKey(rp.key) + for _, r := range rec.compPtrs { + s.stCompPtrs[r.level] = iKey(r.ikey) } // commit record to version staging staging.commit(rec) - } else if strict { - return ErrCorrupted{Type: CorruptedManifest, Err: err} } else { - s.logf("manifest error: %v (skipped)", err) + err = errors.SetFile(err, m) + if strict || !errors.IsCorrupted(err) { + return + } else { + s.logf("manifest error: %v (skipped)", errors.SetFile(err, m)) + } } - rec.resetCompactionPointers() + rec.resetCompPtrs() rec.resetAddedTables() rec.resetDeletedTables() } switch { case !rec.has(recComparer): - return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing comparer name")} + return newErrManifestCorrupted(m, "comparer", "missing") case rec.comparer != s.icmp.uName(): - return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: comparer mismatch, " + "want '" + s.icmp.uName() + "', " + "got '" + rec.comparer + "'")} - case !rec.has(recNextNum): - return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing next file number")} + return newErrManifestCorrupted(m, "comparer", fmt.Sprintf("mismatch: want '%s', got '%s'", s.icmp.uName(), rec.comparer)) + case !rec.has(recNextFileNum): + return newErrManifestCorrupted(m, "next-file-num", "missing") case !rec.has(recJournalNum): - return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing journal file number")} - case !rec.has(recSeq): - return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing seq number")} + return newErrManifestCorrupted(m, "journal-file-num", "missing") + case !rec.has(recSeqNum): + return newErrManifestCorrupted(m, "seq-num", "missing") } - s.manifestFile = file + s.manifestFile = m s.setVersion(staging.finish()) - s.setFileNum(rec.nextNum) + s.setNextFileNum(rec.nextFileNum) s.recordCommited(rec) return nil } // Commit session; need external synchronization. func (s *session) commit(r *sessionRecord) (err error) { + v := s.version() + defer v.release() + // spawn new version based on current version - nv := s.version_NB().spawn(r) + nv := v.spawn(r) if s.manifest == nil { // manifest journal writer not yet created, create one @@ -189,22 +209,22 @@ func (s *session) commit(r *sessionRecord) (err error) { // Pick a compaction based on current state; need external synchronization. func (s *session) pickCompaction() *compaction { - v := s.version_NB() + v := s.version() var level int var t0 tFiles if v.cScore >= 1 { level = v.cLevel - cp := s.stCPtrs[level] - tt := v.tables[level] - for _, t := range tt { - if cp == nil || s.icmp.Compare(t.max, cp) > 0 { + cptr := s.stCompPtrs[level] + tables := v.tables[level] + for _, t := range tables { + if cptr == nil || s.icmp.Compare(t.imax, cptr) > 0 { t0 = append(t0, t) break } } if len(t0) == 0 { - t0 = append(t0, tt[0]) + t0 = append(t0, tables[0]) } } else { if p := atomic.LoadPointer(&v.cSeek); p != nil { @@ -212,29 +232,21 @@ func (s *session) pickCompaction() *compaction { level = ts.level t0 = append(t0, ts.table) } else { + v.release() return nil } } - c := &compaction{s: s, version: v, level: level} - if level == 0 { - min, max := t0.getRange(s.icmp) - t0 = nil - v.tables[0].getOverlaps(min.ukey(), max.ukey(), &t0, false, s.icmp.ucmp) - } - - c.tables[0] = t0 - c.expand() - return c + return newCompaction(s, v, level, t0) } // Create compaction from given level and range; need external synchronization. -func (s *session) getCompactionRange(level int, min, max []byte) *compaction { - v := s.version_NB() +func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction { + v := s.version() - var t0 tFiles - v.tables[level].getOverlaps(min, max, &t0, level != 0, s.icmp.ucmp) + t0 := v.tables[level].getOverlaps(nil, s.icmp, umin, umax, level == 0) if len(t0) == 0 { + v.release() return nil } @@ -243,7 +255,7 @@ func (s *session) getCompactionRange(level int, min, max []byte) *compaction { // and we must not pick one file and drop another older file if the // two files overlap. if level > 0 { - limit := uint64(kMaxTableSize) + limit := uint64(v.s.o.GetCompactionSourceLimit(level)) total := uint64(0) for i, t := range t0 { total += t.size @@ -255,90 +267,124 @@ func (s *session) getCompactionRange(level int, min, max []byte) *compaction { } } - c := &compaction{s: s, version: v, level: level} - c.tables[0] = t0 + return newCompaction(s, v, level, t0) +} + +func newCompaction(s *session, v *version, level int, t0 tFiles) *compaction { + c := &compaction{ + s: s, + v: v, + level: level, + tables: [2]tFiles{t0, nil}, + maxGPOverlaps: uint64(s.o.GetCompactionGPOverlaps(level)), + tPtrs: make([]int, s.o.GetNumLevel()), + } c.expand() + c.save() return c } -// compaction represent a compaction state +// compaction represent a compaction state. type compaction struct { - s *session - version *version + s *session + v *version + + level int + tables [2]tFiles + maxGPOverlaps uint64 + + gp tFiles + gpi int + seenKey bool + gpOverlappedBytes uint64 + imin, imax iKey + tPtrs []int + released bool + + snapGPI int + snapSeenKey bool + snapGPOverlappedBytes uint64 + snapTPtrs []int +} - level int - tables [2]tFiles +func (c *compaction) save() { + c.snapGPI = c.gpi + c.snapSeenKey = c.seenKey + c.snapGPOverlappedBytes = c.gpOverlappedBytes + c.snapTPtrs = append(c.snapTPtrs[:0], c.tPtrs...) +} - gp tFiles - gpidx int - seenKey bool - overlappedBytes uint64 - min, max iKey +func (c *compaction) restore() { + c.gpi = c.snapGPI + c.seenKey = c.snapSeenKey + c.gpOverlappedBytes = c.snapGPOverlappedBytes + c.tPtrs = append(c.tPtrs[:0], c.snapTPtrs...) +} - tPtrs [kNumLevels]int +func (c *compaction) release() { + if !c.released { + c.released = true + c.v.release() + } } // Expand compacted tables; need external synchronization. func (c *compaction) expand() { - s := c.s - v := c.version - - level := c.level - vt0, vt1 := v.tables[level], v.tables[level+1] + limit := uint64(c.s.o.GetCompactionExpandLimit(c.level)) + vt0, vt1 := c.v.tables[c.level], c.v.tables[c.level+1] t0, t1 := c.tables[0], c.tables[1] - min, max := t0.getRange(s.icmp) - vt1.getOverlaps(min.ukey(), max.ukey(), &t1, true, s.icmp.ucmp) - - // Get entire range covered by compaction - amin, amax := append(t0, t1...).getRange(s.icmp) + imin, imax := t0.getRange(c.s.icmp) + // We expand t0 here just incase ukey hop across tables. + t0 = vt0.getOverlaps(t0, c.s.icmp, imin.ukey(), imax.ukey(), c.level == 0) + if len(t0) != len(c.tables[0]) { + imin, imax = t0.getRange(c.s.icmp) + } + t1 = vt1.getOverlaps(t1, c.s.icmp, imin.ukey(), imax.ukey(), false) + // Get entire range covered by compaction. + amin, amax := append(t0, t1...).getRange(c.s.icmp) // See if we can grow the number of inputs in "level" without // changing the number of "level+1" files we pick up. if len(t1) > 0 { - var exp0 tFiles - vt0.getOverlaps(amin.ukey(), amax.ukey(), &exp0, level != 0, s.icmp.ucmp) - if len(exp0) > len(t0) && t1.size()+exp0.size() < kExpCompactionMaxBytes { - var exp1 tFiles - xmin, xmax := exp0.getRange(s.icmp) - vt1.getOverlaps(xmin.ukey(), xmax.ukey(), &exp1, true, s.icmp.ucmp) + exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), c.level == 0) + if len(exp0) > len(t0) && t1.size()+exp0.size() < limit { + xmin, xmax := exp0.getRange(c.s.icmp) + exp1 := vt1.getOverlaps(nil, c.s.icmp, xmin.ukey(), xmax.ukey(), false) if len(exp1) == len(t1) { - s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)", - level, level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())), + c.s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)", + c.level, c.level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())), len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size()))) - min, max = xmin, xmax + imin, imax = xmin, xmax t0, t1 = exp0, exp1 - amin, amax = append(t0, t1...).getRange(s.icmp) + amin, amax = append(t0, t1...).getRange(c.s.icmp) } } } // Compute the set of grandparent files that overlap this compaction // (parent == level+1; grandparent == level+2) - if level+2 < kNumLevels { - v.tables[level+2].getOverlaps(amin.ukey(), amax.ukey(), &c.gp, true, s.icmp.ucmp) + if c.level+2 < c.s.o.GetNumLevel() { + c.gp = c.v.tables[c.level+2].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false) } c.tables[0], c.tables[1] = t0, t1 - c.min, c.max = min, max + c.imin, c.imax = imin, imax } // Check whether compaction is trivial. func (c *compaction) trivial() bool { - return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= kMaxGrandParentOverlapBytes + return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= c.maxGPOverlaps } -func (c *compaction) isBaseLevelForKey(key []byte) bool { - s := c.s - v := c.version - - for level, tt := range v.tables[c.level+2:] { - for c.tPtrs[level] < len(tt) { - t := tt[c.tPtrs[level]] - if s.icmp.uCompare(key, t.max.ukey()) <= 0 { - // We've advanced far enough - if s.icmp.uCompare(key, t.min.ukey()) >= 0 { - // Key falls in this file's range, so definitely not base level +func (c *compaction) baseLevelForKey(ukey []byte) bool { + for level, tables := range c.v.tables[c.level+2:] { + for c.tPtrs[level] < len(tables) { + t := tables[c.tPtrs[level]] + if c.s.icmp.uCompare(ukey, t.imax.ukey()) <= 0 { + // We've advanced far enough. + if c.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 { + // Key falls in this file's range, so definitely not base level. return false } break @@ -349,55 +395,61 @@ func (c *compaction) isBaseLevelForKey(key []byte) bool { return true } -func (c *compaction) shouldStopBefore(key iKey) bool { - for ; c.gpidx < len(c.gp); c.gpidx++ { - gp := c.gp[c.gpidx] - if c.s.icmp.Compare(key, gp.max) <= 0 { +func (c *compaction) shouldStopBefore(ikey iKey) bool { + for ; c.gpi < len(c.gp); c.gpi++ { + gp := c.gp[c.gpi] + if c.s.icmp.Compare(ikey, gp.imax) <= 0 { break } if c.seenKey { - c.overlappedBytes += gp.size + c.gpOverlappedBytes += gp.size } } c.seenKey = true - if c.overlappedBytes > kMaxGrandParentOverlapBytes { - // Too much overlap for current output; start new output - c.overlappedBytes = 0 + if c.gpOverlappedBytes > c.maxGPOverlaps { + // Too much overlap for current output; start new output. + c.gpOverlappedBytes = 0 return true } return false } +// Creates an iterator. func (c *compaction) newIterator() iterator.Iterator { - s := c.s - - level := c.level - icap := 2 + // Creates iterator slice. + icap := len(c.tables) if c.level == 0 { + // Special case for level-0 icap = len(c.tables[0]) + 1 } its := make([]iterator.Iterator, 0, icap) + // Options. ro := &opt.ReadOptions{ DontFillCache: true, + Strict: opt.StrictOverride, + } + strict := c.s.o.GetStrict(opt.StrictCompaction) + if strict { + ro.Strict |= opt.StrictReader } - strict := s.o.GetStrict(opt.StrictIterator) - for i, tt := range c.tables { - if len(tt) == 0 { + for i, tables := range c.tables { + if len(tables) == 0 { continue } - if level+i == 0 { - for _, t := range tt { - its = append(its, s.tops.newIterator(t, nil, ro)) + // Level-0 is not sorted and may overlaps each other. + if c.level+i == 0 { + for _, t := range tables { + its = append(its, c.s.tops.newIterator(t, nil, ro)) } } else { - it := iterator.NewIndexedIterator(tt.newIndexIterator(s.tops, s.icmp, nil, ro), strict, true) + it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict) its = append(its, it) } } - return iterator.NewMergedIterator(its, s.icmp, true) + return iterator.NewMergedIterator(its, c.s.icmp, strict) } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go index c50fda737..1bdcc68f5 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go @@ -9,11 +9,11 @@ package leveldb import ( "bufio" "encoding/binary" - "errors" "io" -) + "strings" -var errCorruptManifest = errors.New("leveldb: corrupt manifest") + "github.com/syndtr/goleveldb/leveldb/errors" +) type byteReader interface { io.Reader @@ -22,32 +22,28 @@ type byteReader interface { // These numbers are written to disk and should not be changed. const ( - recComparer = 1 - recJournalNum = 2 - recNextNum = 3 - recSeq = 4 - recCompactionPointer = 5 - recDeletedTable = 6 - recNewTable = 7 + recComparer = 1 + recJournalNum = 2 + recNextFileNum = 3 + recSeqNum = 4 + recCompPtr = 5 + recDelTable = 6 + recAddTable = 7 // 8 was used for large value refs recPrevJournalNum = 9 ) type cpRecord struct { level int - key iKey + ikey iKey } -type ntRecord struct { +type atRecord struct { level int num uint64 size uint64 - min iKey - max iKey -} - -func (r ntRecord) makeFile(s *session) *tFile { - return newTFile(s.getTableFile(r.num), r.size, r.min, r.max) + imin iKey + imax iKey } type dtRecord struct { @@ -56,17 +52,20 @@ type dtRecord struct { } type sessionRecord struct { - hasRec int - comparer string - journalNum uint64 - prevJournalNum uint64 - nextNum uint64 - seq uint64 - compactionPointers []cpRecord - addedTables []ntRecord - deletedTables []dtRecord - scratch [binary.MaxVarintLen64]byte - err error + numLevel int + + hasRec int + comparer string + journalNum uint64 + prevJournalNum uint64 + nextFileNum uint64 + seqNum uint64 + compPtrs []cpRecord + addedTables []atRecord + deletedTables []dtRecord + + scratch [binary.MaxVarintLen64]byte + err error } func (p *sessionRecord) has(rec int) bool { @@ -88,47 +87,47 @@ func (p *sessionRecord) setPrevJournalNum(num uint64) { p.prevJournalNum = num } -func (p *sessionRecord) setNextNum(num uint64) { - p.hasRec |= 1 << recNextNum - p.nextNum = num +func (p *sessionRecord) setNextFileNum(num uint64) { + p.hasRec |= 1 << recNextFileNum + p.nextFileNum = num } -func (p *sessionRecord) setSeq(seq uint64) { - p.hasRec |= 1 << recSeq - p.seq = seq +func (p *sessionRecord) setSeqNum(num uint64) { + p.hasRec |= 1 << recSeqNum + p.seqNum = num } -func (p *sessionRecord) addCompactionPointer(level int, key iKey) { - p.hasRec |= 1 << recCompactionPointer - p.compactionPointers = append(p.compactionPointers, cpRecord{level, key}) +func (p *sessionRecord) addCompPtr(level int, ikey iKey) { + p.hasRec |= 1 << recCompPtr + p.compPtrs = append(p.compPtrs, cpRecord{level, ikey}) } -func (p *sessionRecord) resetCompactionPointers() { - p.hasRec &= ^(1 << recCompactionPointer) - p.compactionPointers = p.compactionPointers[:0] +func (p *sessionRecord) resetCompPtrs() { + p.hasRec &= ^(1 << recCompPtr) + p.compPtrs = p.compPtrs[:0] } -func (p *sessionRecord) addTable(level int, num, size uint64, min, max iKey) { - p.hasRec |= 1 << recNewTable - p.addedTables = append(p.addedTables, ntRecord{level, num, size, min, max}) +func (p *sessionRecord) addTable(level int, num, size uint64, imin, imax iKey) { + p.hasRec |= 1 << recAddTable + p.addedTables = append(p.addedTables, atRecord{level, num, size, imin, imax}) } func (p *sessionRecord) addTableFile(level int, t *tFile) { - p.addTable(level, t.file.Num(), t.size, t.min, t.max) + p.addTable(level, t.file.Num(), t.size, t.imin, t.imax) } func (p *sessionRecord) resetAddedTables() { - p.hasRec &= ^(1 << recNewTable) + p.hasRec &= ^(1 << recAddTable) p.addedTables = p.addedTables[:0] } -func (p *sessionRecord) deleteTable(level int, num uint64) { - p.hasRec |= 1 << recDeletedTable +func (p *sessionRecord) delTable(level int, num uint64) { + p.hasRec |= 1 << recDelTable p.deletedTables = append(p.deletedTables, dtRecord{level, num}) } func (p *sessionRecord) resetDeletedTables() { - p.hasRec &= ^(1 << recDeletedTable) + p.hasRec &= ^(1 << recDelTable) p.deletedTables = p.deletedTables[:0] } @@ -161,43 +160,45 @@ func (p *sessionRecord) encode(w io.Writer) error { p.putUvarint(w, recJournalNum) p.putUvarint(w, p.journalNum) } - if p.has(recNextNum) { - p.putUvarint(w, recNextNum) - p.putUvarint(w, p.nextNum) + if p.has(recNextFileNum) { + p.putUvarint(w, recNextFileNum) + p.putUvarint(w, p.nextFileNum) } - if p.has(recSeq) { - p.putUvarint(w, recSeq) - p.putUvarint(w, p.seq) + if p.has(recSeqNum) { + p.putUvarint(w, recSeqNum) + p.putUvarint(w, p.seqNum) } - for _, cp := range p.compactionPointers { - p.putUvarint(w, recCompactionPointer) - p.putUvarint(w, uint64(cp.level)) - p.putBytes(w, cp.key) + for _, r := range p.compPtrs { + p.putUvarint(w, recCompPtr) + p.putUvarint(w, uint64(r.level)) + p.putBytes(w, r.ikey) } - for _, t := range p.deletedTables { - p.putUvarint(w, recDeletedTable) - p.putUvarint(w, uint64(t.level)) - p.putUvarint(w, t.num) + for _, r := range p.deletedTables { + p.putUvarint(w, recDelTable) + p.putUvarint(w, uint64(r.level)) + p.putUvarint(w, r.num) } - for _, t := range p.addedTables { - p.putUvarint(w, recNewTable) - p.putUvarint(w, uint64(t.level)) - p.putUvarint(w, t.num) - p.putUvarint(w, t.size) - p.putBytes(w, t.min) - p.putBytes(w, t.max) + for _, r := range p.addedTables { + p.putUvarint(w, recAddTable) + p.putUvarint(w, uint64(r.level)) + p.putUvarint(w, r.num) + p.putUvarint(w, r.size) + p.putBytes(w, r.imin) + p.putBytes(w, r.imax) } return p.err } -func (p *sessionRecord) readUvarint(r io.ByteReader) uint64 { +func (p *sessionRecord) readUvarintMayEOF(field string, r io.ByteReader, mayEOF bool) uint64 { if p.err != nil { return 0 } x, err := binary.ReadUvarint(r) if err != nil { - if err == io.EOF { - p.err = errCorruptManifest + if err == io.ErrUnexpectedEOF || (mayEOF == false && err == io.EOF) { + p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "short read"}) + } else if strings.HasPrefix(err.Error(), "binary:") { + p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, err.Error()}) } else { p.err = err } @@ -206,35 +207,39 @@ func (p *sessionRecord) readUvarint(r io.ByteReader) uint64 { return x } -func (p *sessionRecord) readBytes(r byteReader) []byte { +func (p *sessionRecord) readUvarint(field string, r io.ByteReader) uint64 { + return p.readUvarintMayEOF(field, r, false) +} + +func (p *sessionRecord) readBytes(field string, r byteReader) []byte { if p.err != nil { return nil } - n := p.readUvarint(r) + n := p.readUvarint(field, r) if p.err != nil { return nil } x := make([]byte, n) _, p.err = io.ReadFull(r, x) if p.err != nil { - if p.err == io.EOF { - p.err = errCorruptManifest + if p.err == io.ErrUnexpectedEOF { + p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "short read"}) } return nil } return x } -func (p *sessionRecord) readLevel(r io.ByteReader) int { +func (p *sessionRecord) readLevel(field string, r io.ByteReader) int { if p.err != nil { return 0 } - x := p.readUvarint(r) + x := p.readUvarint(field, r) if p.err != nil { return 0 } - if x >= kNumLevels { - p.err = errCorruptManifest + if x >= uint64(p.numLevel) { + p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "invalid level number"}) return 0 } return int(x) @@ -247,59 +252,59 @@ func (p *sessionRecord) decode(r io.Reader) error { } p.err = nil for p.err == nil { - rec, err := binary.ReadUvarint(br) - if err != nil { - if err == io.EOF { - err = nil + rec := p.readUvarintMayEOF("field-header", br, true) + if p.err != nil { + if p.err == io.EOF { + return nil } - return err + return p.err } switch rec { case recComparer: - x := p.readBytes(br) + x := p.readBytes("comparer", br) if p.err == nil { p.setComparer(string(x)) } case recJournalNum: - x := p.readUvarint(br) + x := p.readUvarint("journal-num", br) if p.err == nil { p.setJournalNum(x) } case recPrevJournalNum: - x := p.readUvarint(br) + x := p.readUvarint("prev-journal-num", br) if p.err == nil { p.setPrevJournalNum(x) } - case recNextNum: - x := p.readUvarint(br) + case recNextFileNum: + x := p.readUvarint("next-file-num", br) if p.err == nil { - p.setNextNum(x) + p.setNextFileNum(x) } - case recSeq: - x := p.readUvarint(br) + case recSeqNum: + x := p.readUvarint("seq-num", br) if p.err == nil { - p.setSeq(x) + p.setSeqNum(x) } - case recCompactionPointer: - level := p.readLevel(br) - key := p.readBytes(br) + case recCompPtr: + level := p.readLevel("comp-ptr.level", br) + ikey := p.readBytes("comp-ptr.ikey", br) if p.err == nil { - p.addCompactionPointer(level, iKey(key)) + p.addCompPtr(level, iKey(ikey)) } - case recNewTable: - level := p.readLevel(br) - num := p.readUvarint(br) - size := p.readUvarint(br) - min := p.readBytes(br) - max := p.readBytes(br) + case recAddTable: + level := p.readLevel("add-table.level", br) + num := p.readUvarint("add-table.num", br) + size := p.readUvarint("add-table.size", br) + imin := p.readBytes("add-table.imin", br) + imax := p.readBytes("add-table.imax", br) if p.err == nil { - p.addTable(level, num, size, min, max) + p.addTable(level, num, size, imin, imax) } - case recDeletedTable: - level := p.readLevel(br) - num := p.readUvarint(br) + case recDelTable: + level := p.readLevel("del-table.level", br) + num := p.readUvarint("del-table.num", br) if p.err == nil { - p.deleteTable(level, num) + p.delTable(level, num) } } } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go index 029fabfe6..c0c035ae3 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go @@ -9,6 +9,8 @@ package leveldb import ( "bytes" "testing" + + "github.com/syndtr/goleveldb/leveldb/opt" ) func decodeEncode(v *sessionRecord) (res bool, err error) { @@ -17,7 +19,7 @@ func decodeEncode(v *sessionRecord) (res bool, err error) { if err != nil { return } - v2 := new(sessionRecord) + v2 := &sessionRecord{numLevel: opt.DefaultNumLevel} err = v.decode(b) if err != nil { return @@ -32,7 +34,7 @@ func decodeEncode(v *sessionRecord) (res bool, err error) { func TestSessionRecord_EncodeDecode(t *testing.T) { big := uint64(1) << 50 - v := new(sessionRecord) + v := &sessionRecord{numLevel: opt.DefaultNumLevel} i := uint64(0) test := func() { res, err := decodeEncode(v) @@ -47,16 +49,16 @@ func TestSessionRecord_EncodeDecode(t *testing.T) { for ; i < 4; i++ { test() v.addTable(3, big+300+i, big+400+i, - newIKey([]byte("foo"), big+500+1, tVal), - newIKey([]byte("zoo"), big+600+1, tDel)) - v.deleteTable(4, big+700+i) - v.addCompactionPointer(int(i), newIKey([]byte("x"), big+900+1, tVal)) + newIkey([]byte("foo"), big+500+1, ktVal), + newIkey([]byte("zoo"), big+600+1, ktDel)) + v.delTable(4, big+700+i) + v.addCompPtr(int(i), newIkey([]byte("x"), big+900+1, ktVal)) } v.setComparer("foo") v.setJournalNum(big + 100) v.setPrevJournalNum(big + 99) - v.setNextNum(big + 200) - v.setSeq(big + 1000) + v.setNextFileNum(big + 200) + v.setSeqNum(big + 1000) test() } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go index bf412b030..007c02cde 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go @@ -14,7 +14,7 @@ import ( "github.com/syndtr/goleveldb/leveldb/storage" ) -// logging +// Logging. type dropper struct { s *session @@ -22,22 +22,17 @@ type dropper struct { } func (d dropper) Drop(err error) { - if e, ok := err.(journal.DroppedError); ok { + if e, ok := err.(*journal.ErrCorrupted); ok { d.s.logf("journal@drop %s-%d S·%s %q", d.file.Type(), d.file.Num(), shortenb(e.Size), e.Reason) } else { d.s.logf("journal@drop %s-%d %q", d.file.Type(), d.file.Num(), err) } } -func (s *session) log(v ...interface{}) { - s.stor.Log(fmt.Sprint(v...)) -} - -func (s *session) logf(format string, v ...interface{}) { - s.stor.Log(fmt.Sprintf(format, v...)) -} +func (s *session) log(v ...interface{}) { s.stor.Log(fmt.Sprint(v...)) } +func (s *session) logf(format string, v ...interface{}) { s.stor.Log(fmt.Sprintf(format, v...)) } -// file utils +// File utils. func (s *session) getJournalFile(num uint64) storage.File { return s.stor.GetFile(num, storage.TypeJournal) @@ -56,9 +51,14 @@ func (s *session) newTemp() storage.File { return s.stor.GetFile(num, storage.TypeTemp) } -// session state +func (s *session) tableFileFromRecord(r atRecord) *tFile { + return newTableFile(s.getTableFile(r.num), r.size, r.imin, r.imax) +} + +// Session state. -// Get current version. +// Get current version. This will incr version ref, must call +// version.release (exactly once) after use. func (s *session) version() *version { s.vmu.Lock() defer s.vmu.Unlock() @@ -66,85 +66,80 @@ func (s *session) version() *version { return s.stVersion } -// Get current version; no barrier. -func (s *session) version_NB() *version { - return s.stVersion -} - // Set current version to v. func (s *session) setVersion(v *version) { s.vmu.Lock() - v.ref = 1 + v.ref = 1 // Holds by session. if old := s.stVersion; old != nil { - v.ref++ + v.ref++ // Holds by old version. old.next = v - old.release_NB() + old.releaseNB() } s.stVersion = v s.vmu.Unlock() } // Get current unused file number. -func (s *session) fileNum() uint64 { - return atomic.LoadUint64(&s.stFileNum) +func (s *session) nextFileNum() uint64 { + return atomic.LoadUint64(&s.stNextFileNum) } -// Get current unused file number to num. -func (s *session) setFileNum(num uint64) { - atomic.StoreUint64(&s.stFileNum, num) +// Set current unused file number to num. +func (s *session) setNextFileNum(num uint64) { + atomic.StoreUint64(&s.stNextFileNum, num) } // Mark file number as used. func (s *session) markFileNum(num uint64) { - num += 1 + nextFileNum := num + 1 for { - old, x := s.stFileNum, num + old, x := s.stNextFileNum, nextFileNum if old > x { x = old } - if atomic.CompareAndSwapUint64(&s.stFileNum, old, x) { + if atomic.CompareAndSwapUint64(&s.stNextFileNum, old, x) { break } } } // Allocate a file number. -func (s *session) allocFileNum() (num uint64) { - return atomic.AddUint64(&s.stFileNum, 1) - 1 +func (s *session) allocFileNum() uint64 { + return atomic.AddUint64(&s.stNextFileNum, 1) - 1 } // Reuse given file number. func (s *session) reuseFileNum(num uint64) { for { - old, x := s.stFileNum, num + old, x := s.stNextFileNum, num if old != x+1 { x = old } - if atomic.CompareAndSwapUint64(&s.stFileNum, old, x) { + if atomic.CompareAndSwapUint64(&s.stNextFileNum, old, x) { break } } } -// manifest related utils +// Manifest related utils. // Fill given session record obj with current states; need external // synchronization. func (s *session) fillRecord(r *sessionRecord, snapshot bool) { - r.setNextNum(s.fileNum()) + r.setNextFileNum(s.nextFileNum()) if snapshot { if !r.has(recJournalNum) { r.setJournalNum(s.stJournalNum) } - if !r.has(recSeq) { - r.setSeq(s.stSeq) + if !r.has(recSeqNum) { + r.setSeqNum(s.stSeqNum) } - for level, ik := range s.stCPtrs { + for level, ik := range s.stCompPtrs { if ik != nil { - r.addCompactionPointer(level, ik) + r.addCompPtr(level, ik) } } @@ -152,7 +147,7 @@ func (s *session) fillRecord(r *sessionRecord, snapshot bool) { } } -// Mark if record has been commited, this will update session state; +// Mark if record has been committed, this will update session state; // need external synchronization. func (s *session) recordCommited(r *sessionRecord) { if r.has(recJournalNum) { @@ -163,12 +158,12 @@ func (s *session) recordCommited(r *sessionRecord) { s.stPrevJournalNum = r.prevJournalNum } - if r.has(recSeq) { - s.stSeq = r.seq + if r.has(recSeqNum) { + s.stSeqNum = r.seqNum } - for _, p := range r.compactionPointers { - s.stCPtrs[p.level] = iKey(p.key) + for _, p := range r.compPtrs { + s.stCompPtrs[p.level] = iKey(p.ikey) } } @@ -183,10 +178,11 @@ func (s *session) newManifest(rec *sessionRecord, v *version) (err error) { jw := journal.NewWriter(writer) if v == nil { - v = s.version_NB() + v = s.version() + defer v.release() } if rec == nil { - rec = new(sessionRecord) + rec = &sessionRecord{numLevel: s.o.GetNumLevel()} } s.fillRecord(rec, true) v.fillRecord(rec) diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go index 75439f6db..46cc9d070 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go @@ -221,7 +221,7 @@ func (fs *fileStorage) GetManifest() (f File, err error) { fs.log(fmt.Sprintf("skipping %s: invalid file name", fn)) continue } - if _, e1 := strconv.ParseUint(fn[7:], 10, 0); e1 != nil { + if _, e1 := strconv.ParseUint(fn[8:], 10, 0); e1 != nil { fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1)) continue } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_solaris.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_solaris.go new file mode 100644 index 000000000..102031bfd --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_solaris.go @@ -0,0 +1,68 @@ +// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com> +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// +build solaris + +package storage + +import ( + "os" + "syscall" +) + +type unixFileLock struct { + f *os.File +} + +func (fl *unixFileLock) release() error { + if err := setFileLock(fl.f, false); err != nil { + return err + } + return fl.f.Close() +} + +func newFileLock(path string) (fl fileLock, err error) { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return + } + err = setFileLock(f, true) + if err != nil { + f.Close() + return + } + fl = &unixFileLock{f: f} + return +} + +func setFileLock(f *os.File, lock bool) error { + flock := syscall.Flock_t{ + Type: syscall.F_UNLCK, + Start: 0, + Len: 0, + Whence: 1, + } + if lock { + flock.Type = syscall.F_WRLCK + } + return syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &flock) +} + +func rename(oldpath, newpath string) error { + return os.Rename(oldpath, newpath) +} + +func syncDir(name string) error { + f, err := os.Open(name) + if err != nil { + return err + } + defer f.Close() + if err := f.Sync(); err != nil { + return err + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go index 73499afc8..d0a604b7a 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go @@ -4,7 +4,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// +build darwin freebsd linux netbsd openbsd +// +build darwin dragonfly freebsd linux netbsd openbsd package storage diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/storage.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/storage.go index de5694888..85dd70b06 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/storage.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/storage.go @@ -67,7 +67,7 @@ type Writer interface { Syncer } -// File is the file. +// File is the file. A file instance must be goroutine-safe. type File interface { // Open opens the file for read. Returns os.ErrNotExist error // if the file does not exist. @@ -94,7 +94,7 @@ type File interface { Remove() error } -// Storage is the storage. +// Storage is the storage. A storage instance must be goroutine-safe. type Storage interface { // Lock locks the storage. Any subsequent attempt to call Lock will fail // until the last lock released. @@ -125,3 +125,33 @@ type Storage interface { // Other methods should not be called after the storage has been closed. Close() error } + +// FileInfo wraps basic file info. +type FileInfo struct { + Type FileType + Num uint64 +} + +func (fi FileInfo) String() string { + switch fi.Type { + case TypeManifest: + return fmt.Sprintf("MANIFEST-%06d", fi.Num) + case TypeJournal: + return fmt.Sprintf("%06d.log", fi.Num) + case TypeTable: + return fmt.Sprintf("%06d.ldb", fi.Num) + case TypeTemp: + return fmt.Sprintf("%06d.tmp", fi.Num) + default: + return fmt.Sprintf("%#x-%d", fi.Type, fi.Num) + } +} + +// NewFileInfo creates new FileInfo from the given File. It will returns nil +// if File is nil. +func NewFileInfo(f File) *FileInfo { + if f == nil { + return nil + } + return &FileInfo{f.Type(), f.Num()} +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go index 27e76d707..dc1f1fb54 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "io/ioutil" + "math/rand" "os" "path/filepath" "sync" @@ -28,11 +29,25 @@ var ( ) var ( - tsFSEnv = os.Getenv("GOLEVELDB_USEFS") - tsKeepFS = tsFSEnv == "2" - tsFS = tsKeepFS || tsFSEnv == "" || tsFSEnv == "1" - tsMU = &sync.Mutex{} - tsNum = 0 + tsFSEnv = os.Getenv("GOLEVELDB_USEFS") + tsTempdir = os.Getenv("GOLEVELDB_TEMPDIR") + tsKeepFS = tsFSEnv == "2" + tsFS = tsKeepFS || tsFSEnv == "" || tsFSEnv == "1" + tsMU = &sync.Mutex{} + tsNum = 0 +) + +type tsOp uint + +const ( + tsOpOpen tsOp = iota + tsOpCreate + tsOpRead + tsOpReadAt + tsOpWrite + tsOpSync + + tsOpNum ) type tsLock struct { @@ -53,6 +68,9 @@ type tsReader struct { func (tr tsReader) Read(b []byte) (n int, err error) { ts := tr.tf.ts ts.countRead(tr.tf.Type()) + if tr.tf.shouldErrLocked(tsOpRead) { + return 0, errors.New("leveldb.testStorage: emulated read error") + } n, err = tr.Reader.Read(b) if err != nil && err != io.EOF { ts.t.Errorf("E: read error, num=%d type=%v n=%d: %v", tr.tf.Num(), tr.tf.Type(), n, err) @@ -63,6 +81,9 @@ func (tr tsReader) Read(b []byte) (n int, err error) { func (tr tsReader) ReadAt(b []byte, off int64) (n int, err error) { ts := tr.tf.ts ts.countRead(tr.tf.Type()) + if tr.tf.shouldErrLocked(tsOpReadAt) { + return 0, errors.New("leveldb.testStorage: emulated readAt error") + } n, err = tr.Reader.ReadAt(b, off) if err != nil && err != io.EOF { ts.t.Errorf("E: readAt error, num=%d type=%v off=%d n=%d: %v", tr.tf.Num(), tr.tf.Type(), off, n, err) @@ -82,15 +103,12 @@ type tsWriter struct { } func (tw tsWriter) Write(b []byte) (n int, err error) { - ts := tw.tf.ts - ts.mu.Lock() - defer ts.mu.Unlock() - if ts.emuWriteErr&tw.tf.Type() != 0 { + if tw.tf.shouldErrLocked(tsOpWrite) { return 0, errors.New("leveldb.testStorage: emulated write error") } n, err = tw.Writer.Write(b) if err != nil { - ts.t.Errorf("E: write error, num=%d type=%v n=%d: %v", tw.tf.Num(), tw.tf.Type(), n, err) + tw.tf.ts.t.Errorf("E: write error, num=%d type=%v n=%d: %v", tw.tf.Num(), tw.tf.Type(), n, err) } return } @@ -98,23 +116,23 @@ func (tw tsWriter) Write(b []byte) (n int, err error) { func (tw tsWriter) Sync() (err error) { ts := tw.tf.ts ts.mu.Lock() - defer ts.mu.Unlock() for ts.emuDelaySync&tw.tf.Type() != 0 { ts.cond.Wait() } - if ts.emuSyncErr&tw.tf.Type() != 0 { + ts.mu.Unlock() + if tw.tf.shouldErrLocked(tsOpSync) { return errors.New("leveldb.testStorage: emulated sync error") } err = tw.Writer.Sync() if err != nil { - ts.t.Errorf("E: sync error, num=%d type=%v: %v", tw.tf.Num(), tw.tf.Type(), err) + tw.tf.ts.t.Errorf("E: sync error, num=%d type=%v: %v", tw.tf.Num(), tw.tf.Type(), err) } return } func (tw tsWriter) Close() (err error) { err = tw.Writer.Close() - tw.tf.close("reader", err) + tw.tf.close("writer", err) return } @@ -127,6 +145,16 @@ func (tf tsFile) x() uint64 { return tf.Num()<<typeShift | uint64(tf.Type()) } +func (tf tsFile) shouldErr(op tsOp) bool { + return tf.ts.shouldErr(tf, op) +} + +func (tf tsFile) shouldErrLocked(op tsOp) bool { + tf.ts.mu.Lock() + defer tf.ts.mu.Unlock() + return tf.shouldErr(op) +} + func (tf tsFile) checkOpen(m string) error { ts := tf.ts if writer, ok := ts.opens[tf.x()]; ok { @@ -163,7 +191,7 @@ func (tf tsFile) Open() (r storage.Reader, err error) { if err != nil { return } - if ts.emuOpenErr&tf.Type() != 0 { + if tf.shouldErr(tsOpOpen) { err = errors.New("leveldb.testStorage: emulated open error") return } @@ -190,7 +218,7 @@ func (tf tsFile) Create() (w storage.Writer, err error) { if err != nil { return } - if ts.emuCreateErr&tf.Type() != 0 { + if tf.shouldErr(tsOpCreate) { err = errors.New("leveldb.testStorage: emulated create error") return } @@ -205,6 +233,23 @@ func (tf tsFile) Create() (w storage.Writer, err error) { return } +func (tf tsFile) Replace(newfile storage.File) (err error) { + ts := tf.ts + ts.mu.Lock() + defer ts.mu.Unlock() + err = tf.checkOpen("replace") + if err != nil { + return + } + err = tf.File.Replace(newfile.(tsFile).File) + if err != nil { + ts.t.Errorf("E: cannot replace file, num=%d type=%v: %v", tf.Num(), tf.Type(), err) + } else { + ts.t.Logf("I: file replace, num=%d type=%v", tf.Num(), tf.Type()) + } + return +} + func (tf tsFile) Remove() (err error) { ts := tf.ts ts.mu.Lock() @@ -231,51 +276,75 @@ type testStorage struct { cond sync.Cond // Open files, true=writer, false=reader opens map[uint64]bool - emuOpenErr storage.FileType - emuCreateErr storage.FileType emuDelaySync storage.FileType - emuWriteErr storage.FileType - emuSyncErr storage.FileType ignoreOpenErr storage.FileType readCnt uint64 readCntEn storage.FileType + + emuErr [tsOpNum]storage.FileType + emuErrOnce [tsOpNum]storage.FileType + emuRandErr [tsOpNum]storage.FileType + emuRandErrProb int + emuErrOnceMap map[uint64]uint + emuRandRand *rand.Rand +} + +func (ts *testStorage) shouldErr(tf tsFile, op tsOp) bool { + if ts.emuErr[op]&tf.Type() != 0 { + return true + } else if ts.emuRandErr[op]&tf.Type() != 0 || ts.emuErrOnce[op]&tf.Type() != 0 { + sop := uint(1) << op + eop := ts.emuErrOnceMap[tf.x()] + if eop&sop == 0 && (ts.emuRandRand.Int()%ts.emuRandErrProb == 0 || ts.emuErrOnce[op]&tf.Type() != 0) { + ts.emuErrOnceMap[tf.x()] = eop | sop + ts.t.Logf("I: emulated error: file=%d type=%v op=%v", tf.Num(), tf.Type(), op) + return true + } + } + return false } -func (ts *testStorage) SetOpenErr(t storage.FileType) { +func (ts *testStorage) SetEmuErr(t storage.FileType, ops ...tsOp) { ts.mu.Lock() - ts.emuOpenErr = t + for _, op := range ops { + ts.emuErr[op] = t + } ts.mu.Unlock() } -func (ts *testStorage) SetCreateErr(t storage.FileType) { +func (ts *testStorage) SetEmuErrOnce(t storage.FileType, ops ...tsOp) { ts.mu.Lock() - ts.emuCreateErr = t + for _, op := range ops { + ts.emuErrOnce[op] = t + } ts.mu.Unlock() } -func (ts *testStorage) DelaySync(t storage.FileType) { +func (ts *testStorage) SetEmuRandErr(t storage.FileType, ops ...tsOp) { ts.mu.Lock() - ts.emuDelaySync |= t - ts.cond.Broadcast() + for _, op := range ops { + ts.emuRandErr[op] = t + } ts.mu.Unlock() } -func (ts *testStorage) ReleaseSync(t storage.FileType) { +func (ts *testStorage) SetEmuRandErrProb(prob int) { ts.mu.Lock() - ts.emuDelaySync &= ^t - ts.cond.Broadcast() + ts.emuRandErrProb = prob ts.mu.Unlock() } -func (ts *testStorage) SetWriteErr(t storage.FileType) { +func (ts *testStorage) DelaySync(t storage.FileType) { ts.mu.Lock() - ts.emuWriteErr = t + ts.emuDelaySync |= t + ts.cond.Broadcast() ts.mu.Unlock() } -func (ts *testStorage) SetSyncErr(t storage.FileType) { +func (ts *testStorage) ReleaseSync(t storage.FileType) { ts.mu.Lock() - ts.emuSyncErr = t + ts.emuDelaySync &= ^t + ts.cond.Broadcast() ts.mu.Unlock() } @@ -413,7 +482,11 @@ func newTestStorage(t *testing.T) *testStorage { num := tsNum tsNum++ tsMU.Unlock() - path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num)) + tempdir := tsTempdir + if tempdir == "" { + tempdir = os.TempDir() + } + path := filepath.Join(tempdir, fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num)) if _, err := os.Stat(path); err != nil { stor, err = storage.OpenFile(path) if err != nil { @@ -436,6 +509,10 @@ func newTestStorage(t *testing.T) *testStorage { } f.Close() } + if t.Failed() { + t.Logf("testing failed, test DB preserved at %s", path) + return nil + } if tsKeepFS { return nil } @@ -449,10 +526,13 @@ func newTestStorage(t *testing.T) *testStorage { stor = storage.NewMemStorage() } ts := &testStorage{ - t: t, - Storage: stor, - closeFn: closeFn, - opens: make(map[uint64]bool), + t: t, + Storage: stor, + closeFn: closeFn, + opens: make(map[uint64]bool), + emuErrOnceMap: make(map[uint64]uint), + emuRandErrProb: 0x999, + emuRandRand: rand.New(rand.NewSource(0xfacedead)), } ts.cond.L = &ts.mu return ts diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go index fdd5d2bcf..3e8df6af5 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go @@ -7,11 +7,11 @@ package leveldb import ( + "fmt" "sort" "sync/atomic" "github.com/syndtr/goleveldb/leveldb/cache" - "github.com/syndtr/goleveldb/leveldb/comparer" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/storage" @@ -19,34 +19,41 @@ import ( "github.com/syndtr/goleveldb/leveldb/util" ) -// table file +// tFile holds basic information about a table. type tFile struct { - file storage.File - seekLeft int32 - size uint64 - min, max iKey + file storage.File + seekLeft int32 + size uint64 + imin, imax iKey } -// test if key is after t -func (t *tFile) isAfter(key []byte, ucmp comparer.BasicComparer) bool { - return key != nil && ucmp.Compare(key, t.max.ukey()) > 0 +// Returns true if given key is after largest key of this table. +func (t *tFile) after(icmp *iComparer, ukey []byte) bool { + return ukey != nil && icmp.uCompare(ukey, t.imax.ukey()) > 0 } -// test if key is before t -func (t *tFile) isBefore(key []byte, ucmp comparer.BasicComparer) bool { - return key != nil && ucmp.Compare(key, t.min.ukey()) < 0 +// Returns true if given key is before smallest key of this table. +func (t *tFile) before(icmp *iComparer, ukey []byte) bool { + return ukey != nil && icmp.uCompare(ukey, t.imin.ukey()) < 0 } -func (t *tFile) incrSeek() int32 { +// Returns true if given key range overlaps with this table key range. +func (t *tFile) overlaps(icmp *iComparer, umin, umax []byte) bool { + return !t.after(icmp, umin) && !t.before(icmp, umax) +} + +// Cosumes one seek and return current seeks left. +func (t *tFile) consumeSeek() int32 { return atomic.AddInt32(&t.seekLeft, -1) } -func newTFile(file storage.File, size uint64, min, max iKey) *tFile { +// Creates new tFile. +func newTableFile(file storage.File, size uint64, imin, imax iKey) *tFile { f := &tFile{ file: file, size: size, - min: min, - max: max, + imin: imin, + imax: imax, } // We arrange to automatically compact this file after @@ -70,33 +77,52 @@ func newTFile(file storage.File, size uint64, min, max iKey) *tFile { return f } -// table files +// tFiles hold multiple tFile. type tFiles []*tFile func (tf tFiles) Len() int { return len(tf) } func (tf tFiles) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] } +func (tf tFiles) nums() string { + x := "[ " + for i, f := range tf { + if i != 0 { + x += ", " + } + x += fmt.Sprint(f.file.Num()) + } + x += " ]" + return x +} + +// Returns true if i smallest key is less than j. +// This used for sort by key in ascending order. func (tf tFiles) lessByKey(icmp *iComparer, i, j int) bool { a, b := tf[i], tf[j] - n := icmp.Compare(a.min, b.min) + n := icmp.Compare(a.imin, b.imin) if n == 0 { return a.file.Num() < b.file.Num() } return n < 0 } +// Returns true if i file number is greater than j. +// This used for sort by file number in descending order. func (tf tFiles) lessByNum(i, j int) bool { return tf[i].file.Num() > tf[j].file.Num() } +// Sorts tables by key in ascending order. func (tf tFiles) sortByKey(icmp *iComparer) { sort.Sort(&tFilesSortByKey{tFiles: tf, icmp: icmp}) } +// Sorts tables by file number in descending order. func (tf tFiles) sortByNum() { sort.Sort(&tFilesSortByNum{tFiles: tf}) } +// Returns sum of all tables size. func (tf tFiles) size() (sum uint64) { for _, t := range tf { sum += t.size @@ -104,94 +130,107 @@ func (tf tFiles) size() (sum uint64) { return sum } -func (tf tFiles) searchMin(key iKey, icmp *iComparer) int { +// Searches smallest index of tables whose its smallest +// key is after or equal with given key. +func (tf tFiles) searchMin(icmp *iComparer, ikey iKey) int { return sort.Search(len(tf), func(i int) bool { - return icmp.Compare(tf[i].min, key) >= 0 + return icmp.Compare(tf[i].imin, ikey) >= 0 }) } -func (tf tFiles) searchMax(key iKey, icmp *iComparer) int { +// Searches smallest index of tables whose its largest +// key is after or equal with given key. +func (tf tFiles) searchMax(icmp *iComparer, ikey iKey) int { return sort.Search(len(tf), func(i int) bool { - return icmp.Compare(tf[i].max, key) >= 0 + return icmp.Compare(tf[i].imax, ikey) >= 0 }) } -func (tf tFiles) isOverlaps(min, max []byte, disjSorted bool, icmp *iComparer) bool { - if !disjSorted { - // Need to check against all files +// Returns true if given key range overlaps with one or more +// tables key range. If unsorted is true then binary search will not be used. +func (tf tFiles) overlaps(icmp *iComparer, umin, umax []byte, unsorted bool) bool { + if unsorted { + // Check against all files. for _, t := range tf { - if !t.isAfter(min, icmp.ucmp) && !t.isBefore(max, icmp.ucmp) { + if t.overlaps(icmp, umin, umax) { return true } } return false } - var idx int - if len(min) > 0 { - // Find the earliest possible internal key for min - idx = tf.searchMax(newIKey(min, kMaxSeq, tSeek), icmp) + i := 0 + if len(umin) > 0 { + // Find the earliest possible internal key for min. + i = tf.searchMax(icmp, newIkey(umin, kMaxSeq, ktSeek)) } - - if idx >= len(tf) { - // beginning of range is after all files, so no overlap + if i >= len(tf) { + // Beginning of range is after all files, so no overlap. return false } - return !tf[idx].isBefore(max, icmp.ucmp) + return !tf[i].before(icmp, umax) } -func (tf tFiles) getOverlaps(min, max []byte, r *tFiles, disjSorted bool, ucmp comparer.BasicComparer) { +// Returns tables whose its key range overlaps with given key range. +// Range will be expanded if ukey found hop across tables. +// If overlapped is true then the search will be restarted if umax +// expanded. +// The dst content will be overwritten. +func (tf tFiles) getOverlaps(dst tFiles, icmp *iComparer, umin, umax []byte, overlapped bool) tFiles { + dst = dst[:0] for i := 0; i < len(tf); { t := tf[i] - i++ - if t.isAfter(min, ucmp) || t.isBefore(max, ucmp) { - continue - } - - *r = append(*r, t) - if !disjSorted { - // Level-0 files may overlap each other. So check if the newly - // added file has expanded the range. If so, restart search. - if min != nil && ucmp.Compare(t.min.ukey(), min) < 0 { - min = t.min.ukey() - *r = nil - i = 0 - } else if max != nil && ucmp.Compare(t.max.ukey(), max) > 0 { - max = t.max.ukey() - *r = nil + if t.overlaps(icmp, umin, umax) { + if umin != nil && icmp.uCompare(t.imin.ukey(), umin) < 0 { + umin = t.imin.ukey() + dst = dst[:0] i = 0 + continue + } else if umax != nil && icmp.uCompare(t.imax.ukey(), umax) > 0 { + umax = t.imax.ukey() + // Restart search if it is overlapped. + if overlapped { + dst = dst[:0] + i = 0 + continue + } } + + dst = append(dst, t) } + i++ } - return + return dst } -func (tf tFiles) getRange(icmp *iComparer) (min, max iKey) { +// Returns tables key range. +func (tf tFiles) getRange(icmp *iComparer) (imin, imax iKey) { for i, t := range tf { if i == 0 { - min, max = t.min, t.max + imin, imax = t.imin, t.imax continue } - if icmp.Compare(t.min, min) < 0 { - min = t.min + if icmp.Compare(t.imin, imin) < 0 { + imin = t.imin } - if icmp.Compare(t.max, max) > 0 { - max = t.max + if icmp.Compare(t.imax, imax) > 0 { + imax = t.imax } } return } +// Creates iterator index from tables. func (tf tFiles) newIndexIterator(tops *tOps, icmp *iComparer, slice *util.Range, ro *opt.ReadOptions) iterator.IteratorIndexer { if slice != nil { var start, limit int if slice.Start != nil { - start = tf.searchMax(iKey(slice.Start), icmp) + start = tf.searchMax(icmp, iKey(slice.Start)) } if slice.Limit != nil { - limit = tf.searchMin(iKey(slice.Limit), icmp) + limit = tf.searchMin(icmp, iKey(slice.Limit)) } else { limit = tf.Len() } @@ -206,6 +245,7 @@ func (tf tFiles) newIndexIterator(tops *tOps, icmp *iComparer, slice *util.Range }) } +// Tables iterator index. type tFilesArrayIndexer struct { tFiles tops *tOps @@ -215,7 +255,7 @@ type tFilesArrayIndexer struct { } func (a *tFilesArrayIndexer) Search(key []byte) int { - return a.searchMax(iKey(key), a.icmp) + return a.searchMax(a.icmp, iKey(key)) } func (a *tFilesArrayIndexer) Get(i int) iterator.Iterator { @@ -225,6 +265,7 @@ func (a *tFilesArrayIndexer) Get(i int) iterator.Iterator { return a.tops.newIterator(a.tFiles[i], nil, a.ro) } +// Helper type for sortByKey. type tFilesSortByKey struct { tFiles icmp *iComparer @@ -234,6 +275,7 @@ func (x *tFilesSortByKey) Less(i, j int) bool { return x.lessByKey(x.icmp, i, j) } +// Helper type for sortByNum. type tFilesSortByNum struct { tFiles } @@ -242,19 +284,15 @@ func (x *tFilesSortByNum) Less(i, j int) bool { return x.lessByNum(i, j) } -// table operations +// Table operations. type tOps struct { - s *session - cache cache.Cache - cacheNS cache.Namespace -} - -func newTableOps(s *session, cacheCap int) *tOps { - c := cache.NewLRUCache(cacheCap) - ns := c.GetNamespace(0) - return &tOps{s, c, ns} + s *session + cache *cache.Cache + bcache *cache.Cache + bpool *util.BufferPool } +// Creates an empty table and returns table writer. func (t *tOps) create() (*tWriter, error) { file := t.s.getTableFile(t.s.allocFileNum()) fw, err := file.Create() @@ -265,14 +303,15 @@ func (t *tOps) create() (*tWriter, error) { t: t, file: file, w: fw, - tw: table.NewWriter(fw, t.s.o), + tw: table.NewWriter(fw, t.s.o.Options), }, nil } +// Builds table from src iterator. func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) { w, err := t.create() if err != nil { - return f, n, err + return } defer func() { @@ -282,7 +321,7 @@ func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) { }() for src.Next() { - err = w.add(src.Key(), src.Value()) + err = w.append(src.Key(), src.Value()) if err != nil { return } @@ -297,84 +336,132 @@ func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) { return } -func (t *tOps) lookup(f *tFile) (c cache.Object, err error) { +// Opens table. It returns a cache handle, which should +// be released after use. +func (t *tOps) open(f *tFile) (ch *cache.Handle, err error) { num := f.file.Num() - c, ok := t.cacheNS.Get(num, func() (ok bool, value interface{}, charge int, fin cache.SetFin) { + ch = t.cache.Get(0, num, func() (size int, value cache.Value) { var r storage.Reader r, err = f.file.Open() if err != nil { - return + return 0, nil } - o := t.s.o - - var cacheNS cache.Namespace - if bc := o.GetBlockCache(); bc != nil { - cacheNS = bc.GetNamespace(num) + var bcache *cache.CacheGetter + if t.bcache != nil { + bcache = &cache.CacheGetter{Cache: t.bcache, NS: num} } - ok = true - value = table.NewReader(r, int64(f.size), cacheNS, o) - charge = 1 - fin = func() { + var tr *table.Reader + tr, err = table.NewReader(r, int64(f.size), storage.NewFileInfo(f.file), bcache, t.bpool, t.s.o.Options) + if err != nil { r.Close() + return 0, nil } - return + return 1, tr + }) - if !ok && err == nil { + if ch == nil && err == nil { err = ErrClosed } return } -func (t *tOps) get(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []byte, err error) { - c, err := t.lookup(f) +// Finds key/value pair whose key is greater than or equal to the +// given key. +func (t *tOps) find(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []byte, err error) { + ch, err := t.open(f) if err != nil { return nil, nil, err } - defer c.Release() - return c.Value().(*table.Reader).Find(key, ro) + defer ch.Release() + return ch.Value().(*table.Reader).Find(key, true, ro) +} + +// Finds key that is greater than or equal to the given key. +func (t *tOps) findKey(f *tFile, key []byte, ro *opt.ReadOptions) (rkey []byte, err error) { + ch, err := t.open(f) + if err != nil { + return nil, err + } + defer ch.Release() + return ch.Value().(*table.Reader).FindKey(key, true, ro) } +// Returns approximate offset of the given key. func (t *tOps) offsetOf(f *tFile, key []byte) (offset uint64, err error) { - c, err := t.lookup(f) + ch, err := t.open(f) if err != nil { return } - _offset, err := c.Value().(*table.Reader).OffsetOf(key) - offset = uint64(_offset) - c.Release() - return + defer ch.Release() + offset_, err := ch.Value().(*table.Reader).OffsetOf(key) + return uint64(offset_), err } +// Creates an iterator from the given table. func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { - c, err := t.lookup(f) + ch, err := t.open(f) if err != nil { return iterator.NewEmptyIterator(err) } - iter := c.Value().(*table.Reader).NewIterator(slice, ro) - iter.SetReleaser(c) + iter := ch.Value().(*table.Reader).NewIterator(slice, ro) + iter.SetReleaser(ch) return iter } +// Removes table from persistent storage. It waits until +// no one use the the table. func (t *tOps) remove(f *tFile) { num := f.file.Num() - t.cacheNS.Delete(num, func(exist bool) { + t.cache.Delete(0, num, func() { if err := f.file.Remove(); err != nil { t.s.logf("table@remove removing @%d %q", num, err) } else { t.s.logf("table@remove removed @%d", num) } - if bc := t.s.o.GetBlockCache(); bc != nil { - bc.GetNamespace(num).Zap(false) + if t.bcache != nil { + t.bcache.EvictNS(num) } }) } +// Closes the table ops instance. It will close all tables, +// regadless still used or not. func (t *tOps) close() { - t.cache.Zap(true) + t.bpool.Close() + t.cache.Close() + if t.bcache != nil { + t.bcache.Close() + } +} + +// Creates new initialized table ops instance. +func newTableOps(s *session) *tOps { + var ( + cacher cache.Cacher + bcache *cache.Cache + ) + if s.o.GetOpenFilesCacheCapacity() > 0 { + cacher = cache.NewLRU(s.o.GetOpenFilesCacheCapacity()) + } + if !s.o.DisableBlockCache { + var bcacher cache.Cacher + if s.o.GetBlockCacheCapacity() > 0 { + bcacher = cache.NewLRU(s.o.GetBlockCacheCapacity()) + } + bcache = cache.NewCache(bcacher) + } + return &tOps{ + s: s, + cache: cache.NewCache(cacher), + bcache: bcache, + bpool: util.NewBufferPool(s.o.GetBlockSize() + 5), + } } +// tWriter wraps the table writer. It keep track of file descriptor +// and added key range. type tWriter struct { t *tOps @@ -385,7 +472,8 @@ type tWriter struct { first, last []byte } -func (w *tWriter) add(key, value []byte) error { +// Append key/value pair to the table. +func (w *tWriter) append(key, value []byte) error { if w.first == nil { w.first = append([]byte{}, key...) } @@ -393,30 +481,39 @@ func (w *tWriter) add(key, value []byte) error { return w.tw.Append(key, value) } +// Returns true if the table is empty. func (w *tWriter) empty() bool { return w.first == nil } +// Closes the storage.Writer. +func (w *tWriter) close() { + if w.w != nil { + w.w.Close() + w.w = nil + } +} + +// Finalizes the table and returns table file. func (w *tWriter) finish() (f *tFile, err error) { + defer w.close() err = w.tw.Close() if err != nil { return } err = w.w.Sync() if err != nil { - w.w.Close() return } - w.w.Close() - f = newTFile(w.file, uint64(w.tw.BytesLen()), iKey(w.first), iKey(w.last)) + f = newTableFile(w.file, uint64(w.tw.BytesLen()), iKey(w.first), iKey(w.last)) return } +// Drops the table. func (w *tWriter) drop() { - w.w.Close() + w.close() w.file.Remove() w.t.s.reuseFileNum(w.file.Num()) - w.w = nil w.file = nil w.tw = nil w.first = nil diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/block_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/block_test.go index ca598f4f5..00e6f9eea 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/block_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/block_test.go @@ -19,13 +19,18 @@ import ( "github.com/syndtr/goleveldb/leveldb/util" ) -func (b *block) TestNewIterator(slice *util.Range) iterator.Iterator { - return b.newIterator(slice, false, nil) +type blockTesting struct { + tr *Reader + b *block +} + +func (t *blockTesting) TestNewIterator(slice *util.Range) iterator.Iterator { + return t.tr.newBlockIter(t.b, nil, slice, false) } var _ = testutil.Defer(func() { Describe("Block", func() { - Build := func(kv *testutil.KeyValue, restartInterval int) *block { + Build := func(kv *testutil.KeyValue, restartInterval int) *blockTesting { // Building the block. bw := &blockWriter{ restartInterval: restartInterval, @@ -39,11 +44,13 @@ var _ = testutil.Defer(func() { // Opening the block. data := bw.buf.Bytes() restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:])) - return &block{ - cmp: comparer.DefaultComparer, - data: data, - restartsLen: restartsLen, - restartsOffset: len(data) - (restartsLen+1)*4, + return &blockTesting{ + tr: &Reader{cmp: comparer.DefaultComparer}, + b: &block{ + data: data, + restartsLen: restartsLen, + restartsOffset: len(data) - (restartsLen+1)*4, + }, } } @@ -59,7 +66,7 @@ var _ = testutil.Defer(func() { // Make block. br := Build(kv, restartInterval) // Do testing. - testutil.KeyValueTesting(nil, br, kv.Clone()) + testutil.KeyValueTesting(nil, kv.Clone(), br, nil, nil) } Describe(Text(), Test) @@ -102,11 +109,11 @@ var _ = testutil.Defer(func() { for restartInterval := 1; restartInterval <= 5; restartInterval++ { Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() { // Make block. - br := Build(kv, restartInterval) + bt := Build(kv, restartInterval) Test := func(r *util.Range) func(done Done) { return func(done Done) { - iter := br.newIterator(r, false, nil) + iter := bt.TestNewIterator(r) Expect(iter.Error()).ShouldNot(HaveOccurred()) t := testutil.IteratorTesting{ @@ -115,6 +122,7 @@ var _ = testutil.Defer(func() { } testutil.DoIteratorTesting(&t) + iter.Release() done <- true } } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go index 8acb9f720..6f38e84b3 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go @@ -8,27 +8,41 @@ package table import ( "encoding/binary" - "errors" "fmt" "io" "sort" "strings" + "sync" - "code.google.com/p/snappy-go/snappy" + "github.com/syndtr/gosnappy/snappy" "github.com/syndtr/goleveldb/leveldb/cache" "github.com/syndtr/goleveldb/leveldb/comparer" + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/storage" "github.com/syndtr/goleveldb/leveldb/util" ) var ( - ErrNotFound = util.ErrNotFound - ErrIterReleased = errors.New("leveldb/table: iterator released") + ErrNotFound = errors.ErrNotFound + ErrReaderReleased = errors.New("leveldb/table: reader released") + ErrIterReleased = errors.New("leveldb/table: iterator released") ) +type ErrCorrupted struct { + Pos int64 + Size int64 + Kind string + Reason string +} + +func (e *ErrCorrupted) Error() string { + return fmt.Sprintf("leveldb/table: corruption on %s (pos=%d): %s", e.Kind, e.Pos, e.Reason) +} + func max(x, y int) int { if x > y { return x @@ -37,40 +51,33 @@ func max(x, y int) int { } type block struct { - cmp comparer.BasicComparer + bpool *util.BufferPool + bh blockHandle data []byte restartsLen int restartsOffset int - // Whether checksum is verified and valid. - checksum bool } -func (b *block) seek(rstart, rlimit int, key []byte) (index, offset int, err error) { - n := b.restartsOffset - data := b.data - cmp := b.cmp - +func (b *block) seek(cmp comparer.Comparer, rstart, rlimit int, key []byte) (index, offset int, err error) { index = sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool { - offset := int(binary.LittleEndian.Uint32(data[n+4*(rstart+i):])) - offset += 1 // shared always zero, since this is a restart point - v1, n1 := binary.Uvarint(data[offset:]) // key length - _, n2 := binary.Uvarint(data[offset+n1:]) // value length + offset := int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*(rstart+i):])) + offset += 1 // shared always zero, since this is a restart point + v1, n1 := binary.Uvarint(b.data[offset:]) // key length + _, n2 := binary.Uvarint(b.data[offset+n1:]) // value length m := offset + n1 + n2 - return cmp.Compare(data[m:m+int(v1)], key) > 0 + return cmp.Compare(b.data[m:m+int(v1)], key) > 0 }) + rstart - 1 if index < rstart { // The smallest key is greater-than key sought. index = rstart } - offset = int(binary.LittleEndian.Uint32(data[n+4*index:])) + offset = int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*index:])) return } func (b *block) restartIndex(rstart, rlimit, offset int) int { - n := b.restartsOffset - data := b.data return sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool { - return int(binary.LittleEndian.Uint32(data[n+4*(rstart+i):])) > offset + return int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*(rstart+i):])) > offset }) + rstart - 1 } @@ -81,7 +88,7 @@ func (b *block) restartOffset(index int) int { func (b *block) entry(offset int) (key, value []byte, nShared, n int, err error) { if offset >= b.restartsOffset { if offset != b.restartsOffset { - err = errors.New("leveldb/table: Reader: BlockEntry: invalid block (block entries offset not aligned)") + err = &ErrCorrupted{Reason: "entries offset not aligned"} } return } @@ -91,7 +98,7 @@ func (b *block) entry(offset int) (key, value []byte, nShared, n int, err error) m := n0 + n1 + n2 n = m + int(v1) + int(v2) if n0 <= 0 || n1 <= 0 || n2 <= 0 || offset+n > b.restartsOffset { - err = errors.New("leveldb/table: Reader: invalid block (block entries corrupted)") + err = &ErrCorrupted{Reason: "entries corrupted"} return } key = b.data[offset+m : offset+m+int(v1)] @@ -100,43 +107,10 @@ func (b *block) entry(offset int) (key, value []byte, nShared, n int, err error) return } -func (b *block) newIterator(slice *util.Range, inclLimit bool, cache util.Releaser) *blockIter { - bi := &blockIter{ - block: b, - cache: cache, - // Valid key should never be nil. - key: make([]byte, 0), - dir: dirSOI, - riStart: 0, - riLimit: b.restartsLen, - offsetStart: 0, - offsetRealStart: 0, - offsetLimit: b.restartsOffset, - } - if slice != nil { - if slice.Start != nil { - if bi.Seek(slice.Start) { - bi.riStart = b.restartIndex(bi.restartIndex, b.restartsLen, bi.prevOffset) - bi.offsetStart = b.restartOffset(bi.riStart) - bi.offsetRealStart = bi.prevOffset - } else { - bi.riStart = b.restartsLen - bi.offsetStart = b.restartsOffset - bi.offsetRealStart = b.restartsOffset - } - } - if slice.Limit != nil { - if bi.Seek(slice.Limit) && (!inclLimit || bi.Next()) { - bi.offsetLimit = bi.prevOffset - bi.riLimit = bi.restartIndex + 1 - } - } - bi.reset() - if bi.offsetStart > bi.offsetLimit { - bi.sErr(errors.New("leveldb/table: Reader: invalid slice range")) - } - } - return bi +func (b *block) Release() { + b.bpool.Put(b.data) + b.bpool = nil + b.data = nil } type dir int @@ -150,10 +124,12 @@ const ( ) type blockIter struct { - block *block - cache, releaser util.Releaser - key, value []byte - offset int + tr *Reader + block *block + blockReleaser util.Releaser + releaser util.Releaser + key, value []byte + offset int // Previous offset, only filled by Next. prevOffset int prevNode []int @@ -250,7 +226,7 @@ func (i *blockIter) Seek(key []byte) bool { return false } - ri, offset, err := i.block.seek(i.riStart, i.riLimit, key) + ri, offset, err := i.block.seek(i.tr.cmp, i.riStart, i.riLimit, key) if err != nil { i.sErr(err) return false @@ -261,7 +237,7 @@ func (i *blockIter) Seek(key []byte) bool { i.dir = dirForward } for i.Next() { - if i.block.cmp.Compare(i.key, key) >= 0 { + if i.tr.cmp.Compare(i.key, key) >= 0 { return true } } @@ -286,7 +262,7 @@ func (i *blockIter) Next() bool { for i.offset < i.offsetRealStart { key, value, nShared, n, err := i.block.entry(i.offset) if err != nil { - i.sErr(err) + i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err)) return false } if n == 0 { @@ -300,13 +276,13 @@ func (i *blockIter) Next() bool { if i.offset >= i.offsetLimit { i.dir = dirEOI if i.offset != i.offsetLimit { - i.sErr(errors.New("leveldb/table: Reader: Next: invalid block (block entries offset not aligned)")) + i.sErr(i.tr.newErrCorruptedBH(i.block.bh, "entries offset not aligned")) } return false } key, value, nShared, n, err := i.block.entry(i.offset) if err != nil { - i.sErr(err) + i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err)) return false } if n == 0 { @@ -391,7 +367,7 @@ func (i *blockIter) Prev() bool { for { key, value, nShared, n, err := i.block.entry(offset) if err != nil { - i.sErr(err) + i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err)) return false } if offset >= i.offsetRealStart { @@ -410,7 +386,7 @@ func (i *blockIter) Prev() bool { // Stop if target offset reached. if offset >= i.offset { if offset != i.offset { - i.sErr(errors.New("leveldb/table: Reader: Prev: invalid block (block entries offset not aligned)")) + i.sErr(i.tr.newErrCorruptedBH(i.block.bh, "entries offset not aligned")) return false } @@ -437,25 +413,33 @@ func (i *blockIter) Value() []byte { } func (i *blockIter) Release() { - i.prevNode = nil - i.prevKeys = nil - i.key = nil - i.value = nil - i.dir = dirReleased - if i.cache != nil { - i.cache.Release() - i.cache = nil - } - if i.releaser != nil { - i.releaser.Release() - i.releaser = nil + if i.dir != dirReleased { + i.tr = nil + i.block = nil + i.prevNode = nil + i.prevKeys = nil + i.key = nil + i.value = nil + i.dir = dirReleased + if i.blockReleaser != nil { + i.blockReleaser.Release() + i.blockReleaser = nil + } + if i.releaser != nil { + i.releaser.Release() + i.releaser = nil + } } } func (i *blockIter) SetReleaser(releaser util.Releaser) { - if i.dir > dirReleased { - i.releaser = releaser + if i.dir == dirReleased { + panic(util.ErrReleased) } + if i.releaser != nil && releaser != nil { + panic(util.ErrHasReleaser) + } + i.releaser = releaser } func (i *blockIter) Valid() bool { @@ -467,21 +451,21 @@ func (i *blockIter) Error() error { } type filterBlock struct { - filter filter.Filter + bpool *util.BufferPool data []byte oOffset int baseLg uint filtersNum int } -func (b *filterBlock) contains(offset uint64, key []byte) bool { +func (b *filterBlock) contains(filter filter.Filter, offset uint64, key []byte) bool { i := int(offset >> b.baseLg) if i < b.filtersNum { o := b.data[b.oOffset+i*4:] n := int(binary.LittleEndian.Uint32(o)) m := int(binary.LittleEndian.Uint32(o[4:])) if n < m && m <= b.oOffset { - return b.filter.Contains(b.data[n:m], key) + return filter.Contains(b.data[n:m], key) } else if n == m { return false } @@ -489,12 +473,17 @@ func (b *filterBlock) contains(offset uint64, key []byte) bool { return true } +func (b *filterBlock) Release() { + b.bpool.Put(b.data) + b.bpool = nil + b.data = nil +} + type indexIter struct { - blockIter - tableReader *Reader - slice *util.Range + *blockIter + tr *Reader + slice *util.Range // Options - checksum bool fillCache bool } @@ -505,95 +494,173 @@ func (i *indexIter) Get() iterator.Iterator { } dataBH, n := decodeBlockHandle(value) if n == 0 { - return iterator.NewEmptyIterator(errors.New("leveldb/table: Reader: invalid table (bad data block handle)")) + return iterator.NewEmptyIterator(i.tr.newErrCorruptedBH(i.tr.indexBH, "bad data block handle")) } + var slice *util.Range if i.slice != nil && (i.blockIter.isFirst() || i.blockIter.isLast()) { slice = i.slice } - return i.tableReader.getDataIter(dataBH, slice, i.checksum, i.fillCache) + return i.tr.getDataIterErr(dataBH, slice, i.tr.verifyChecksum, i.fillCache) } // Reader is a table reader. type Reader struct { + mu sync.RWMutex + fi *storage.FileInfo reader io.ReaderAt - cache cache.Namespace + cache *cache.CacheGetter err error + bpool *util.BufferPool // Options - cmp comparer.Comparer - filter filter.Filter - checksum bool - strictIter bool + o *opt.Options + cmp comparer.Comparer + filter filter.Filter + verifyChecksum bool - dataEnd int64 - indexBlock *block - filterBlock *filterBlock + dataEnd int64 + metaBH, indexBH, filterBH blockHandle + indexBlock *block + filterBlock *filterBlock } -func verifyChecksum(data []byte) bool { - n := len(data) - 4 - checksum0 := binary.LittleEndian.Uint32(data[n:]) - checksum1 := util.NewCRC(data[:n]).Value() - return checksum0 == checksum1 +func (r *Reader) blockKind(bh blockHandle) string { + switch bh.offset { + case r.metaBH.offset: + return "meta-block" + case r.indexBH.offset: + return "index-block" + case r.filterBH.offset: + if r.filterBH.length > 0 { + return "filter-block" + } + } + return "data-block" } -func (r *Reader) readRawBlock(bh blockHandle, checksum bool) ([]byte, error) { - data := make([]byte, bh.length+blockTrailerLen) +func (r *Reader) newErrCorrupted(pos, size int64, kind, reason string) error { + return &errors.ErrCorrupted{File: r.fi, Err: &ErrCorrupted{Pos: pos, Size: size, Kind: kind, Reason: reason}} +} + +func (r *Reader) newErrCorruptedBH(bh blockHandle, reason string) error { + return r.newErrCorrupted(int64(bh.offset), int64(bh.length), r.blockKind(bh), reason) +} + +func (r *Reader) fixErrCorruptedBH(bh blockHandle, err error) error { + if cerr, ok := err.(*ErrCorrupted); ok { + cerr.Pos = int64(bh.offset) + cerr.Size = int64(bh.length) + cerr.Kind = r.blockKind(bh) + return &errors.ErrCorrupted{File: r.fi, Err: cerr} + } + return err +} + +func (r *Reader) readRawBlock(bh blockHandle, verifyChecksum bool) ([]byte, error) { + data := r.bpool.Get(int(bh.length + blockTrailerLen)) if _, err := r.reader.ReadAt(data, int64(bh.offset)); err != nil && err != io.EOF { return nil, err } - if checksum || r.checksum { - if !verifyChecksum(data) { - return nil, errors.New("leveldb/table: Reader: invalid block (checksum mismatch)") + + if verifyChecksum { + n := bh.length + 1 + checksum0 := binary.LittleEndian.Uint32(data[n:]) + checksum1 := util.NewCRC(data[:n]).Value() + if checksum0 != checksum1 { + r.bpool.Put(data) + return nil, r.newErrCorruptedBH(bh, fmt.Sprintf("checksum mismatch, want=%#x got=%#x", checksum0, checksum1)) } } + switch data[bh.length] { case blockTypeNoCompression: data = data[:bh.length] case blockTypeSnappyCompression: - var err error - data, err = snappy.Decode(nil, data[:bh.length]) + decLen, err := snappy.DecodedLen(data[:bh.length]) if err != nil { - return nil, err + return nil, r.newErrCorruptedBH(bh, err.Error()) + } + decData := r.bpool.Get(decLen) + decData, err = snappy.Decode(decData, data[:bh.length]) + r.bpool.Put(data) + if err != nil { + r.bpool.Put(decData) + return nil, r.newErrCorruptedBH(bh, err.Error()) } + data = decData default: - return nil, fmt.Errorf("leveldb/table: Reader: unknown block compression type: %d", data[bh.length]) + r.bpool.Put(data) + return nil, r.newErrCorruptedBH(bh, fmt.Sprintf("unknown compression type %#x", data[bh.length])) } return data, nil } -func (r *Reader) readBlock(bh blockHandle, checksum bool) (*block, error) { - data, err := r.readRawBlock(bh, checksum) +func (r *Reader) readBlock(bh blockHandle, verifyChecksum bool) (*block, error) { + data, err := r.readRawBlock(bh, verifyChecksum) if err != nil { return nil, err } restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:])) b := &block{ - cmp: r.cmp, + bpool: r.bpool, + bh: bh, data: data, restartsLen: restartsLen, restartsOffset: len(data) - (restartsLen+1)*4, - checksum: checksum || r.checksum, } return b, nil } -func (r *Reader) readFilterBlock(bh blockHandle, filter filter.Filter) (*filterBlock, error) { +func (r *Reader) readBlockCached(bh blockHandle, verifyChecksum, fillCache bool) (*block, util.Releaser, error) { + if r.cache != nil { + var ( + err error + ch *cache.Handle + ) + if fillCache { + ch = r.cache.Get(bh.offset, func() (size int, value cache.Value) { + var b *block + b, err = r.readBlock(bh, verifyChecksum) + if err != nil { + return 0, nil + } + return cap(b.data), b + }) + } else { + ch = r.cache.Get(bh.offset, nil) + } + if ch != nil { + b, ok := ch.Value().(*block) + if !ok { + ch.Release() + return nil, nil, errors.New("leveldb/table: inconsistent block type") + } + return b, ch, err + } else if err != nil { + return nil, nil, err + } + } + + b, err := r.readBlock(bh, verifyChecksum) + return b, b, err +} + +func (r *Reader) readFilterBlock(bh blockHandle) (*filterBlock, error) { data, err := r.readRawBlock(bh, true) if err != nil { return nil, err } n := len(data) if n < 5 { - return nil, errors.New("leveldb/table: Reader: invalid filter block (too short)") + return nil, r.newErrCorruptedBH(bh, "too short") } m := n - 5 oOffset := int(binary.LittleEndian.Uint32(data[m:])) if oOffset > m { - return nil, errors.New("leveldb/table: Reader: invalid filter block (invalid offset)") + return nil, r.newErrCorruptedBH(bh, "invalid data-offsets offset") } b := &filterBlock{ - filter: filter, + bpool: r.bpool, data: data, oOffset: oOffset, baseLg: uint(data[n-1]), @@ -602,44 +669,111 @@ func (r *Reader) readFilterBlock(bh blockHandle, filter filter.Filter) (*filterB return b, nil } -func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fillCache bool) iterator.Iterator { +func (r *Reader) readFilterBlockCached(bh blockHandle, fillCache bool) (*filterBlock, util.Releaser, error) { if r.cache != nil { - // Get/set block cache. - var err error - cache, ok := r.cache.Get(dataBH.offset, func() (ok bool, value interface{}, charge int, fin cache.SetFin) { - if !fillCache { - return + var ( + err error + ch *cache.Handle + ) + if fillCache { + ch = r.cache.Get(bh.offset, func() (size int, value cache.Value) { + var b *filterBlock + b, err = r.readFilterBlock(bh) + if err != nil { + return 0, nil + } + return cap(b.data), b + }) + } else { + ch = r.cache.Get(bh.offset, nil) + } + if ch != nil { + b, ok := ch.Value().(*filterBlock) + if !ok { + ch.Release() + return nil, nil, errors.New("leveldb/table: inconsistent block type") } - var dataBlock *block - dataBlock, err = r.readBlock(dataBH, checksum) - if err == nil { - ok = true - value = dataBlock - charge = int(dataBH.length) + return b, ch, err + } else if err != nil { + return nil, nil, err + } + } + + b, err := r.readFilterBlock(bh) + return b, b, err +} + +func (r *Reader) getIndexBlock(fillCache bool) (b *block, rel util.Releaser, err error) { + if r.indexBlock == nil { + return r.readBlockCached(r.indexBH, true, fillCache) + } + return r.indexBlock, util.NoopReleaser{}, nil +} + +func (r *Reader) getFilterBlock(fillCache bool) (*filterBlock, util.Releaser, error) { + if r.filterBlock == nil { + return r.readFilterBlockCached(r.filterBH, fillCache) + } + return r.filterBlock, util.NoopReleaser{}, nil +} + +func (r *Reader) newBlockIter(b *block, bReleaser util.Releaser, slice *util.Range, inclLimit bool) *blockIter { + bi := &blockIter{ + tr: r, + block: b, + blockReleaser: bReleaser, + // Valid key should never be nil. + key: make([]byte, 0), + dir: dirSOI, + riStart: 0, + riLimit: b.restartsLen, + offsetStart: 0, + offsetRealStart: 0, + offsetLimit: b.restartsOffset, + } + if slice != nil { + if slice.Start != nil { + if bi.Seek(slice.Start) { + bi.riStart = b.restartIndex(bi.restartIndex, b.restartsLen, bi.prevOffset) + bi.offsetStart = b.restartOffset(bi.riStart) + bi.offsetRealStart = bi.prevOffset + } else { + bi.riStart = b.restartsLen + bi.offsetStart = b.restartsOffset + bi.offsetRealStart = b.restartsOffset } - return - }) - if err != nil { - return iterator.NewEmptyIterator(err) } - if ok { - dataBlock := cache.Value().(*block) - if !dataBlock.checksum && (r.checksum || checksum) { - if !verifyChecksum(dataBlock.data) { - return iterator.NewEmptyIterator(errors.New("leveldb/table: Reader: invalid block (checksum mismatch)")) - } - dataBlock.checksum = true + if slice.Limit != nil { + if bi.Seek(slice.Limit) && (!inclLimit || bi.Next()) { + bi.offsetLimit = bi.prevOffset + bi.riLimit = bi.restartIndex + 1 } - iter := dataBlock.newIterator(slice, false, cache) - return iter + } + bi.reset() + if bi.offsetStart > bi.offsetLimit { + bi.sErr(errors.New("leveldb/table: invalid slice range")) } } - dataBlock, err := r.readBlock(dataBH, checksum) + return bi +} + +func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, verifyChecksum, fillCache bool) iterator.Iterator { + b, rel, err := r.readBlockCached(dataBH, verifyChecksum, fillCache) if err != nil { return iterator.NewEmptyIterator(err) } - iter := dataBlock.newIterator(slice, false, nil) - return iter + return r.newBlockIter(b, rel, slice, false) +} + +func (r *Reader) getDataIterErr(dataBH blockHandle, slice *util.Range, verifyChecksum, fillCache bool) iterator.Iterator { + r.mu.RLock() + defer r.mu.RUnlock() + + if r.err != nil { + return iterator.NewEmptyIterator(r.err) + } + + return r.getDataIter(dataBH, slice, verifyChecksum, fillCache) } // NewIterator creates an iterator from the table. @@ -653,35 +787,44 @@ func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fi // when not used. // // Also read Iterator documentation of the leveldb/iterator package. - func (r *Reader) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { + r.mu.RLock() + defer r.mu.RUnlock() + if r.err != nil { return iterator.NewEmptyIterator(r.err) } + fillCache := !ro.GetDontFillCache() + indexBlock, rel, err := r.getIndexBlock(fillCache) + if err != nil { + return iterator.NewEmptyIterator(err) + } index := &indexIter{ - blockIter: *r.indexBlock.newIterator(slice, true, nil), - tableReader: r, - slice: slice, - checksum: ro.GetStrict(opt.StrictBlockChecksum), - fillCache: !ro.GetDontFillCache(), + blockIter: r.newBlockIter(indexBlock, rel, slice, true), + tr: r, + slice: slice, + fillCache: !ro.GetDontFillCache(), } - return iterator.NewIndexedIterator(index, r.strictIter || ro.GetStrict(opt.StrictIterator), false) + return iterator.NewIndexedIterator(index, opt.GetStrict(r.o, ro, opt.StrictReader)) } -// Find finds key/value pair whose key is greater than or equal to the -// given key. It returns ErrNotFound if the table doesn't contain -// such pair. -// -// The caller should not modify the contents of the returned slice, but -// it is safe to modify the contents of the argument after Find returns. -func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err error) { +func (r *Reader) find(key []byte, filtered bool, ro *opt.ReadOptions, noValue bool) (rkey, value []byte, err error) { + r.mu.RLock() + defer r.mu.RUnlock() + if r.err != nil { err = r.err return } - index := r.indexBlock.newIterator(nil, true, nil) + indexBlock, rel, err := r.getIndexBlock(true) + if err != nil { + return + } + defer rel.Release() + + index := r.newBlockIter(indexBlock, nil, nil, true) defer index.Release() if !index.Seek(key) { err = index.Error() @@ -692,14 +835,23 @@ func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err } dataBH, n := decodeBlockHandle(index.Value()) if n == 0 { - err = errors.New("leveldb/table: Reader: invalid table (bad data block handle)") + r.err = r.newErrCorruptedBH(r.indexBH, "bad data block handle") return } - if r.filterBlock != nil && !r.filterBlock.contains(dataBH.offset, key) { - err = ErrNotFound - return + if filtered && r.filter != nil { + filterBlock, frel, ferr := r.getFilterBlock(true) + if ferr == nil { + if !filterBlock.contains(r.filter, dataBH.offset, key) { + frel.Release() + return nil, nil, ErrNotFound + } + frel.Release() + } else if !errors.IsCorrupted(ferr) { + err = ferr + return + } } - data := r.getDataIter(dataBH, nil, ro.GetStrict(opt.StrictBlockChecksum), !ro.GetDontFillCache()) + data := r.getDataIter(dataBH, nil, r.verifyChecksum, !ro.GetDontFillCache()) defer data.Release() if !data.Seek(key) { err = data.Error() @@ -708,23 +860,64 @@ func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err } return } + // Don't use block buffer, no need to copy the buffer. rkey = data.Key() - value = data.Value() + if !noValue { + if r.bpool == nil { + value = data.Value() + } else { + // Use block buffer, and since the buffer will be recycled, the buffer + // need to be copied. + value = append([]byte{}, data.Value()...) + } + } + return +} + +// Find finds key/value pair whose key is greater than or equal to the +// given key. It returns ErrNotFound if the table doesn't contain +// such pair. +// If filtered is true then the nearest 'block' will be checked against +// 'filter data' (if present) and will immediately return ErrNotFound if +// 'filter data' indicates that such pair doesn't exist. +// +// The caller may modify the contents of the returned slice as it is its +// own copy. +// It is safe to modify the contents of the argument after Find returns. +func (r *Reader) Find(key []byte, filtered bool, ro *opt.ReadOptions) (rkey, value []byte, err error) { + return r.find(key, filtered, ro, false) +} + +// Find finds key that is greater than or equal to the given key. +// It returns ErrNotFound if the table doesn't contain such key. +// If filtered is true then the nearest 'block' will be checked against +// 'filter data' (if present) and will immediately return ErrNotFound if +// 'filter data' indicates that such key doesn't exist. +// +// The caller may modify the contents of the returned slice as it is its +// own copy. +// It is safe to modify the contents of the argument after Find returns. +func (r *Reader) FindKey(key []byte, filtered bool, ro *opt.ReadOptions) (rkey []byte, err error) { + rkey, _, err = r.find(key, filtered, ro, true) return } // Get gets the value for the given key. It returns errors.ErrNotFound // if the table does not contain the key. // -// The caller should not modify the contents of the returned slice, but -// it is safe to modify the contents of the argument after Get returns. +// The caller may modify the contents of the returned slice as it is its +// own copy. +// It is safe to modify the contents of the argument after Find returns. func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { + r.mu.RLock() + defer r.mu.RUnlock() + if r.err != nil { err = r.err return } - rkey, value, err := r.Find(key, ro) + rkey, value, err := r.find(key, false, ro, false) if err == nil && r.cmp.Compare(rkey, key) != 0 { value = nil err = ErrNotFound @@ -736,17 +929,26 @@ func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) // // It is safe to modify the contents of the argument after Get returns. func (r *Reader) OffsetOf(key []byte) (offset int64, err error) { + r.mu.RLock() + defer r.mu.RUnlock() + if r.err != nil { err = r.err return } - index := r.indexBlock.newIterator(nil, true, nil) + indexBlock, rel, err := r.readBlockCached(r.indexBH, true, true) + if err != nil { + return + } + defer rel.Release() + + index := r.newBlockIter(indexBlock, nil, nil, true) defer index.Release() if index.Seek(key) { dataBH, n := decodeBlockHandle(index.Value()) if n == 0 { - err = errors.New("leveldb/table: Reader: invalid table (bad data block handle)") + r.err = r.newErrCorruptedBH(r.indexBH, "bad data block handle") return } offset = int64(dataBH.offset) @@ -759,90 +961,147 @@ func (r *Reader) OffsetOf(key []byte) (offset int64, err error) { return } -// NewReader creates a new initialized table reader for the file. -// The cache is optional and can be nil. -func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, o *opt.Options) *Reader { - r := &Reader{ - reader: f, - cache: cache, - cmp: o.GetComparer(), - checksum: o.GetStrict(opt.StrictBlockChecksum), - strictIter: o.GetStrict(opt.StrictIterator), +// Release implements util.Releaser. +// It also close the file if it is an io.Closer. +func (r *Reader) Release() { + r.mu.Lock() + defer r.mu.Unlock() + + if closer, ok := r.reader.(io.Closer); ok { + closer.Close() + } + if r.indexBlock != nil { + r.indexBlock.Release() + r.indexBlock = nil } + if r.filterBlock != nil { + r.filterBlock.Release() + r.filterBlock = nil + } + r.reader = nil + r.cache = nil + r.bpool = nil + r.err = ErrReaderReleased +} + +// NewReader creates a new initialized table reader for the file. +// The fi, cache and bpool is optional and can be nil. +// +// The returned table reader instance is goroutine-safe. +func NewReader(f io.ReaderAt, size int64, fi *storage.FileInfo, cache *cache.CacheGetter, bpool *util.BufferPool, o *opt.Options) (*Reader, error) { if f == nil { - r.err = errors.New("leveldb/table: Reader: nil file") - return r + return nil, errors.New("leveldb/table: nil file") } + + r := &Reader{ + fi: fi, + reader: f, + cache: cache, + bpool: bpool, + o: o, + cmp: o.GetComparer(), + verifyChecksum: o.GetStrict(opt.StrictBlockChecksum), + } + if size < footerLen { - r.err = errors.New("leveldb/table: Reader: invalid table (file size is too small)") - return r + r.err = r.newErrCorrupted(0, size, "table", "too small") + return r, nil } + + footerPos := size - footerLen var footer [footerLen]byte - if _, err := r.reader.ReadAt(footer[:], size-footerLen); err != nil && err != io.EOF { - r.err = fmt.Errorf("leveldb/table: Reader: invalid table (could not read footer): %v", err) + if _, err := r.reader.ReadAt(footer[:], footerPos); err != nil && err != io.EOF { + return nil, err } if string(footer[footerLen-len(magic):footerLen]) != magic { - r.err = errors.New("leveldb/table: Reader: invalid table (bad magic number)") - return r + r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad magic number") + return r, nil } + + var n int // Decode the metaindex block handle. - metaBH, n := decodeBlockHandle(footer[:]) + r.metaBH, n = decodeBlockHandle(footer[:]) if n == 0 { - r.err = errors.New("leveldb/table: Reader: invalid table (bad metaindex block handle)") - return r + r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad metaindex block handle") + return r, nil } + // Decode the index block handle. - indexBH, n := decodeBlockHandle(footer[n:]) + r.indexBH, n = decodeBlockHandle(footer[n:]) if n == 0 { - r.err = errors.New("leveldb/table: Reader: invalid table (bad index block handle)") - return r - } - // Read index block. - r.indexBlock, r.err = r.readBlock(indexBH, true) - if r.err != nil { - return r + r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad index block handle") + return r, nil } + // Read metaindex block. - metaBlock, err := r.readBlock(metaBH, true) + metaBlock, err := r.readBlock(r.metaBH, true) if err != nil { - r.err = err - return r + if errors.IsCorrupted(err) { + r.err = err + return r, nil + } else { + return nil, err + } } + // Set data end. - r.dataEnd = int64(metaBH.offset) - metaIter := metaBlock.newIterator(nil, false, nil) + r.dataEnd = int64(r.metaBH.offset) + + // Read metaindex. + metaIter := r.newBlockIter(metaBlock, nil, nil, true) for metaIter.Next() { key := string(metaIter.Key()) if !strings.HasPrefix(key, "filter.") { continue } fn := key[7:] - var filter filter.Filter if f0 := o.GetFilter(); f0 != nil && f0.Name() == fn { - filter = f0 + r.filter = f0 } else { for _, f0 := range o.GetAltFilters() { if f0.Name() == fn { - filter = f0 + r.filter = f0 break } } } - if filter != nil { + if r.filter != nil { filterBH, n := decodeBlockHandle(metaIter.Value()) if n == 0 { continue } + r.filterBH = filterBH // Update data end. r.dataEnd = int64(filterBH.offset) - filterBlock, err := r.readFilterBlock(filterBH, filter) - if err != nil { - continue - } - r.filterBlock = filterBlock break } } metaIter.Release() - return r + metaBlock.Release() + + // Cache index and filter block locally, since we don't have global cache. + if cache == nil { + r.indexBlock, err = r.readBlock(r.indexBH, true) + if err != nil { + if errors.IsCorrupted(err) { + r.err = err + return r, nil + } else { + return nil, err + } + } + if r.filter != nil { + r.filterBlock, err = r.readFilterBlock(r.filterBH) + if err != nil { + if !errors.IsCorrupted(err) { + return nil, err + } + + // Don't use filter then. + r.filter = nil + } + } + } + + return r, nil } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table.go index c0ac70d9e..beacdc1f0 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table.go @@ -133,9 +133,9 @@ Filter block trailer: +- 4-bytes -+ / \ - +---------------+---------------+---------------+-------------------------+------------------+ - | offset 1 | .... | offset n | filter offset (4-bytes) | base Lg (1-byte) | - +-------------- +---------------+---------------+-------------------------+------------------+ + +---------------+---------------+---------------+-------------------------------+------------------+ + | data 1 offset | .... | data n offset | data-offsets offset (4-bytes) | base Lg (1-byte) | + +-------------- +---------------+---------------+-------------------------------+------------------+ NOTE: All fixed-length integer are little-endian. diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go index bc9eb83cc..6465da6e3 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go @@ -3,15 +3,9 @@ package table import ( "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/syndtr/goleveldb/leveldb/testutil" ) func TestTable(t *testing.T) { - testutil.RunDefer() - - RegisterFailHandler(Fail) - RunSpecs(t, "Table Suite") + testutil.RunSuite(t, "Table Suite") } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_test.go index d7d3b2a4b..4b59b31f5 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_test.go @@ -23,7 +23,7 @@ type tableWrapper struct { } func (t tableWrapper) TestFind(key []byte) (rkey, rvalue []byte, err error) { - return t.Reader.Find(key, nil) + return t.Reader.Find(key, false, nil) } func (t tableWrapper) TestGet(key []byte) (value []byte, err error) { @@ -59,7 +59,8 @@ var _ = testutil.Defer(func() { It("Should be able to approximate offset of a key correctly", func() { Expect(err).ShouldNot(HaveOccurred()) - tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, o) + tr, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, nil, o) + Expect(err).ShouldNot(HaveOccurred()) CheckOffset := func(key string, expect, threshold int) { offset, err := tr.OffsetOf([]byte(key)) Expect(err).ShouldNot(HaveOccurred()) @@ -95,7 +96,7 @@ var _ = testutil.Defer(func() { tw.Close() // Opening the table. - tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, o) + tr, _ := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, nil, o) return tableWrapper{tr} } Test := func(kv *testutil.KeyValue, body func(r *Reader)) func() { @@ -104,14 +105,16 @@ var _ = testutil.Defer(func() { if body != nil { body(db.(tableWrapper).Reader) } - testutil.KeyValueTesting(nil, db, *kv) + testutil.KeyValueTesting(nil, *kv, db, nil, nil) } } - testutil.AllKeyValueTesting(nil, Build) + testutil.AllKeyValueTesting(nil, Build, nil, nil) Describe("with one key per block", Test(testutil.KeyValue_Generate(nil, 9, 1, 10, 512, 512), func(r *Reader) { It("should have correct blocks number", func() { - Expect(r.indexBlock.restartsLen).Should(Equal(9)) + indexBlock, err := r.readBlock(r.indexBH, true) + Expect(err).To(BeNil()) + Expect(indexBlock.restartsLen).Should(Equal(9)) }) })) }) diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go index 4e19e93a9..274c95fad 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go @@ -12,7 +12,7 @@ import ( "fmt" "io" - "code.google.com/p/snappy-go/snappy" + "github.com/syndtr/gosnappy/snappy" "github.com/syndtr/goleveldb/leveldb/comparer" "github.com/syndtr/goleveldb/leveldb/filter" diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/db.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/db.go index 4b87b5ef6..ec3f177a1 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/db.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/db.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/gomega" + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/util" ) @@ -34,6 +35,10 @@ type Get interface { TestGet(key []byte) (value []byte, err error) } +type Has interface { + TestHas(key []byte) (ret bool, err error) +} + type NewIterator interface { TestNewIterator(slice *util.Range) iterator.Iterator } @@ -110,7 +115,7 @@ func (t *DBTesting) TestAllPresent() { func (t *DBTesting) TestDeletedKey(key []byte) { _, err := t.DB.TestGet(key) - Expect(err).Should(Equal(util.ErrNotFound), "Get on deleted key %q, %s", key, t.text()) + Expect(err).Should(Equal(errors.ErrNotFound), "Get on deleted key %q, %s", key, t.text()) } func (t *DBTesting) TestAllDeleted() { @@ -212,5 +217,6 @@ func DoDBTesting(t *DBTesting) { } DoIteratorTesting(&it) + iter.Release() } } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/ginkgo.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/ginkgo.go new file mode 100644 index 000000000..82f3d0e81 --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/ginkgo.go @@ -0,0 +1,21 @@ +package testutil + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func RunSuite(t GinkgoTestingT, name string) { + RunDefer() + + SynchronizedBeforeSuite(func() []byte { + RunDefer("setup") + return nil + }, func(data []byte) {}) + SynchronizedAfterSuite(func() { + RunDefer("teardown") + }, func() {}) + + RegisterFailHandler(Fail) + RunSpecs(t, name) +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go index 4fc75b6f2..a0b58f0e7 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go @@ -13,16 +13,28 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/util" ) -func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) { +func KeyValueTesting(rnd *rand.Rand, kv KeyValue, p DB, setup func(KeyValue) DB, teardown func(DB)) { if rnd == nil { rnd = NewRand() } - if db, ok := p.(Find); ok { - It("Should find all keys with Find", func() { + if p == nil { + BeforeEach(func() { + p = setup(kv) + }) + if teardown != nil { + AfterEach(func() { + teardown(p) + }) + } + } + + It("Should find all keys with Find", func() { + if db, ok := p.(Find); ok { ShuffledIndex(nil, kv.Len(), 1, func(i int) { key_, key, value := kv.IndexInexact(i) @@ -38,9 +50,11 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) { Expect(rkey).Should(Equal(key)) Expect(rvalue).Should(Equal(value), "Value for key %q (%q)", key_, key) }) - }) + } + }) - It("Should return error if the key is not present", func() { + It("Should return error if the key is not present", func() { + if db, ok := p.(Find); ok { var key []byte if kv.Len() > 0 { key_, _ := kv.Index(kv.Len() - 1) @@ -48,12 +62,12 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) { } rkey, _, err := db.TestFind(key) Expect(err).Should(HaveOccurred(), "Find for key %q yield key %q", key, rkey) - Expect(err).Should(Equal(util.ErrNotFound)) - }) - } + Expect(err).Should(Equal(errors.ErrNotFound)) + } + }) - if db, ok := p.(Get); ok { - It("Should only find exact key with Get", func() { + It("Should only find exact key with Get", func() { + if db, ok := p.(Get); ok { ShuffledIndex(nil, kv.Len(), 1, func(i int) { key_, key, value := kv.IndexInexact(i) @@ -66,14 +80,34 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) { if len(key_) > 0 { _, err = db.TestGet(key_) Expect(err).Should(HaveOccurred(), "Error for key %q", key_) - Expect(err).Should(Equal(util.ErrNotFound)) + Expect(err).Should(Equal(errors.ErrNotFound)) } }) - }) - } + } + }) + + It("Should only find present key with Has", func() { + if db, ok := p.(Has); ok { + ShuffledIndex(nil, kv.Len(), 1, func(i int) { + key_, key, _ := kv.IndexInexact(i) + + // Using exact key. + ret, err := db.TestHas(key) + Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key) + Expect(ret).Should(BeTrue(), "False for key %q", key) - if db, ok := p.(NewIterator); ok { - TestIter := func(r *util.Range, _kv KeyValue) { + // Using inexact key. + if len(key_) > 0 { + ret, err = db.TestHas(key_) + Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key_) + Expect(ret).ShouldNot(BeTrue(), "True for key %q", key) + } + }) + } + }) + + TestIter := func(r *util.Range, _kv KeyValue) { + if db, ok := p.(NewIterator); ok { iter := db.TestNewIterator(r) Expect(iter.Error()).ShouldNot(HaveOccurred()) @@ -83,46 +117,62 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) { } DoIteratorTesting(&t) + iter.Release() } + } - It("Should iterates and seeks correctly", func(done Done) { - TestIter(nil, kv.Clone()) - done <- true - }, 3.0) - - RandomIndex(rnd, kv.Len(), kv.Len(), func(i int) { - type slice struct { - r *util.Range - start, limit int - } + It("Should iterates and seeks correctly", func(done Done) { + TestIter(nil, kv.Clone()) + done <- true + }, 3.0) - key_, _, _ := kv.IndexInexact(i) - for _, x := range []slice{ - {&util.Range{Start: key_, Limit: nil}, i, kv.Len()}, - {&util.Range{Start: nil, Limit: key_}, 0, i}, - } { - It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) { - TestIter(x.r, kv.Slice(x.start, x.limit)) - done <- true - }, 3.0) - } - }) + RandomIndex(rnd, kv.Len(), Min(kv.Len(), 50), func(i int) { + type slice struct { + r *util.Range + start, limit int + } - RandomRange(rnd, kv.Len(), kv.Len(), func(start, limit int) { - It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) { - r := kv.Range(start, limit) - TestIter(&r, kv.Slice(start, limit)) + key_, _, _ := kv.IndexInexact(i) + for _, x := range []slice{ + {&util.Range{Start: key_, Limit: nil}, i, kv.Len()}, + {&util.Range{Start: nil, Limit: key_}, 0, i}, + } { + It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) { + TestIter(x.r, kv.Slice(x.start, x.limit)) done <- true }, 3.0) - }) - } + } + }) + + RandomRange(rnd, kv.Len(), Min(kv.Len(), 50), func(start, limit int) { + It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) { + r := kv.Range(start, limit) + TestIter(&r, kv.Slice(start, limit)) + done <- true + }, 3.0) + }) } -func AllKeyValueTesting(rnd *rand.Rand, body func(kv KeyValue) DB) { +func AllKeyValueTesting(rnd *rand.Rand, body, setup func(KeyValue) DB, teardown func(DB)) { Test := func(kv *KeyValue) func() { return func() { - db := body(*kv) - KeyValueTesting(rnd, db, *kv) + var p DB + if setup != nil { + Defer("setup", func() { + p = setup(*kv) + }) + } + if teardown != nil { + Defer("teardown", func() { + teardown(p) + }) + } + if body != nil { + p = body(*kv) + } + KeyValueTesting(rnd, *kv, p, func(KeyValue) DB { + return p + }, nil) } } @@ -133,4 +183,5 @@ func AllKeyValueTesting(rnd *rand.Rand, body func(kv KeyValue) DB) { Describe("with big value", Test(KeyValue_BigValue())) Describe("with special key", Test(KeyValue_SpecialKey())) Describe("with multiple key/value", Test(KeyValue_MultipleKeyValue())) + Describe("with generated key/value", Test(KeyValue_Generate(nil, 120, 1, 50, 10, 120))) } diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/storage.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/storage.go index 0f8d77a73..59c496d54 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/storage.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/storage.go @@ -397,6 +397,7 @@ func (s *Storage) logI(format string, args ...interface{}) { func (s *Storage) Log(str string) { s.log(1, "Log: "+str) + s.Storage.Log(str) } func (s *Storage) Lock() (r util.Releaser, err error) { diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/util.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/util.go index 38fe25d52..97c5294b1 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/util.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/util.go @@ -155,3 +155,17 @@ func RandomRange(rnd *rand.Rand, n, round int, fn func(start, limit int)) { } return } + +func Max(x, y int) int { + if x > y { + return x + } + return y +} + +func Min(x, y int) int { + if x < y { + return x + } + return y +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil_test.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil_test.go index c1402fda3..25bf2b29f 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil_test.go @@ -34,6 +34,10 @@ func (t *testingDB) TestGet(key []byte) (value []byte, err error) { return t.Get(key, t.ro) } +func (t *testingDB) TestHas(key []byte) (ret bool, err error) { + return t.Has(key, t.ro) +} + func (t *testingDB) TestNewIterator(slice *util.Range) iterator.Iterator { return t.NewIterator(slice, t.ro) } @@ -48,6 +52,7 @@ func (t *testingDB) TestClose() { func newTestingDB(o *opt.Options, ro *opt.ReadOptions, wo *opt.WriteOptions) *testingDB { stor := testutil.NewStorage() db, err := Open(stor, o) + // FIXME: This may be called from outside It, which may cause panic. Expect(err).NotTo(HaveOccurred()) return &testingDB{ DB: db, diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util.go index a43d2e460..1a5bf71a3 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util.go @@ -14,10 +14,10 @@ import ( ) func shorten(str string) string { - if len(str) <= 4 { + if len(str) <= 8 { return str } - return str[:1] + ".." + str[len(str)-1:] + return str[:3] + ".." + str[len(str)-3:] } var bunits = [...]string{"", "Ki", "Mi", "Gi"} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go new file mode 100644 index 000000000..2b8453d75 --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go @@ -0,0 +1,238 @@ +// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com> +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package util + +import ( + "fmt" + "sync" + "sync/atomic" + "time" +) + +type buffer struct { + b []byte + miss int +} + +// BufferPool is a 'buffer pool'. +type BufferPool struct { + pool [6]chan []byte + size [5]uint32 + sizeMiss [5]uint32 + sizeHalf [5]uint32 + baseline [4]int + baseline0 int + + mu sync.RWMutex + closed bool + closeC chan struct{} + + get uint32 + put uint32 + half uint32 + less uint32 + equal uint32 + greater uint32 + miss uint32 +} + +func (p *BufferPool) poolNum(n int) int { + if n <= p.baseline0 && n > p.baseline0/2 { + return 0 + } + for i, x := range p.baseline { + if n <= x { + return i + 1 + } + } + return len(p.baseline) + 1 +} + +// Get returns buffer with length of n. +func (p *BufferPool) Get(n int) []byte { + if p == nil { + return make([]byte, n) + } + + p.mu.RLock() + defer p.mu.RUnlock() + + if p.closed { + return make([]byte, n) + } + + atomic.AddUint32(&p.get, 1) + + poolNum := p.poolNum(n) + pool := p.pool[poolNum] + if poolNum == 0 { + // Fast path. + select { + case b := <-pool: + switch { + case cap(b) > n: + if cap(b)-n >= n { + atomic.AddUint32(&p.half, 1) + select { + case pool <- b: + default: + } + return make([]byte, n) + } else { + atomic.AddUint32(&p.less, 1) + return b[:n] + } + case cap(b) == n: + atomic.AddUint32(&p.equal, 1) + return b[:n] + default: + atomic.AddUint32(&p.greater, 1) + } + default: + atomic.AddUint32(&p.miss, 1) + } + + return make([]byte, n, p.baseline0) + } else { + sizePtr := &p.size[poolNum-1] + + select { + case b := <-pool: + switch { + case cap(b) > n: + if cap(b)-n >= n { + atomic.AddUint32(&p.half, 1) + sizeHalfPtr := &p.sizeHalf[poolNum-1] + if atomic.AddUint32(sizeHalfPtr, 1) == 20 { + atomic.StoreUint32(sizePtr, uint32(cap(b)/2)) + atomic.StoreUint32(sizeHalfPtr, 0) + } else { + select { + case pool <- b: + default: + } + } + return make([]byte, n) + } else { + atomic.AddUint32(&p.less, 1) + return b[:n] + } + case cap(b) == n: + atomic.AddUint32(&p.equal, 1) + return b[:n] + default: + atomic.AddUint32(&p.greater, 1) + if uint32(cap(b)) >= atomic.LoadUint32(sizePtr) { + select { + case pool <- b: + default: + } + } + } + default: + atomic.AddUint32(&p.miss, 1) + } + + if size := atomic.LoadUint32(sizePtr); uint32(n) > size { + if size == 0 { + atomic.CompareAndSwapUint32(sizePtr, 0, uint32(n)) + } else { + sizeMissPtr := &p.sizeMiss[poolNum-1] + if atomic.AddUint32(sizeMissPtr, 1) == 20 { + atomic.StoreUint32(sizePtr, uint32(n)) + atomic.StoreUint32(sizeMissPtr, 0) + } + } + return make([]byte, n) + } else { + return make([]byte, n, size) + } + } +} + +// Put adds given buffer to the pool. +func (p *BufferPool) Put(b []byte) { + if p == nil { + return + } + + p.mu.RLock() + defer p.mu.RUnlock() + + if p.closed { + return + } + + atomic.AddUint32(&p.put, 1) + + pool := p.pool[p.poolNum(cap(b))] + select { + case pool <- b: + default: + } + +} + +func (p *BufferPool) Close() { + if p == nil { + return + } + + p.mu.Lock() + if !p.closed { + p.closed = true + p.closeC <- struct{}{} + } + p.mu.Unlock() +} + +func (p *BufferPool) String() string { + if p == nil { + return "<nil>" + } + + return fmt.Sprintf("BufferPool{B·%d Z·%v Zm·%v Zh·%v G·%d P·%d H·%d <·%d =·%d >·%d M·%d}", + p.baseline0, p.size, p.sizeMiss, p.sizeHalf, p.get, p.put, p.half, p.less, p.equal, p.greater, p.miss) +} + +func (p *BufferPool) drain() { + ticker := time.NewTicker(2 * time.Second) + for { + select { + case <-ticker.C: + for _, ch := range p.pool { + select { + case <-ch: + default: + } + } + case <-p.closeC: + close(p.closeC) + for _, ch := range p.pool { + close(ch) + } + return + } + } +} + +// NewBufferPool creates a new initialized 'buffer pool'. +func NewBufferPool(baseline int) *BufferPool { + if baseline <= 0 { + panic("baseline can't be <= 0") + } + p := &BufferPool{ + baseline0: baseline, + baseline: [...]int{baseline / 4, baseline / 2, baseline * 2, baseline * 4}, + closeC: make(chan struct{}, 1), + } + for i, cap := range []int{2, 2, 4, 4, 2, 1} { + p.pool[i] = make(chan []byte, cap) + } + go p.drain() + return p +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/pool.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/pool.go new file mode 100644 index 000000000..1f7fdd41f --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/pool.go @@ -0,0 +1,21 @@ +// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com> +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// +build go1.3 + +package util + +import ( + "sync" +) + +type Pool struct { + sync.Pool +} + +func NewPool(cap int) *Pool { + return &Pool{} +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/pool_legacy.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/pool_legacy.go new file mode 100644 index 000000000..27b8d03be --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/pool_legacy.go @@ -0,0 +1,33 @@ +// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com> +// All rights reserved. +// +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// +build !go1.3 + +package util + +type Pool struct { + pool chan interface{} +} + +func (p *Pool) Get() interface{} { + select { + case x := <-p.pool: + return x + default: + return nil + } +} + +func (p *Pool) Put(x interface{}) { + select { + case p.pool <- x: + default: + } +} + +func NewPool(cap int) *Pool { + return &Pool{pool: make(chan interface{}, cap)} +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/range.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/range.go index da0583123..85159583d 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/range.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/range.go @@ -14,3 +14,19 @@ type Range struct { // Limit of the key range, not include in the range. Limit []byte } + +// BytesPrefix returns key range that satisfy the given prefix. +// This only applicable for the standard 'bytes comparer'. +func BytesPrefix(prefix []byte) *Range { + var limit []byte + for i := len(prefix) - 1; i >= 0; i-- { + c := prefix[i] + if c < 0xff { + limit = make([]byte, i+1) + copy(limit, prefix) + limit[i] = c + 1 + break + } + } + return &Range{prefix, limit} +} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/util.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/util.go index 229c7d41f..f35976865 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/util.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/util.go @@ -12,7 +12,8 @@ import ( ) var ( - ErrNotFound = errors.New("leveldb: not found") + ErrReleased = errors.New("leveldb: resource already relesed") + ErrHasReleaser = errors.New("leveldb: releaser already defined") ) // Releaser is the interface that wraps the basic Release method. @@ -27,23 +28,46 @@ type ReleaseSetter interface { // SetReleaser associates the given releaser to the resources. The // releaser will be called once coresponding resources released. // Calling SetReleaser with nil will clear the releaser. + // + // This will panic if a releaser already present or coresponding + // resource is already released. Releaser should be cleared first + // before assigned a new one. SetReleaser(releaser Releaser) } // BasicReleaser provides basic implementation of Releaser and ReleaseSetter. type BasicReleaser struct { releaser Releaser + released bool +} + +// Released returns whether Release method already called. +func (r *BasicReleaser) Released() bool { + return r.released } // Release implements Releaser.Release. func (r *BasicReleaser) Release() { - if r.releaser != nil { - r.releaser.Release() - r.releaser = nil + if !r.released { + if r.releaser != nil { + r.releaser.Release() + r.releaser = nil + } + r.released = true } } // SetReleaser implements ReleaseSetter.SetReleaser. func (r *BasicReleaser) SetReleaser(releaser Releaser) { + if r.released { + panic(ErrReleased) + } + if r.releaser != nil && releaser != nil { + panic(ErrHasReleaser) + } r.releaser = releaser } + +type NoopReleaser struct{} + +func (NoopReleaser) Release() {} diff --git a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go index 4c54d6480..88a52f53e 100644 --- a/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go +++ b/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go @@ -7,7 +7,6 @@ package leveldb import ( - "errors" "sync/atomic" "unsafe" @@ -16,19 +15,6 @@ import ( "github.com/syndtr/goleveldb/leveldb/util" ) -var levelMaxSize [kNumLevels]float64 - -func init() { - // Precompute max size of each level - for level := range levelMaxSize { - res := float64(10 * 1048576) - for n := level; n > 1; n-- { - res *= 10 - } - levelMaxSize[level] = res - } -} - type tSet struct { level int table *tFile @@ -37,21 +23,26 @@ type tSet struct { type version struct { s *session - tables [kNumLevels]tFiles + tables []tFiles // Level that should be compacted next and its compaction score. - // Score < 1 means compaction is not strictly needed. These fields - // are initialized by ComputeCompaction() + // Score < 1 means compaction is not strictly needed. These fields + // are initialized by computeCompaction() cLevel int cScore float64 cSeek unsafe.Pointer - ref int + ref int + // Succeeding version. next *version } -func (v *version) release_NB() { +func newVersion(s *session) *version { + return &version{s: s, tables: make([]tFiles, s.o.GetNumLevel())} +} + +func (v *version) releaseNB() { v.ref-- if v.ref > 0 { return @@ -60,8 +51,6 @@ func (v *version) release_NB() { panic("negative version ref") } - s := v.s - tables := make(map[uint64]bool) for _, tt := range v.next.tables { for _, t := range tt { @@ -74,145 +63,184 @@ func (v *version) release_NB() { for _, t := range tt { num := t.file.Num() if _, ok := tables[num]; !ok { - s.tops.remove(t) + v.s.tops.remove(t) } } } - v.next.release_NB() + v.next.releaseNB() v.next = nil } func (v *version) release() { v.s.vmu.Lock() - v.release_NB() + v.releaseNB() v.s.vmu.Unlock() } -func (v *version) get(key iKey, ro *opt.ReadOptions) (value []byte, cstate bool, err error) { - s := v.s - - ukey := key.ukey() +func (v *version) walkOverlapping(ikey iKey, f func(level int, t *tFile) bool, lf func(level int) bool) { + ukey := ikey.ukey() - var tset *tSet - tseek := true - - // We can search level-by-level since entries never hop across - // levels. Therefore we are guaranteed that if we find data - // in an smaller level, later levels are irrelevant. - for level, ts := range v.tables { - if len(ts) == 0 { + // Walk tables level-by-level. + for level, tables := range v.tables { + if len(tables) == 0 { continue } if level == 0 { // Level-0 files may overlap each other. Find all files that - // overlap user_key and process them in order from newest to - var tmp tFiles - for _, t := range ts { - if s.icmp.uCompare(ukey, t.min.ukey()) >= 0 && - s.icmp.uCompare(ukey, t.max.ukey()) <= 0 { - tmp = append(tmp, t) + // overlap ukey. + for _, t := range tables { + if t.overlaps(v.s.icmp, ukey, ukey) { + if !f(level, t) { + return + } } } - - if len(tmp) == 0 { - continue - } - - tmp.sortByNum() - ts = tmp } else { - i := ts.searchMax(key, s.icmp) - if i >= len(ts) || s.icmp.uCompare(ukey, ts[i].min.ukey()) < 0 { - continue + if i := tables.searchMax(v.s.icmp, ikey); i < len(tables) { + t := tables[i] + if v.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 { + if !f(level, t) { + return + } + } } + } - ts = ts[i : i+1] + if lf != nil && !lf(level) { + return } + } +} - var l0found bool - var l0seq uint64 - var l0type vType - var l0value []byte - for _, t := range ts { - if tseek { - if tset == nil { - tset = &tSet{level, t} - } else if tset.table.incrSeek() <= 0 { - cstate = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset)) - tseek = false - } - } +func (v *version) get(ikey iKey, ro *opt.ReadOptions, noValue bool) (value []byte, tcomp bool, err error) { + ukey := ikey.ukey() - var _rkey, rval []byte - _rkey, rval, err = s.tops.get(t, key, ro) - if err == ErrNotFound { - continue - } else if err != nil { - return + var ( + tset *tSet + tseek bool + + // Level-0. + zfound bool + zseq uint64 + zkt kType + zval []byte + ) + + err = ErrNotFound + + // Since entries never hope across level, finding key/value + // in smaller level make later levels irrelevant. + v.walkOverlapping(ikey, func(level int, t *tFile) bool { + if !tseek { + if tset == nil { + tset = &tSet{level, t} + } else { + tseek = true } + } - rkey := iKey(_rkey) - if seq, t, ok := rkey.parseNum(); ok { - if s.icmp.uCompare(ukey, rkey.ukey()) == 0 { - if level == 0 { - if seq >= l0seq { - l0found = true - l0seq = seq - l0type = t - l0value = rval - } - } else { - switch t { - case tVal: - value = rval - case tDel: - err = ErrNotFound - default: - panic("invalid type") - } - return + var ( + fikey, fval []byte + ferr error + ) + if noValue { + fikey, ferr = v.s.tops.findKey(t, ikey, ro) + } else { + fikey, fval, ferr = v.s.tops.find(t, ikey, ro) + } + switch ferr { + case nil: + case ErrNotFound: + return true + default: + err = ferr + return false + } + + if fukey, fseq, fkt, fkerr := parseIkey(fikey); fkerr == nil { + if v.s.icmp.uCompare(ukey, fukey) == 0 { + if level == 0 { + if fseq >= zseq { + zfound = true + zseq = fseq + zkt = fkt + zval = fval } + } else { + switch fkt { + case ktVal: + value = fval + err = nil + case ktDel: + default: + panic("leveldb: invalid iKey type") + } + return false } - } else { - err = errors.New("leveldb: internal key corrupted") - return } + } else { + err = fkerr + return false } - if level == 0 && l0found { - switch l0type { - case tVal: - value = l0value - case tDel: - err = ErrNotFound + + return true + }, func(level int) bool { + if zfound { + switch zkt { + case ktVal: + value = zval + err = nil + case ktDel: default: - panic("invalid type") + panic("leveldb: invalid iKey type") } - return + return false } + + return true + }) + + if tseek && tset.table.consumeSeek() <= 0 { + tcomp = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset)) } - err = ErrNotFound return } -func (v *version) getIterators(slice *util.Range, ro *opt.ReadOptions) (its []iterator.Iterator) { - s := v.s +func (v *version) sampleSeek(ikey iKey) (tcomp bool) { + var tset *tSet + v.walkOverlapping(ikey, func(level int, t *tFile) bool { + if tset == nil { + tset = &tSet{level, t} + return true + } else { + if tset.table.consumeSeek() <= 0 { + tcomp = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset)) + } + return false + } + }, nil) + + return +} + +func (v *version) getIterators(slice *util.Range, ro *opt.ReadOptions) (its []iterator.Iterator) { // Merge all level zero files together since they may overlap for _, t := range v.tables[0] { - it := s.tops.newIterator(t, slice, ro) + it := v.s.tops.newIterator(t, slice, ro) its = append(its, it) } - strict := s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator) - for _, tt := range v.tables[1:] { - if len(tt) == 0 { + strict := opt.GetStrict(v.s.o.Options, ro, opt.StrictReader) + for _, tables := range v.tables[1:] { + if len(tables) == 0 { continue } - it := iterator.NewIndexedIterator(tt.newIndexIterator(s.tops, s.icmp, slice, ro), strict, true) + it := iterator.NewIndexedIterator(tables.newIndexIterator(v.s.tops, v.s.icmp, slice, ro), strict) its = append(its, it) } @@ -220,7 +248,7 @@ func (v *version) getIterators(slice *util.Range, ro *opt.ReadOptions) (its []it } func (v *version) newStaging() *versionStaging { - return &versionStaging{base: v} + return &versionStaging{base: v, tables: make([]tablesScratch, v.s.o.GetNumLevel())} } // Spawn a new version based on this version. @@ -242,25 +270,25 @@ func (v *version) tLen(level int) int { return len(v.tables[level]) } -func (v *version) offsetOf(key iKey) (n uint64, err error) { - for level, tt := range v.tables { - for _, t := range tt { - if v.s.icmp.Compare(t.max, key) <= 0 { - // Entire file is before "key", so just add the file size +func (v *version) offsetOf(ikey iKey) (n uint64, err error) { + for level, tables := range v.tables { + for _, t := range tables { + if v.s.icmp.Compare(t.imax, ikey) <= 0 { + // Entire file is before "ikey", so just add the file size n += t.size - } else if v.s.icmp.Compare(t.min, key) > 0 { - // Entire file is after "key", so ignore + } else if v.s.icmp.Compare(t.imin, ikey) > 0 { + // Entire file is after "ikey", so ignore if level > 0 { // Files other than level 0 are sorted by meta->min, so // no further files in this level will contain data for - // "key". + // "ikey". break } } else { - // "key" falls in the range for this table. Add the - // approximate offset of "key" within the table. + // "ikey" falls in the range for this table. Add the + // approximate offset of "ikey" within the table. var nn uint64 - nn, err = v.s.tops.offsetOf(t, key) + nn, err = v.s.tops.offsetOf(t, ikey) if err != nil { return 0, err } @@ -272,15 +300,16 @@ func (v *version) offsetOf(key iKey) (n uint64, err error) { return } -func (v *version) pickLevel(min, max []byte) (level int) { - if !v.tables[0].isOverlaps(min, max, false, v.s.icmp) { - var r tFiles - for ; level < kMaxMemCompactLevel; level++ { - if v.tables[level+1].isOverlaps(min, max, true, v.s.icmp) { +func (v *version) pickLevel(umin, umax []byte) (level int) { + if !v.tables[0].overlaps(v.s.icmp, umin, umax, true) { + var overlaps tFiles + maxLevel := v.s.o.GetMaxMemCompationLevel() + for ; level < maxLevel; level++ { + if v.tables[level+1].overlaps(v.s.icmp, umin, umax, false) { break } - v.tables[level+2].getOverlaps(min, max, &r, true, v.s.icmp.ucmp) - if r.size() > kMaxGrandParentOverlapBytes { + overlaps = v.tables[level+2].getOverlaps(overlaps, v.s.icmp, umin, umax, false) + if overlaps.size() > uint64(v.s.o.GetCompactionGPOverlaps(level)) { break } } @@ -294,7 +323,7 @@ func (v *version) computeCompaction() { var bestLevel int = -1 var bestScore float64 = -1 - for level, ff := range v.tables { + for level, tables := range v.tables { var score float64 if level == 0 { // We treat level-0 specially by bounding the number of files @@ -308,9 +337,9 @@ func (v *version) computeCompaction() { // file size is small (perhaps because of a small write-buffer // setting, or very high compression ratios, or lots of // overwrites/deletions). - score = float64(len(ff)) / kL0_CompactionTrigger + score = float64(len(tables)) / float64(v.s.o.GetCompactionL0Trigger()) } else { - score = float64(ff.size()) / levelMaxSize[level] + score = float64(tables.size()) / float64(v.s.o.GetCompactionTotalSize(level)) } if score > bestScore { @@ -327,66 +356,62 @@ func (v *version) needCompaction() bool { return v.cScore >= 1 || atomic.LoadPointer(&v.cSeek) != nil } +type tablesScratch struct { + added map[uint64]atRecord + deleted map[uint64]struct{} +} + type versionStaging struct { base *version - tables [kNumLevels]struct { - added map[uint64]ntRecord - deleted map[uint64]struct{} - } + tables []tablesScratch } func (p *versionStaging) commit(r *sessionRecord) { - btt := p.base.tables - - // deleted tables - for _, tr := range r.deletedTables { - tm := &(p.tables[tr.level]) + // Deleted tables. + for _, r := range r.deletedTables { + tm := &(p.tables[r.level]) - bt := btt[tr.level] - if len(bt) > 0 { + if len(p.base.tables[r.level]) > 0 { if tm.deleted == nil { tm.deleted = make(map[uint64]struct{}) } - tm.deleted[tr.num] = struct{}{} + tm.deleted[r.num] = struct{}{} } if tm.added != nil { - delete(tm.added, tr.num) + delete(tm.added, r.num) } } - // new tables - for _, tr := range r.addedTables { - tm := &(p.tables[tr.level]) + // New tables. + for _, r := range r.addedTables { + tm := &(p.tables[r.level]) if tm.added == nil { - tm.added = make(map[uint64]ntRecord) + tm.added = make(map[uint64]atRecord) } - tm.added[tr.num] = tr + tm.added[r.num] = r if tm.deleted != nil { - delete(tm.deleted, tr.num) + delete(tm.deleted, r.num) } } } func (p *versionStaging) finish() *version { - s := p.base.s - btt := p.base.tables - - // build new version - nv := &version{s: s} + // Build new version. + nv := newVersion(p.base.s) for level, tm := range p.tables { - bt := btt[level] + btables := p.base.tables[level] - n := len(bt) + len(tm.added) - len(tm.deleted) + n := len(btables) + len(tm.added) - len(tm.deleted) if n < 0 { n = 0 } nt := make(tFiles, 0, n) - // base tables - for _, t := range bt { + // Base tables. + for _, t := range btables { if _, ok := tm.deleted[t.file.Num()]; ok { continue } @@ -396,17 +421,21 @@ func (p *versionStaging) finish() *version { nt = append(nt, t) } - // new tables - for _, tr := range tm.added { - nt = append(nt, tr.makeFile(s)) + // New tables. + for _, r := range tm.added { + nt = append(nt, p.base.s.tableFileFromRecord(r)) } - // sort tables - nt.sortByKey(s.icmp) + // Sort tables. + if level == 0 { + nt.sortByNum() + } else { + nt.sortByKey(p.base.s.icmp) + } nv.tables[level] = nt } - // compute compaction score for new version + // Compute compaction score for new version. nv.computeCompaction() return nv @@ -421,7 +450,7 @@ func (vr *versionReleaser) Release() { v := vr.v v.s.vmu.Lock() if !vr.once { - v.release_NB() + v.releaseNB() vr.once = true } v.s.vmu.Unlock() diff --git a/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go new file mode 100644 index 000000000..552a17bfb --- /dev/null +++ b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go @@ -0,0 +1,292 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package snappy + +import ( + "encoding/binary" + "errors" + "io" +) + +var ( + // ErrCorrupt reports that the input is invalid. + ErrCorrupt = errors.New("snappy: corrupt input") + // ErrUnsupported reports that the input isn't supported. + ErrUnsupported = errors.New("snappy: unsupported input") +) + +// DecodedLen returns the length of the decoded block. +func DecodedLen(src []byte) (int, error) { + v, _, err := decodedLen(src) + return v, err +} + +// decodedLen returns the length of the decoded block and the number of bytes +// that the length header occupied. +func decodedLen(src []byte) (blockLen, headerLen int, err error) { + v, n := binary.Uvarint(src) + if n == 0 { + return 0, 0, ErrCorrupt + } + if uint64(int(v)) != v { + return 0, 0, errors.New("snappy: decoded block is too large") + } + return int(v), n, nil +} + +// Decode returns the decoded form of src. The returned slice may be a sub- +// slice of dst if dst was large enough to hold the entire decoded block. +// Otherwise, a newly allocated slice will be returned. +// It is valid to pass a nil dst. +func Decode(dst, src []byte) ([]byte, error) { + dLen, s, err := decodedLen(src) + if err != nil { + return nil, err + } + if len(dst) < dLen { + dst = make([]byte, dLen) + } + + var d, offset, length int + for s < len(src) { + switch src[s] & 0x03 { + case tagLiteral: + x := uint(src[s] >> 2) + switch { + case x < 60: + s += 1 + case x == 60: + s += 2 + if s > len(src) { + return nil, ErrCorrupt + } + x = uint(src[s-1]) + case x == 61: + s += 3 + if s > len(src) { + return nil, ErrCorrupt + } + x = uint(src[s-2]) | uint(src[s-1])<<8 + case x == 62: + s += 4 + if s > len(src) { + return nil, ErrCorrupt + } + x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16 + case x == 63: + s += 5 + if s > len(src) { + return nil, ErrCorrupt + } + x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24 + } + length = int(x + 1) + if length <= 0 { + return nil, errors.New("snappy: unsupported literal length") + } + if length > len(dst)-d || length > len(src)-s { + return nil, ErrCorrupt + } + copy(dst[d:], src[s:s+length]) + d += length + s += length + continue + + case tagCopy1: + s += 2 + if s > len(src) { + return nil, ErrCorrupt + } + length = 4 + int(src[s-2])>>2&0x7 + offset = int(src[s-2])&0xe0<<3 | int(src[s-1]) + + case tagCopy2: + s += 3 + if s > len(src) { + return nil, ErrCorrupt + } + length = 1 + int(src[s-3])>>2 + offset = int(src[s-2]) | int(src[s-1])<<8 + + case tagCopy4: + return nil, errors.New("snappy: unsupported COPY_4 tag") + } + + end := d + length + if offset > d || end > len(dst) { + return nil, ErrCorrupt + } + for ; d < end; d++ { + dst[d] = dst[d-offset] + } + } + if d != dLen { + return nil, ErrCorrupt + } + return dst[:d], nil +} + +// NewReader returns a new Reader that decompresses from r, using the framing +// format described at +// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt +func NewReader(r io.Reader) *Reader { + return &Reader{ + r: r, + decoded: make([]byte, maxUncompressedChunkLen), + buf: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize), + } +} + +// Reader is an io.Reader than can read Snappy-compressed bytes. +type Reader struct { + r io.Reader + err error + decoded []byte + buf []byte + // decoded[i:j] contains decoded bytes that have not yet been passed on. + i, j int + readHeader bool +} + +// Reset discards any buffered data, resets all state, and switches the Snappy +// reader to read from r. This permits reusing a Reader rather than allocating +// a new one. +func (r *Reader) Reset(reader io.Reader) { + r.r = reader + r.err = nil + r.i = 0 + r.j = 0 + r.readHeader = false +} + +func (r *Reader) readFull(p []byte) (ok bool) { + if _, r.err = io.ReadFull(r.r, p); r.err != nil { + if r.err == io.ErrUnexpectedEOF { + r.err = ErrCorrupt + } + return false + } + return true +} + +// Read satisfies the io.Reader interface. +func (r *Reader) Read(p []byte) (int, error) { + if r.err != nil { + return 0, r.err + } + for { + if r.i < r.j { + n := copy(p, r.decoded[r.i:r.j]) + r.i += n + return n, nil + } + if !r.readFull(r.buf[:4]) { + return 0, r.err + } + chunkType := r.buf[0] + if !r.readHeader { + if chunkType != chunkTypeStreamIdentifier { + r.err = ErrCorrupt + return 0, r.err + } + r.readHeader = true + } + chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 + if chunkLen > len(r.buf) { + r.err = ErrUnsupported + return 0, r.err + } + + // The chunk types are specified at + // https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt + switch chunkType { + case chunkTypeCompressedData: + // Section 4.2. Compressed data (chunk type 0x00). + if chunkLen < checksumSize { + r.err = ErrCorrupt + return 0, r.err + } + buf := r.buf[:chunkLen] + if !r.readFull(buf) { + return 0, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + buf = buf[checksumSize:] + + n, err := DecodedLen(buf) + if err != nil { + r.err = err + return 0, r.err + } + if n > len(r.decoded) { + r.err = ErrCorrupt + return 0, r.err + } + if _, err := Decode(r.decoded, buf); err != nil { + r.err = err + return 0, r.err + } + if crc(r.decoded[:n]) != checksum { + r.err = ErrCorrupt + return 0, r.err + } + r.i, r.j = 0, n + continue + + case chunkTypeUncompressedData: + // Section 4.3. Uncompressed data (chunk type 0x01). + if chunkLen < checksumSize { + r.err = ErrCorrupt + return 0, r.err + } + buf := r.buf[:checksumSize] + if !r.readFull(buf) { + return 0, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + // Read directly into r.decoded instead of via r.buf. + n := chunkLen - checksumSize + if !r.readFull(r.decoded[:n]) { + return 0, r.err + } + if crc(r.decoded[:n]) != checksum { + r.err = ErrCorrupt + return 0, r.err + } + r.i, r.j = 0, n + continue + + case chunkTypeStreamIdentifier: + // Section 4.1. Stream identifier (chunk type 0xff). + if chunkLen != len(magicBody) { + r.err = ErrCorrupt + return 0, r.err + } + if !r.readFull(r.buf[:len(magicBody)]) { + return 0, r.err + } + for i := 0; i < len(magicBody); i++ { + if r.buf[i] != magicBody[i] { + r.err = ErrCorrupt + return 0, r.err + } + } + continue + } + + if chunkType <= 0x7f { + // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). + r.err = ErrUnsupported + return 0, r.err + + } else { + // Section 4.4 Padding (chunk type 0xfe). + // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). + if !r.readFull(r.buf[:chunkLen]) { + return 0, r.err + } + } + } +} diff --git a/Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/encode.go b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go index b2371db11..dda372422 100644 --- a/Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/encode.go +++ b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go @@ -6,6 +6,7 @@ package snappy import ( "encoding/binary" + "io" ) // We limit how far copy back-references can go, the same as the C++ code. @@ -172,3 +173,86 @@ func MaxEncodedLen(srcLen int) int { // This last factor dominates the blowup, so the final estimate is: return 32 + srcLen + srcLen/6 } + +// NewWriter returns a new Writer that compresses to w, using the framing +// format described at +// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt +func NewWriter(w io.Writer) *Writer { + return &Writer{ + w: w, + enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)), + } +} + +// Writer is an io.Writer than can write Snappy-compressed bytes. +type Writer struct { + w io.Writer + err error + enc []byte + buf [checksumSize + chunkHeaderSize]byte + wroteHeader bool +} + +// Reset discards the writer's state and switches the Snappy writer to write to +// w. This permits reusing a Writer rather than allocating a new one. +func (w *Writer) Reset(writer io.Writer) { + w.w = writer + w.err = nil + w.wroteHeader = false +} + +// Write satisfies the io.Writer interface. +func (w *Writer) Write(p []byte) (n int, errRet error) { + if w.err != nil { + return 0, w.err + } + if !w.wroteHeader { + copy(w.enc, magicChunk) + if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil { + w.err = err + return n, err + } + w.wroteHeader = true + } + for len(p) > 0 { + var uncompressed []byte + if len(p) > maxUncompressedChunkLen { + uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:] + } else { + uncompressed, p = p, nil + } + checksum := crc(uncompressed) + + // Compress the buffer, discarding the result if the improvement + // isn't at least 12.5%. + chunkType := uint8(chunkTypeCompressedData) + chunkBody, err := Encode(w.enc, uncompressed) + if err != nil { + w.err = err + return n, err + } + if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 { + chunkType, chunkBody = chunkTypeUncompressedData, uncompressed + } + + chunkLen := 4 + len(chunkBody) + w.buf[0] = chunkType + w.buf[1] = uint8(chunkLen >> 0) + w.buf[2] = uint8(chunkLen >> 8) + w.buf[3] = uint8(chunkLen >> 16) + w.buf[4] = uint8(checksum >> 0) + w.buf[5] = uint8(checksum >> 8) + w.buf[6] = uint8(checksum >> 16) + w.buf[7] = uint8(checksum >> 24) + if _, err = w.w.Write(w.buf[:]); err != nil { + w.err = err + return n, err + } + if _, err = w.w.Write(chunkBody); err != nil { + w.err = err + return n, err + } + n += len(uncompressed) + } + return n, nil +} diff --git a/Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/snappy.go b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go index 2f1b790d0..043bf3d81 100644 --- a/Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/snappy.go +++ b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go @@ -8,6 +8,10 @@ // The C++ snappy implementation is at http://code.google.com/p/snappy/ package snappy +import ( + "hash/crc32" +) + /* Each encoded block begins with the varint-encoded length of the decoded data, followed by a sequence of chunks. Chunks begin and end on byte boundaries. The @@ -36,3 +40,29 @@ const ( tagCopy2 = 0x02 tagCopy4 = 0x03 ) + +const ( + checksumSize = 4 + chunkHeaderSize = 4 + magicChunk = "\xff\x06\x00\x00" + magicBody + magicBody = "sNaPpY" + // https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt says + // that "the uncompressed data in a chunk must be no longer than 65536 bytes". + maxUncompressedChunkLen = 65536 +) + +const ( + chunkTypeCompressedData = 0x00 + chunkTypeUncompressedData = 0x01 + chunkTypePadding = 0xfe + chunkTypeStreamIdentifier = 0xff +) + +var crcTable = crc32.MakeTable(crc32.Castagnoli) + +// crc implements the checksum specified in section 3 of +// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt +func crc(b []byte) uint32 { + c := crc32.Update(0, crcTable, b) + return uint32(c>>15|c<<17) + 0xa282ead8 +} diff --git a/Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/snappy_test.go b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go index 7ba839244..0623385b7 100644 --- a/Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/snappy_test.go +++ b/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go @@ -18,7 +18,10 @@ import ( "testing" ) -var download = flag.Bool("download", false, "If true, download any missing files before running benchmarks") +var ( + download = flag.Bool("download", false, "If true, download any missing files before running benchmarks") + testdata = flag.String("testdata", "testdata", "Directory containing the test data") +) func roundtrip(b, ebuf, dbuf []byte) error { e, err := Encode(ebuf, b) @@ -55,11 +58,11 @@ func TestSmallCopy(t *testing.T) { } func TestSmallRand(t *testing.T) { - rand.Seed(27354294) + rng := rand.New(rand.NewSource(27354294)) for n := 1; n < 20000; n += 23 { b := make([]byte, n) - for i, _ := range b { - b[i] = uint8(rand.Uint32()) + for i := range b { + b[i] = uint8(rng.Uint32()) } if err := roundtrip(b, nil, nil); err != nil { t.Fatal(err) @@ -70,7 +73,7 @@ func TestSmallRand(t *testing.T) { func TestSmallRegular(t *testing.T) { for n := 1; n < 20000; n += 23 { b := make([]byte, n) - for i, _ := range b { + for i := range b { b[i] = uint8(i%10 + 'a') } if err := roundtrip(b, nil, nil); err != nil { @@ -79,6 +82,120 @@ func TestSmallRegular(t *testing.T) { } } +func cmp(a, b []byte) error { + if len(a) != len(b) { + return fmt.Errorf("got %d bytes, want %d", len(a), len(b)) + } + for i := range a { + if a[i] != b[i] { + return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i]) + } + } + return nil +} + +func TestFramingFormat(t *testing.T) { + // src is comprised of alternating 1e5-sized sequences of random + // (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen + // because it is larger than maxUncompressedChunkLen (64k). + src := make([]byte, 1e6) + rng := rand.New(rand.NewSource(1)) + for i := 0; i < 10; i++ { + if i%2 == 0 { + for j := 0; j < 1e5; j++ { + src[1e5*i+j] = uint8(rng.Intn(256)) + } + } else { + for j := 0; j < 1e5; j++ { + src[1e5*i+j] = uint8(i) + } + } + } + + buf := new(bytes.Buffer) + if _, err := NewWriter(buf).Write(src); err != nil { + t.Fatalf("Write: encoding: %v", err) + } + dst, err := ioutil.ReadAll(NewReader(buf)) + if err != nil { + t.Fatalf("ReadAll: decoding: %v", err) + } + if err := cmp(dst, src); err != nil { + t.Fatal(err) + } +} + +func TestReaderReset(t *testing.T) { + gold := bytes.Repeat([]byte("All that is gold does not glitter,\n"), 10000) + buf := new(bytes.Buffer) + if _, err := NewWriter(buf).Write(gold); err != nil { + t.Fatalf("Write: %v", err) + } + encoded, invalid, partial := buf.String(), "invalid", "partial" + r := NewReader(nil) + for i, s := range []string{encoded, invalid, partial, encoded, partial, invalid, encoded, encoded} { + if s == partial { + r.Reset(strings.NewReader(encoded)) + if _, err := r.Read(make([]byte, 101)); err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + continue + } + r.Reset(strings.NewReader(s)) + got, err := ioutil.ReadAll(r) + switch s { + case encoded: + if err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + if err := cmp(got, gold); err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + case invalid: + if err == nil { + t.Errorf("#%d: got nil error, want non-nil", i) + continue + } + } + } +} + +func TestWriterReset(t *testing.T) { + gold := bytes.Repeat([]byte("Not all those who wander are lost;\n"), 10000) + var gots, wants [][]byte + const n = 20 + w, failed := NewWriter(nil), false + for i := 0; i <= n; i++ { + buf := new(bytes.Buffer) + w.Reset(buf) + want := gold[:len(gold)*i/n] + if _, err := w.Write(want); err != nil { + t.Errorf("#%d: Write: %v", i, err) + failed = true + continue + } + got, err := ioutil.ReadAll(NewReader(buf)) + if err != nil { + t.Errorf("#%d: ReadAll: %v", i, err) + failed = true + continue + } + gots = append(gots, got) + wants = append(wants, want) + } + if failed { + return + } + for i := range gots { + if err := cmp(gots[i], wants[i]); err != nil { + t.Errorf("#%d: %v", i, err) + } + } +} + func benchDecode(b *testing.B, src []byte) { encoded, err := Encode(nil, src) if err != nil { @@ -102,7 +219,7 @@ func benchEncode(b *testing.B, src []byte) { } } -func readFile(b *testing.B, filename string) []byte { +func readFile(b testing.TB, filename string) []byte { src, err := ioutil.ReadFile(filename) if err != nil { b.Fatalf("failed reading %s: %s", filename, err) @@ -144,7 +261,7 @@ func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) } func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) } // testFiles' values are copied directly from -// https://code.google.com/p/snappy/source/browse/trunk/snappy_unittest.cc. +// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc // The label field is unused in snappy-go. var testFiles = []struct { label string @@ -152,29 +269,36 @@ var testFiles = []struct { }{ {"html", "html"}, {"urls", "urls.10K"}, - {"jpg", "house.jpg"}, - {"pdf", "mapreduce-osdi-1.pdf"}, + {"jpg", "fireworks.jpeg"}, + {"jpg_200", "fireworks.jpeg"}, + {"pdf", "paper-100k.pdf"}, {"html4", "html_x_4"}, - {"cp", "cp.html"}, - {"c", "fields.c"}, - {"lsp", "grammar.lsp"}, - {"xls", "kennedy.xls"}, {"txt1", "alice29.txt"}, {"txt2", "asyoulik.txt"}, {"txt3", "lcet10.txt"}, {"txt4", "plrabn12.txt"}, - {"bin", "ptt5"}, - {"sum", "sum"}, - {"man", "xargs.1"}, {"pb", "geo.protodata"}, {"gaviota", "kppkn.gtb"}, } // The test data files are present at this canonical URL. -const baseURL = "https://snappy.googlecode.com/svn/trunk/testdata/" +const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/" func downloadTestdata(basename string) (errRet error) { - filename := filepath.Join("testdata", basename) + filename := filepath.Join(*testdata, basename) + if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 { + return nil + } + + if !*download { + return fmt.Errorf("test data not found; skipping benchmark without the -download flag") + } + // Download the official snappy C++ implementation reference test data + // files for benchmarking. + if err := os.Mkdir(*testdata, 0777); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create testdata: %s", err) + } + f, err := os.Create(filename) if err != nil { return fmt.Errorf("failed to create %s: %s", filename, err) @@ -185,36 +309,27 @@ func downloadTestdata(basename string) (errRet error) { os.Remove(filename) } }() - resp, err := http.Get(baseURL + basename) + url := baseURL + basename + resp, err := http.Get(url) if err != nil { - return fmt.Errorf("failed to download %s: %s", baseURL+basename, err) + return fmt.Errorf("failed to download %s: %s", url, err) } defer resp.Body.Close() + if s := resp.StatusCode; s != http.StatusOK { + return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s)) + } _, err = io.Copy(f, resp.Body) if err != nil { - return fmt.Errorf("failed to write %s: %s", filename, err) + return fmt.Errorf("failed to download %s to %s: %s", url, filename, err) } return nil } func benchFile(b *testing.B, n int, decode bool) { - filename := filepath.Join("testdata", testFiles[n].filename) - if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 { - if !*download { - b.Fatal("test data not found; skipping benchmark without the -download flag") - } - // Download the official snappy C++ implementation reference test data - // files for benchmarking. - if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) { - b.Fatalf("failed to create testdata: %s", err) - } - for _, tf := range testFiles { - if err := downloadTestdata(tf.filename); err != nil { - b.Fatalf("failed to download testdata: %s", err) - } - } + if err := downloadTestdata(testFiles[n].filename); err != nil { + b.Fatalf("failed to download testdata: %s", err) } - data := readFile(b, filename) + data := readFile(b, filepath.Join(*testdata, testFiles[n].filename)) if decode { benchDecode(b, data) } else { @@ -235,12 +350,6 @@ func Benchmark_UFlat8(b *testing.B) { benchFile(b, 8, true) } func Benchmark_UFlat9(b *testing.B) { benchFile(b, 9, true) } func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) } func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) } -func Benchmark_UFlat12(b *testing.B) { benchFile(b, 12, true) } -func Benchmark_UFlat13(b *testing.B) { benchFile(b, 13, true) } -func Benchmark_UFlat14(b *testing.B) { benchFile(b, 14, true) } -func Benchmark_UFlat15(b *testing.B) { benchFile(b, 15, true) } -func Benchmark_UFlat16(b *testing.B) { benchFile(b, 16, true) } -func Benchmark_UFlat17(b *testing.B) { benchFile(b, 17, true) } func Benchmark_ZFlat0(b *testing.B) { benchFile(b, 0, false) } func Benchmark_ZFlat1(b *testing.B) { benchFile(b, 1, false) } func Benchmark_ZFlat2(b *testing.B) { benchFile(b, 2, false) } @@ -253,9 +362,3 @@ func Benchmark_ZFlat8(b *testing.B) { benchFile(b, 8, false) } func Benchmark_ZFlat9(b *testing.B) { benchFile(b, 9, false) } func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) } func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) } -func Benchmark_ZFlat12(b *testing.B) { benchFile(b, 12, false) } -func Benchmark_ZFlat13(b *testing.B) { benchFile(b, 13, false) } -func Benchmark_ZFlat14(b *testing.B) { benchFile(b, 14, false) } -func Benchmark_ZFlat15(b *testing.B) { benchFile(b, 15, false) } -func Benchmark_ZFlat16(b *testing.B) { benchFile(b, 16, false) } -func Benchmark_ZFlat17(b *testing.B) { benchFile(b, 17, false) } |