indexer: support data exporting/forwarding (#103)

To support more effective and flexible blockchain info exploring, we add
`indexer` package, defines the flow of indexer dameon, and integrate
into dex.Dexon fullnode.

For more export options, we use Golang built-in `plugin` package to
support mulitple implementations.

8 files changed, 190 insertions, 0 deletions

diff --git a/cmd/gdex/main.go b/cmd/gdex/main.go
index 9a1ef7bc9..c054e3ad5 100644
--- a/cmd/gdex/main.go
+++ b/cmd/gdex/main.go
@@ -137,6 +137,8 @@ var (
 		utils.GpoPercentileFlag,
 		utils.EWASMInterpreterFlag,
 		utils.EVMInterpreterFlag,
+		utils.IndexerEnableFlag,
+		utils.IndexerPluginFlag,
 		configFileFlag,
 	}
 
diff --git a/cmd/gdex/usage.go b/cmd/gdex/usage.go
index d67817096..880e447f3 100644
--- a/cmd/gdex/usage.go
+++ b/cmd/gdex/usage.go
@@ -240,6 +240,13 @@ var AppHelpFlagGroups = []flagGroup{
 		},
 	},
 	{
+		Name: "INDEXER",
+		Flags: []cli.Flag{
+			utils.IndexerEnableFlag,
+			utils.IndexerPluginFlag,
+		},
+	},
+	{
 		Name:  "WHISPER (EXPERIMENTAL)",
 		Flags: whisperFlags,
 	},
diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
index 971ccd349..96a642850 100644
--- a/cmd/utils/flags.go
+++ b/cmd/utils/flags.go
@@ -648,6 +648,17 @@ var (
 		Usage: "External EVM configuration (default = built-in interpreter)",
 		Value: "",
 	}
+
+	// Data sources options.
+	IndexerEnableFlag = cli.BoolFlag{
+		Name:  "indexer",
+		Usage: "Enable indexer",
+	}
+	IndexerPluginFlag = cli.StringFlag{
+		Name:  "indexer.plugin",
+		Usage: "External indexer plugin shared object path",
+		Value: "",
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1282,6 +1293,18 @@ func SetDexConfig(ctx *cli.Context, stack *node.Node, cfg *dex.Config) {
 			now.Hour(), now.Minute(), (now.Second()/5+1)*5,
 			0, now.Location()).Unix()
 	}
+
+	// Set indexer config.
+	setIndexerConfig(ctx, cfg)
+}
+
+func setIndexerConfig(ctx *cli.Context, cfg *dex.Config) {
+	cfg.Indexer.Enable = ctx.GlobalBool(IndexerEnableFlag.Name)
+	if !cfg.Indexer.Enable {
+		return
+	}
+
+	cfg.Indexer.Plugin = ctx.GlobalString(IndexerPluginFlag.Name)
 }
 
 // SetDashboardConfig applies dashboard related command line flags to the config.
diff --git a/dex/backend.go b/dex/backend.go
index 8fe38cd45..7b1f4905d 100644
--- a/dex/backend.go
+++ b/dex/backend.go
@@ -33,6 +33,7 @@ import (
 	"github.com/dexon-foundation/dexon/eth/gasprice"
 	"github.com/dexon-foundation/dexon/ethdb"
 	"github.com/dexon-foundation/dexon/event"
+	"github.com/dexon-foundation/dexon/indexer"
 	"github.com/dexon-foundation/dexon/internal/ethapi"
 	"github.com/dexon-foundation/dexon/log"
 	"github.com/dexon-foundation/dexon/node"
@@ -75,6 +76,8 @@ type Dexon struct {
 
 	networkID     uint64
 	netRPCService *ethapi.PublicNetAPI
+
+	indexer indexer.Indexer
 }
 
 func New(ctx *node.ServiceContext, config *Config) (*Dexon, error) {
@@ -132,6 +135,14 @@ func New(ctx *node.ServiceContext, config *Config) (*Dexon, error) {
 	}
 	dex.bloomIndexer.Start(dex.blockchain)
 
+	if config.Indexer.Enable {
+		dex.indexer = indexer.NewIndexerFromConfig(
+			indexer.NewROBlockChain(dex.blockchain),
+			config.Indexer,
+		)
+		dex.indexer.Start()
+	}
+
 	if config.TxPool.Journal != "" {
 		config.TxPool.Journal = ctx.ResolvePath(config.TxPool.Journal)
 	}
@@ -241,6 +252,9 @@ func (s *Dexon) Start(srvr *p2p.Server) error {
 func (s *Dexon) Stop() error {
 	s.bp.Stop()
 	s.app.Stop()
+	if s.indexer != nil {
+		s.indexer.Stop()
+	}
 	return nil
 }
 
diff --git a/dex/config.go b/dex/config.go
index 5f148cc03..7661907cc 100644
--- a/dex/config.go
+++ b/dex/config.go
@@ -28,6 +28,7 @@ import (
 	"github.com/dexon-foundation/dexon/core"
 	"github.com/dexon-foundation/dexon/dex/downloader"
 	"github.com/dexon-foundation/dexon/eth/gasprice"
+	"github.com/dexon-foundation/dexon/indexer"
 	"github.com/dexon-foundation/dexon/params"
 )
 
@@ -48,6 +49,7 @@ var DefaultConfig = Config{
 	},
 	BlockProposerEnabled: false,
 	DefaultGasPrice:      big.NewInt(params.GWei),
+	Indexer:              indexer.Config{},
 }
 
 func init() {
@@ -121,4 +123,7 @@ type Config struct {
 
 	// Dexon options
 	DMoment int64
+
+	// Indexer config
+	Indexer indexer.Config
 }
diff --git a/indexer/blockhain.go b/indexer/blockhain.go
new file mode 100644
index 000000000..bf9a180ce
--- /dev/null
+++ b/indexer/blockhain.go
@@ -0,0 +1,85 @@
+package indexer
+
+import (
+	"math/big"
+
+	coreCommon "github.com/dexon-foundation/dexon-consensus/common"
+	coreTypes "github.com/dexon-foundation/dexon-consensus/core/types"
+
+	"github.com/dexon-foundation/dexon/common"
+	"github.com/dexon-foundation/dexon/core"
+	"github.com/dexon-foundation/dexon/core/state"
+	"github.com/dexon-foundation/dexon/core/types"
+	"github.com/dexon-foundation/dexon/event"
+	"github.com/dexon-foundation/dexon/rlp"
+)
+
+// ReadOnlyBlockChain defines safe reading blockchain interface by removing write
+// methods of core.BlockChain struct.
+type ReadOnlyBlockChain interface {
+	BadBlocks() []*types.Block
+	CurrentBlock() *types.Block
+	CurrentFastBlock() *types.Block
+	CurrentHeader() *types.Header
+	GasLimit() uint64
+	Genesis() *types.Block
+	GetAncestor(common.Hash, uint64, uint64, *uint64) (common.Hash, uint64)
+	GetAddressInfo(uint32, common.Address) (
+		info struct {
+			Nonce   uint64
+			Cost    *big.Int
+			Counter uint64
+		})
+	GetBlock(common.Hash, uint64) *types.Block
+	GetBlockByHash(common.Hash) *types.Block
+	GetBlockByNumber(uint64) *types.Block
+	GetBlockHashesFromHash(common.Hash, uint64) []common.Hash
+	GetBlocksFromHash(common.Hash, int) (blocks []*types.Block)
+	GetBody(common.Hash) *types.Body
+	GetBodyRLP(common.Hash) rlp.RawValue
+	GetChainLastConfirmedHeight(uint32) uint64
+	GetConfirmedBlockByHash(uint32, coreCommon.Hash) (*coreTypes.Block, types.Transactions)
+	GetCostInConfirmedBlocks(uint32, common.Address) (*big.Int, bool)
+	GetGovStateByHash(common.Hash) (*types.GovState, error)
+	GetGovStateByNumber(uint64) (*types.GovState, error)
+	GetHeader(common.Hash, uint64) *types.Header
+	GetHeaderByHash(common.Hash) *types.Header
+	GetHeaderByNumber(number uint64) *types.Header
+	GetLastNonceInConfirmedBlocks(uint32, common.Address) (uint64, bool)
+	GetPending() (*types.Block, *state.StateDB)
+	GetPendingBlockByNumber(uint64) *types.Block
+	GetPendingHeight() uint64
+	GetReceiptsByHash(common.Hash) types.Receipts
+	GetRoundHeight(uint64) (uint64, bool)
+	GetTd(common.Hash, uint64) *big.Int
+	GetTdByHash(common.Hash) *big.Int
+	GetUnclesInChain(*types.Block, int) []*types.Header
+	HasBlock(common.Hash, uint64) bool
+	HasBlockAndState(common.Hash, uint64) bool
+	HasHeader(common.Hash, uint64) bool
+	HasState(common.Hash) bool
+	PendingBlock() *types.Block
+	State() (*state.StateDB, error)
+	StateAt(root common.Hash) (*state.StateDB, error)
+	SubscribeBlockConfirmedEvent(chan<- core.BlockConfirmedEvent) event.Subscription
+	SubscribeChainEvent(chan<- core.ChainEvent) event.Subscription
+	SubscribeChainHeadEvent(chan<- core.ChainHeadEvent) event.Subscription
+	SubscribeChainSideEvent(chan<- core.ChainSideEvent) event.Subscription
+	SubscribeLogsEvent(chan<- []*types.Log) event.Subscription
+	SubscribeRemovedLogsEvent(chan<- core.RemovedLogsEvent) event.Subscription
+}
+
+// access protection
+type ro interface {
+	ReadOnlyBlockChain
+}
+
+// ROBlockChain struct for safe read.
+type ROBlockChain struct {
+	ro
+}
+
+// NewROBlockChain converts original block chain to readonly interface.
+func NewROBlockChain(bc *core.BlockChain) ReadOnlyBlockChain {
+	return &ROBlockChain{ro: bc}
+}
diff --git a/indexer/config.go b/indexer/config.go
new file mode 100644
index 000000000..3b626272a
--- /dev/null
+++ b/indexer/config.go
@@ -0,0 +1,35 @@
+package indexer
+
+import (
+	"plugin"
+)
+
+// Config is data sources related configs struct.
+type Config struct {
+	// Used by dex/backend init flow.
+	Enable bool
+
+	// Plugin path for building components.
+	Plugin string
+}
+
+// NewIndexerFromConfig initialize exporter according to given config.
+func NewIndexerFromConfig(bc ReadOnlyBlockChain, c Config) (idx Indexer) {
+	if c.Plugin == "" {
+		// default
+		return
+	}
+
+	plug, err := plugin.Open(c.Plugin)
+	if err != nil {
+		panic(err)
+	}
+
+	symbol, err := plug.Lookup(NewIndexerFuncName)
+	if err != nil {
+		panic(err)
+	}
+
+	idx = symbol.(NewIndexerFunc)(bc, c)
+	return
+}
diff --git a/indexer/interfaces.go b/indexer/interfaces.go
new file mode 100644
index 000000000..f35bc6547
--- /dev/null
+++ b/indexer/interfaces.go
@@ -0,0 +1,19 @@
+package indexer
+
+// NewIndexerFuncName plugin looks up name.
+var NewIndexerFuncName = "NewIndexer"
+
+// NewIndexerFunc init function alias.
+type NewIndexerFunc = func(ReadOnlyBlockChain, Config) Indexer
+
+// Indexer defines indexer daemon interface. The daemon would hold a
+// core.Blockhain, passed by initialization function, to receiving latest block
+// event or other information query and interaction.
+type Indexer interface {
+	// Start is called by dex.Dexon if config is set.
+	Start() error
+
+	// Stop is called by dex.Dexon if config is set and procedure is
+	// terminating.
+	Stop() error
+}