core: vm: sqlvm: process non-UTF-8 input and escape sequences

Our parser is able to process queries with invalid UTF-8, provided that
it is compatible with ASCII. Since doing so requires encoding the input
before passing to pigeon, Parse* functions generated by pigeon are
unexported because they should not be used directly.

Escape sequences in string literals and identifiers are now recognized.
In addition to escape sequences supported by solidity, we support \U
similar to the one supported by Go to allow users to specify non-BMP
Unicode code point without using multiple \x escapes.

AST printer is modified to quote non-printable characters in strings to
prevent control characters from messing up the terminal.

1 files changed, 10 insertions, 1 deletions

diff --git a/core/vm/sqlvm/parser/parser_test.go b/core/vm/sqlvm/parser/parser_test.go
index a81b1d22d..77c3c16ff 100644
--- a/core/vm/sqlvm/parser/parser_test.go
+++ b/core/vm/sqlvm/parser/parser_test.go
@@ -4,12 +4,13 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/suite"
+	"golang.org/x/text/encoding/traditionalchinese"
 )
 
 type ParserTestSuite struct{ suite.Suite }
 
 func (s *ParserTestSuite) requireParseNoError(sql string) {
-	_, err := ParseString(sql)
+	_, err := Parse([]byte(sql))
 	s.Require().NoError(err)
 }
 
@@ -75,6 +76,14 @@ func (s *ParserTestSuite) TestParse() {
 	// Test create index.
 	s.requireParseNoError(`create unique index a on a (a)`)
 	s.requireParseNoError(`create index "~!@#$%^&*()" on ㄅ ( a , b )`)
+	s.requireParseNoError(`create index ㄅㄆㄇ on 👍 ( 🌍 , 💯 )`)
+}
+
+func (s *ParserTestSuite) TestParseInvalidUTF8() {
+	query := `SELECT ㄅ FROM 東 WHERE — - ─ = ██`
+	query, err := traditionalchinese.Big5.NewEncoder().String(query)
+	s.Require().NoError(err)
+	s.requireParseNoError(query)
 }
 
 func TestParser(t *testing.T) {