Skip to content

Commit f3e9e4a

Browse files
araddonaraddon
committed
BigQuery test fixes
1 parent 33fe545 commit f3e9e4a

File tree

5 files changed

+46
-104
lines changed

5 files changed

+46
-104
lines changed

‎backends/bigquery/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,10 @@ SELECT landmark from bikeshare_stations WHERE landmark like "Palo%"
6464
select count(*) AS ct, landmark FROM bikeshare_stations GROUP BY landmark ORDER BY ct DESC LIMIT 1;
6565

6666

67+
# Drop it when your are done if you want
68+
69+
drop schema bq;
70+
6771
```
6872

6973

‎backends/bigquery/bq_test.go‎

Lines changed: 39 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -26,18 +26,17 @@ import (
2626
)
2727

2828
var (
29-
DbConn = "root@tcp(127.0.0.1:13307)/datauxtest?parseTime=true"
29+
dbConn = "root@tcp(127.0.0.1:13307)/datauxtest?parseTime=true"
3030
loadTestDataOnce sync.Once
3131
now = time.Now()
3232
testServicesRunning bool
3333
bqTable = "datauxtest"
3434
gceProject = os.Getenv("GCEPROJECT")
35-
_ = json.RawMessage(nil)
3635
)
3736

3837
func init() {
3938
if gceProject == "" {
40-
panic("Must have $GCEPROJECT env")
39+
panic("Must have $GCEPROJECT env there is no emulator")
4140
}
4241
tu.Setup()
4342
}
@@ -54,16 +53,31 @@ func RunTestServer(t *testing.T) func() {
5453
planner.GridConf.SchemaLoader = testmysql.SchemaLoader
5554
planner.GridConf.SupressRecover = testmysql.Conf.SupressRecover
5655

57-
var bqconf *schema.ConfigSource
58-
for _, sc := range testmysql.Conf.Sources {
59-
if sc.SourceType == "bigquery" {
60-
bqconf = sc
61-
}
62-
}
63-
if bqconf == nil {
64-
panic("must have bigquery conf")
65-
}
66-
bqconf.Settings["billing_project"] = gceProject
56+
reg := schema.DefaultRegistry()
57+
by := []byte(`{
58+
"name": "bigquery",
59+
"schema":"datauxtest",
60+
"type": "bigquery",
61+
"table_aliases" : {
62+
"bikeshare_stations" : "bigquery-public-data:san_francisco.bikeshare_stations"
63+
},
64+
"settings" : {
65+
"billing_project" : "",
66+
"data_project" : "bigquery-public-data",
67+
"dataset" : "san_francisco"
68+
}
69+
}`)
70+
71+
conf := &schema.ConfigSource{}
72+
err := json.Unmarshal(by, conf)
73+
assert.Equal(t, nil, err)
74+
conf.Settings["billing_project"] = gceProject
75+
err = reg.SchemaAddFromConfig(conf)
76+
assert.Equal(t, nil, err)
77+
78+
s, ok := reg.Schema("datauxtest")
79+
assert.Equal(t, true, ok)
80+
assert.NotEqual(t, nil, s)
6781

6882
testmysql.RunTestServer(t)
6983
}
@@ -104,7 +118,7 @@ func TestBasic(t *testing.T) {
104118
RunTestServer(t)
105119

106120
// This is a connection to RunTestServer, which starts on port 13307
107-
dbx, err := sqlx.Connect("mysql", DbConn)
121+
dbx, err := sqlx.Connect("mysql", dbConn)
108122
assert.True(t, err == nil, "%v", err)
109123
defer dbx.Close()
110124
//u.Debugf("%v", testSpec.Sql)
@@ -210,6 +224,7 @@ func TestSelectEscapeSyntax(t *testing.T) {
210224
}
211225

212226
func TestSelectGroupBy(t *testing.T) {
227+
RunTestServer(t)
213228
data := struct {
214229
Landmark string
215230
Ct int
@@ -218,27 +233,27 @@ func TestSelectGroupBy(t *testing.T) {
218233
Sql: "select count(*) as ct, landmark from bikeshare_stations GROUP BY landmark;",
219234
ExpectRowCt: 5,
220235
ValidateRowData: func() {
221-
//u.Infof("%v", data)
236+
u.Infof("%v", data)
222237
switch data.Landmark {
223238
case "San Jose":
224-
assert.Equal(t, 65, data.Ct, "Should have found 1? %v", data)
239+
assert.Equal(t, 18, data.Ct, "Should have found 18? %v", data)
225240
case "Palo Alto":
226-
assert.Equal(t, 20, data.Ct, "Should have found 2? %v", data)
241+
assert.Equal(t, 5, data.Ct, "Should have found 2? %v", data)
227242
}
228243
},
229244
RowData: &data,
230245
})
231246
}
232247

233248
func TestSelectWhereLike(t *testing.T) {
234-
249+
RunTestServer(t)
235250
// We are testing the LIKE clause
236251
data := struct {
237252
Landmark string
238253
}{}
239254
validateQuerySpec(t, tu.QuerySpec{
240255
Sql: `SELECT landmark from bikeshare_stations WHERE landmark like "Palo%"`,
241-
ExpectRowCt: 20,
256+
ExpectRowCt: 5,
242257
ValidateRowData: func() {
243258
assert.True(t, data.Landmark == "Palo Alto", "%v", data)
244259
},
@@ -259,7 +274,7 @@ func TestSelectOrderBy(t *testing.T) {
259274
ExpectRowCt: 1,
260275
ValidateRowData: func() {
261276
assert.Equal(t, "San Francisco", data.Landmark, "%v", data)
262-
assert.Equal(t, 142, data.Ct, "%v", data)
277+
assert.Equal(t, 37, data.Ct, "%v", data)
263278
},
264279
RowData: &data,
265280
})
@@ -270,7 +285,7 @@ func TestSelectOrderBy(t *testing.T) {
270285
ExpectRowCt: 1,
271286
ValidateRowData: func() {
272287
assert.Equal(t, "Mountain View", data.Landmark, "%v", data)
273-
assert.Equal(t, 28, data.Ct, "%v", data)
288+
assert.Equal(t, 7, data.Ct, "%v", data)
274289
},
275290
RowData: &data,
276291
})
@@ -413,10 +428,10 @@ func TestMutationUpdateSimple(t *testing.T) {
413428

414429
func TestInvalidQuery(t *testing.T) {
415430
RunTestServer(t)
416-
db, err := sql.Open("mysql", DbConn)
417-
assert.True(t, err == nil)
431+
db, err := sql.Open("mysql", dbConn)
432+
assert.Equal(t, nil, err)
418433
// It is parsing the SQL on server side (proxy) not in client
419-
// so hence that is what this is testing, making sure proxy responds gracefully
434+
// so hence that is what this is testing, making sure proxy responds gracefully
420435
rows, err := db.Query("select `stuff`, NOTAKEYWORD fake_tablename NOTWHERE `description` LIKE \"database\";")
421436
assert.True(t, err != nil, "%v", err)
422437
assert.True(t, rows == nil, "must not get rows")

‎backends/bigquery/resultreader.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ func (m *ResultReader) buildProjection() {
7474
cols = append(cols, rel.NewResultColumn(col.SourceField, len(cols), col, fld.Type))
7575
} else {
7676
u.Debugf("Could not find: '%v' in %#v", col.SourceField, m.Req.tbl.FieldMap)
77-
u.Warnf("%#v", col)
77+
//u.Warnf("%#v", col)
7878
}
7979
}
8080
}

‎frontends/mysqlfe/mysql_handler.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -250,8 +250,8 @@ func (m *mySqlHandler) handleQuery(writer models.ResultWriter, sql string) (err
250250
return err
251251
}
252252
return m.conn.WriteOK(nil)
253-
case *rel.SqlCreate:
254-
// DDL?
253+
case *rel.SqlCreate, *rel.SqlDrop, *rel.SqlAlter:
254+
// DDL statements
255255
err = job.Run()
256256
job.Close()
257257
if err != nil {

‎frontends/mysqlfe/testmysql/runtestserver.go‎

Lines changed: 0 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -89,83 +89,6 @@ frontends [
8989
]
9090
`
9191

92-
var oldConfig = `
93-
sources : [
94-
# csv-file "db" of data from http://seanlahman.com/baseball-archive/statistics/
95-
# must have TESTINT=true integration test flag turned on
96-
{
97-
name : baseball
98-
type : cloudstore
99-
settings : {
100-
type : gcs
101-
bucket : "lytics-dataux-tests"
102-
path : "baseball/"
103-
format : "csv"
104-
}
105-
}
106-
107-
{
108-
name : mysql_test
109-
type : mysql
110-
}
111-
112-
{
113-
name : kube
114-
type : kubernetes
115-
}
116-
117-
{
118-
name : bigquery
119-
type : bigquery
120-
# [bigquery-public-data:san_francisco.bikeshare_stations]
121-
"table_aliases" : {
122-
"bikeshare_stations" : "bigquery-public-data:san_francisco.bikeshare_stations"
123-
}
124-
"settings" : {
125-
# project will be loaded from ENV $GCEPROJECT
126-
"test_env" : "${USER}"
127-
"billing_project" : ""
128-
"data_project" : "bigquery-public-data"
129-
"dataset" : "san_francisco"
130-
}
131-
}
132-
133-
{
134-
name : lytics
135-
type : lytics
136-
settings {
137-
138-
}
139-
}
140-
141-
]
142-
143-
# List of nodes hosting data sources
144-
nodes : [
145-
{
146-
name : estest1
147-
source : es_test
148-
address : "http://localhost:9200"
149-
},
150-
{
151-
name : mgotest1
152-
source : mgo_datauxtest
153-
address : "localhost"
154-
},
155-
{
156-
name : csvlocal1
157-
source : csvlocal
158-
address : "$GOPATH/src/github.com/dataux/dataux/data"
159-
},
160-
{
161-
name : googleds1
162-
source : google_ds_test
163-
address : "$GOOGLEJWT"
164-
}
165-
]
166-
167-
`
168-
16992
// TestingT is an interface wrapper around *testing.T so when we import
17093
// this go dep, govendor don't import "testing"
17194
type TestingT interface {

0 commit comments

Comments
 (0)