araddon · araddon · Mar 8, 2016 · Feb 29, 2016 · Mar 1, 2016 · Mar 2, 2016
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,13 @@
 
+
+## v 0.13  March 2016
+
+* Better internal Schema Query planning, system (SHOW, DESCRIBE) with schemadb https://github.com/araddon/qlbridge/pull/68
+  * introspect csv files for types
+  * convert `SHOW`, `DESCRIBE` insto `SELECT` statements
+  * better internal data-source registry
+
+
 ## v 0.12  February 2016
 
 * Enable Distributed runtime by `Executor` interface https://github.com/araddon/qlbridge/pull/66

diff --git a/GLOCKFILE b/GLOCKFILE
@@ -0,0 +1,18 @@
+github.com/araddon/dateparse 9a7831edb0454bdfb5009a08674518d6c625c80c
+github.com/araddon/gou c4d51044125da629d6f2aa594f25b45c451a7d2a
+github.com/bmizerany/assert e17e99893cb6509f428e1728281c2ad60a6b31e3
+github.com/dataux/dataux b155b4c8281e4bd1918e40f893524b33bca28995
+github.com/dchest/siphash 6d8617816bb5d8268011ffbfb8720f17ce9af63c
+github.com/go-sql-driver/mysql 0f2db9e6c9cff80a97ca5c2c5096242cc1554e16
+github.com/gogo/protobuf ff05bbbb0ff143cc11fc3f8b700fc3a2864b884d
+github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032
+github.com/google/btree cc6329d4279e3f025a53a83c397d2339b5705c45
+github.com/kr/pretty e6ac2fc51e89a3249e82157fa0bb7a18ef9dd5bb
+github.com/kr/text bb797dc4fb8320488f47bf11de07a733d7233e1f
+github.com/leekchan/timeutil 28917288c48df3d2c1cfe468c273e0b2adda0aa5
+github.com/lytics/datemath 988020f3ad34814005ab10b6c7863e31672b5f63
+github.com/mb0/glob 1eb79d2de6c448664e7272f8b9fe1938239e3aaa
+github.com/pborman/uuid c55201b036063326c5b1b89ccfe45a184973d073
+github.com/surge/sqlparser 6b860f881ddbb9373d7173bdfa1f052ec3e6b215
+github.com/zhenjl/sqlparser 6b860f881ddbb9373d7173bdfa1f052ec3e6b215
+golang.org/x/net 6acef71eb69611914f7a30939ea9f6e194c78172
diff --git a/datasource/context_test.go b/datasource/context_test.go
@@ -1,11 +1,12 @@
-package datasource
+package datasource_test
 
 import (
 	"testing"
 	"time"
 
 	"github.com/bmizerany/assert"
 
+	"github.com/araddon/qlbridge/datasource"
 	"github.com/araddon/qlbridge/expr"
 	"github.com/araddon/qlbridge/value"
 )
@@ -17,21 +18,21 @@ func TestNested(t *testing.T) {
 	c1 := value.NewStringValue("c1")
 	d1 := value.NewStringValue("d1")
 	readers := []expr.ContextReader{
-		NewContextSimpleData(map[string]value.Value{
+		datasource.NewContextSimpleData(map[string]value.Value{
 			"a": a1,
 			"b": b1,
 		}),
-		NewContextSimpleData(map[string]value.Value{
+		datasource.NewContextSimpleData(map[string]value.Value{
 			"b": value.NewStringValue("b2"),
 			"c": c1,
 		}),
-		NewContextSimpleData(map[string]value.Value{
+		datasource.NewContextSimpleData(map[string]value.Value{
 			"c": value.NewStringValue("b2"),
 			"d": d1,
 		}),
 	}
 
-	nc := NewNestedContextReader(readers, time.Now())
+	nc := datasource.NewNestedContextReader(readers, time.Now())
 	expected := map[string]value.Value{
 		"a": a1,
 		"b": b1,
@@ -60,21 +61,21 @@ func TestNamespaces(t *testing.T) {
 	c1 := value.NewStringValue("c1")
 	d1 := value.NewStringValue("d1")
 	readers := []expr.ContextReader{
-		NewNamespacedContextReader(NewContextSimpleData(map[string]value.Value{
+		datasource.NewNamespacedContextReader(datasource.NewContextSimpleData(map[string]value.Value{
 			"a": a1,
 			"b": b1,
 			"d": d1,
 		}), "foo"),
-		NewNamespacedContextReader(NewContextSimpleData(map[string]value.Value{
+		datasource.NewNamespacedContextReader(datasource.NewContextSimpleData(map[string]value.Value{
 			"b": b2,
 			"c": c1,
 		}), "BAR"),
-		NewContextSimpleData(map[string]value.Value{
+		datasource.NewContextSimpleData(map[string]value.Value{
 			"a": a1,
 		}),
 	}
 
-	nc := NewNestedContextReader(readers, time.Now())
+	nc := datasource.NewNestedContextReader(readers, time.Now())
 	expected := map[string]value.Value{
 		"foo.a": a1,
 		"foo.b": b1,

diff --git a/datasource/context_wrapper_test.go b/datasource/context_wrapper_test.go
@@ -1,4 +1,4 @@
-package datasource
+package datasource_test
 
 import (
 	"encoding/json"
@@ -7,8 +7,8 @@ import (
 
 	"github.com/araddon/dateparse"
 	u "github.com/araddon/gou"
-	//"github.com/bmizerany/assert"
 
+	"github.com/araddon/qlbridge/datasource"
 	"github.com/araddon/qlbridge/expr"
 	"github.com/araddon/qlbridge/value"
 )
@@ -49,16 +49,16 @@ func TestStructWrapper(t *testing.T) {
 	}
 
 	readers := []expr.ContextReader{
-		NewContextWrapper(user),
-		NewContextSimpleNative(map[string]interface{}{
+		datasource.NewContextWrapper(user),
+		datasource.NewContextSimpleNative(map[string]interface{}{
 			"str1": "str1",
 			"int1": 1,
 			"t1":   t1,
 			"Name": "notyoda",
 		}),
 	}
 
-	nc := NewNestedContextReader(readers, time.Now())
+	nc := datasource.NewNestedContextReader(readers, time.Now())
 	expected := value.NewMapValue(map[string]interface{}{
 		"str1":          "str1",
 		"int1":          1,

diff --git a/datasource/csv_test.go b/datasource/csv_test.go
@@ -1,56 +1,44 @@
-package datasource
+package datasource_test
 
 import (
-	"flag"
 	"fmt"
 	"strings"
 	"testing"
 
 	u "github.com/araddon/gou"
 	"github.com/bmizerany/assert"
 
+	"github.com/araddon/qlbridge/datasource"
 	"github.com/araddon/qlbridge/schema"
+	"github.com/araddon/qlbridge/testutil"
 )
 
 var (
-	VerboseTests *bool = flag.Bool("vv", false, "Verbose Logging?")
-	_                  = u.EMPTY
-
+	_        = u.EMPTY
 	testData = map[string]string{
 		"user.csv": `user_id,email,interests,reg_date,item_count
 9Ip1aKbeZe2njCDM,"[email protected]","fishing","2012-10-17T17:29:39.738Z",82
 hT2impsOPUREcVPc,"[email protected]","swimming","2009-12-11T19:53:31.547Z",12
 hT2impsabc345c,"not_an_email","swimming","2009-12-11T19:53:31.547Z",12`,
 	}
 
-	csvSource       schema.DataSource = &CsvDataSource{}
+	csvSource       schema.DataSource = &datasource.CsvDataSource{}
 	csvStringSource schema.DataSource = &csvStaticSource{testData: testData}
 )
 
 func init() {
-	flag.Parse()
-	if *VerboseTests {
-		u.SetupLogging("debug")
-	} else {
-		u.SetupLogging("info")
-	}
-
-	u.SetColorOutput()
-
-	// Register our Datasources in registry
-	Register("csv", csvSource)
-	Register("csvtest", csvStringSource)
+	testutil.Setup()
 }
 
 type csvStaticSource struct {
-	*CsvDataSource
+	*datasource.CsvDataSource
 	testData map[string]string
 }
 
 func (m *csvStaticSource) Open(connInfo string) (schema.SourceConn, error) {
 	if data, ok := m.testData[connInfo]; ok {
 		sr := strings.NewReader(data)
-		return NewCsvSource(connInfo, 0, sr, make(<-chan bool, 1))
+		return datasource.NewCsvSource(connInfo, 0, sr, make(<-chan bool, 1))
 	}
 	return nil, fmt.Errorf("not found")
 }

diff --git a/datasource/datasource.go b/datasource/datasource.go
diff --git a/datasource/datatypes.go b/datasource/datatypes.go
@@ -24,10 +24,10 @@ type (
 	// Convert json to array of strings
 	StringArray []string
 
-	// convert json bytes
+	// json data
 	JsonWrapper json.RawMessage
 
-	// json
+	// json Helper expects map[string]interface
 	JsonHelperScannable u.JsonHelper
 )
 
@@ -96,7 +96,7 @@ func (m *TimeValue) Scan(src interface{}) error {
 }
 
 func (m *TimeValue) Unmarshal(v interface{}) error {
-	u.Warnf("wat? %T %v", v, v)
+	u.Warnf("time value Unmarshall not implemented? %T %v", v, v)
 	//return json.Unmarshal([]byte(*m), v)
 	return fmt.Errorf("not implemented")
 }

diff --git a/datasource/introspect.go b/datasource/introspect.go
@@ -0,0 +1,87 @@
+package datasource
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/araddon/dateparse"
+	u "github.com/araddon/gou"
+
+	"github.com/araddon/qlbridge/schema"
+	"github.com/araddon/qlbridge/value"
+)
+
+var (
+	_               = u.EMPTY
+	IntrospectCount = 20
+)
+
+func IntrospectSchema(s *schema.Schema, name string, iter schema.Iterator) error {
+
+	tbl, err := s.Table(name)
+	if err != nil {
+		u.Errorf("Could not find table %q", name)
+		return err
+	}
+	nameIndex := make(map[int]string, len(tbl.Columns()))
+	for i, colName := range tbl.Columns() {
+		nameIndex[i] = colName
+	}
+	//u.Infof("s:%s INTROSPECT SCHEMA name %q", s, name)
+	ct := 0
+	for {
+		msg := iter.Next()
+		//u.Debugf("msg %#v", msg)
+		if msg == nil || ct > IntrospectCount {
+			break
+		}
+		switch mt := msg.Body().(type) {
+		case *SqlDriverMessageMap:
+			for i, v := range mt.Vals {
+
+				k := nameIndex[i]
+				_, exists := tbl.FieldMap[k]
+
+				//u.Debugf("i:%v k:%s  v: %T %v", i, k, v, v)
+				switch val := v.(type) {
+				case int, int64, int16, int32, uint16, uint64, uint32:
+					tbl.AddFieldType(k, value.IntType)
+				case time.Time, *time.Time:
+					tbl.AddFieldType(k, value.TimeType)
+				case bool:
+					tbl.AddFieldType(k, value.BoolType)
+				case float32, float64:
+					tbl.AddFieldType(k, value.NumberType)
+				case string:
+					valType := guessValueType(val)
+					if !exists {
+						tbl.AddFieldType(k, valType)
+						//fld := tbl.FieldMap[k]
+						//u.Debugf("add field? %+v", fld)
+						//u.Debugf("%s = %v   type: %T   vt:%s new? %v", k, val, val, valType, !exists)
+					}
+				default:
+					u.Warnf("not implemented: %T", val)
+				}
+			}
+		default:
+			u.Warnf("not implemented: %T", mt)
+		}
+
+		ct++
+	}
+	return nil
+}
+
+func guessValueType(val string) value.ValueType {
+	if _, err := strconv.ParseInt(val, 10, 64); err == nil {
+		return value.IntType
+	} else if _, err := strconv.ParseBool(val); err == nil {
+		return value.IntType
+	} else if _, err := strconv.ParseFloat(val, 64); err == nil {
+		return value.NumberType
+	} else if _, err := dateparse.ParseAny(val); err == nil {
+		return value.TimeType
+	}
+	return value.StringType
+}
diff --git a/datasource/introspect_test.go b/datasource/introspect_test.go
@@ -0,0 +1,45 @@
+package datasource_test
+
+import (
+	"testing"
+
+	u "github.com/araddon/gou"
+	"github.com/bmizerany/assert"
+
+	"github.com/araddon/qlbridge/datasource"
+	td "github.com/araddon/qlbridge/datasource/mockcsvtestdata"
+	"github.com/araddon/qlbridge/schema"
+	"github.com/araddon/qlbridge/testutil"
+	"github.com/araddon/qlbridge/value"
+)
+
+var _ = u.EMPTY
+
+func init() {
+	testutil.Setup()
+}
+
+func TestIntrospectedCsvSchema(t *testing.T) {
+	sch := td.MockSchema
+
+	tableName := "users"
+	csvSrc, err := sch.Open(tableName)
+	assert.Tf(t, err == nil, "should not have error: %v", err)
+	scanner, ok := csvSrc.(schema.Scanner)
+	assert.T(t, ok)
+
+	iter := scanner.CreateIterator(nil)
+
+	err = datasource.IntrospectSchema(sch, tableName, iter)
+	assert.Tf(t, err == nil, "should not have error: %v", err)
+	tbl, err := sch.Table("users")
+	assert.Tf(t, err == nil, "should not have error: %v", err)
+	assert.Tf(t, tbl.Name == "users", "wanted users got %s", tbl.Name)
+	assert.Tf(t, len(tbl.Fields) == 5, "want 5 cols got %v", len(tbl.Fields))
+
+	refCt := tbl.FieldMap["referral_count"]
+	assert.Tf(t, refCt.Type == value.IntType, "wanted int got %s", refCt.Type)
+
+	userId := tbl.FieldMap["user_id"]
+	assert.Tf(t, userId.Type == value.StringType, "wanted string got %s", userId.Type)
+}