Update glock dependencies, config fixes

dataux · Jul 2, 2016 · c24f0eb · c24f0eb
1 parent 74c289f
commit c24f0eb
Show file tree

Hide file tree

Showing 9 changed files with 70 additions and 65 deletions.
diff --git a/GLOCKFILE b/GLOCKFILE
@@ -1,41 +1,41 @@
-github.com/araddon/dateparse 2f41da3eb28d713ca2f45682fae66173fad5e85b
-github.com/araddon/gou c4d51044125da629d6f2aa594f25b45c451a7d2a
-github.com/araddon/qlbridge 96c3da4e0c9996c06bdcfc56118c808c10d5f8e6
+github.com/araddon/dateparse a19b713c2e31cec89903deb95b4d053e9e7b4db5
+github.com/araddon/gou 50a94aa4a3fb69e8fbde05df290fcb49fa685e07
+github.com/araddon/qlbridge b890cc9b548b80c97574b870ac8507ee2c7a4db9
 github.com/bitly/go-hostpool d0e59c22a56e8dadfed24f74f452cea5a52722d2
-github.com/bmizerany/assert e17e99893cb6509f428e1728281c2ad60a6b31e3
+github.com/bmizerany/assert b7ed37b82869576c289d7d97fb2bbd8b64a0cb28
 github.com/coreos/etcd 207c92b62785d220edc74ed791fd8a22a18e378e
 github.com/coreos/go-etcd de3514f25635bbfb024fdaf2a8d5f67378492675
 github.com/dchest/siphash 6d8617816bb5d8268011ffbfb8720f17ce9af63c
-github.com/go-sql-driver/mysql 7ebe0a500653eeb1859664bed5e48dec1e164e73
-github.com/gogo/protobuf 4f262e4b0f3a6cea646e15798109335551e21756
-github.com/golang/protobuf f0a097ddac24fb00e07d2ac17f8671423f3ea47c
-github.com/google/btree f06e229e679911bb31a04e07ac891115822e37c3
-github.com/hashicorp/go-immutable-radix 8e8ed81f8f0bf1bdd829593fdd5c29922c1ea990
+github.com/go-sql-driver/mysql 3654d25ec346ee8ce71a68431025458d52a38ac0
+github.com/gogo/protobuf 2752d97bbd91927dd1c43296dbf8700e50e2708c
+github.com/golang/protobuf 3852dcfda249c2097355a6aabb199a28d97b30df
+github.com/google/btree 7d79101e329e5a3adf994758c578dab82b90c017
+github.com/hashicorp/go-immutable-radix afc5a0dbb18abdf82c277a7bc01533e81fa1d6b8
 github.com/hashicorp/go-memdb 98f52f52d7a476958fa9da671354d270c50661a7
 github.com/hashicorp/golang-lru a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4
-github.com/jmoiron/sqlx 398dd5876282499cdfd4cb8ea0f31a672abe9495
+github.com/jmoiron/sqlx bdae0c3219c3bdb92f3d7c70825018712787c933
 github.com/kr/pretty add1dbc86daf0f983cd4a48ceb39deb95c729b67
-github.com/kr/text bb797dc4fb8320488f47bf11de07a733d7233e1f
+github.com/kr/text 7cafcd837844e784b526369c9bce262804aebc60
 github.com/leekchan/timeutil 28917288c48df3d2c1cfe468c273e0b2adda0aa5
-github.com/lytics/cloudstorage 7c7173488d69e97c35140b22d565298f40bb2a0e
+github.com/lytics/cloudstorage 08dcbf0c85da78d1bd02f6d548b83e1ca4763671
 github.com/lytics/confl f12b6bace2ff295fc6730f6dbaf009289e2f6e80
 github.com/lytics/datemath 988020f3ad34814005ab10b6c7863e31672b5f63
 github.com/lytics/dfa 63e35f788f7fa5203fcd2dcd7e318da5a1b981e5
-github.com/lytics/grid 8e444f9e1d9c7366fccfb99c44f398fa1b9c5689
+github.com/lytics/grid c3dbc9fec6398e4c6ce2f451709705774574e159
 github.com/lytics/metafora eb26f00432a98228e44de8e074b9627fc4c1e457
 github.com/lytics/sereno f6128b640de0742892a439b3672ac88637df977d
-github.com/mattbaird/elastigo d0839da905fef1221e2b19ddfaf34985add62e10
+github.com/mattbaird/elastigo 34c4c4d8425cbdcbc8e257943a2044d5e9f7dab5
 github.com/mb0/glob 1eb79d2de6c448664e7272f8b9fe1938239e3aaa
 github.com/nats-io/gnatsd 8e99354b8ca40ae921463cea076abf460dd12d62
-github.com/nats-io/nats b258f90beb7c1e3a45037b4f9b5bdadcde9cc42d
+github.com/nats-io/nats d498de06529cce1536449f220f54217a10c319e5
 github.com/nats-io/nuid 4f84f5f3b2786224e336af2e13dba0a0a80b76fa
 github.com/pborman/uuid c55201b036063326c5b1b89ccfe45a184973d073
-github.com/sony/sonyflake 152b0c313040e9fb0101fcff9285a11965fd42be
+github.com/sony/sonyflake fa881fb1052b152e977c41023052c2f2a1c475e9
 github.com/ugorji/go 45ce7596ace4534e47b69051a92aef7b64ec7b3f
 golang.org/x/crypto 1f22c0103821b9390939b6776727195525381532
-golang.org/x/net fb93926129b8ec0056f2f458b1f519654814edf0
-golang.org/x/oauth2 b0e2337fe6ec0c637fa4f123268b972f334504eb
-google.golang.org/api 9737cc9e103c00d06a8f3993361dec083df3d252
-google.golang.org/cloud 79ffda073da804f325135da5ff645a630a4d7625
-google.golang.org/grpc 7834b974e55fbf85a5b01afb5821391c71084efd
+golang.org/x/net b400c2eff1badec7022a8c8f5bea058b6315eed7
+golang.org/x/oauth2 df5b72659a3b1789a345ea643bb7c28442681652
+google.golang.org/api 57212573ccf9e4f69f4caa0336831697e29adcd6
+google.golang.org/cloud ebfe3819c9ab4acaf086ae57e4cbd7dee683baf0
+google.golang.org/grpc a0ff1e78a98cd140c2d18a0210ab0a2a183a2b4c
 gopkg.in/mgo.v2 22287bab4379e1fbf6002fb4eb769888f3fb224c
diff --git a/backends/elasticsearch/es_datasource.go b/backends/elasticsearch/es_datasource.go
@@ -17,12 +17,13 @@ var (
 )
 
 const (
-	ListenerType = "elasticsearch"
+	// SourceType defines the "source" type from qlbridge datasource
+	SourceType = "elasticsearch"
 )
 
 func init() {
 	// We need to register our DataSource provider here
-	datasource.Register(ListenerType, &ElasticsearchDataSource{})
+	datasource.Register(SourceType, &ElasticsearchDataSource{})
 }
 
 type ElasticsearchDataSource struct {

diff --git a/backends/elasticsearch/es_test.go b/backends/elasticsearch/es_test.go
@@ -38,7 +38,8 @@ func RunTestServer(t *testing.T) func() {
 		planner.GridConf.SchemaLoader = testmysql.SchemaLoader
 		planner.GridConf.SupressRecover = testmysql.Conf.SupressRecover
 		testmysql.RunTestServer(t)
-		planner.RunWorkerNodes(2, testmysql.ServerCtx.Reg)
+		quit := make(chan bool)
+		planner.RunWorkerNodes(quit, 2, testmysql.ServerCtx.Reg)
 	}
 	return func() {
 		// placeholder

diff --git a/backends/elasticsearch/sqltoes.go b/backends/elasticsearch/sqltoes.go
@@ -66,7 +66,12 @@ func (m *SqlToEs) Host() string {
 	return chooseBackend(m.schema)
 }
 func chooseBackend(schema *schema.SchemaSource) string {
-	for _, node := range schema.Nodes {
+	if len(schema.Conf.Nodes) == 0 {
+		if len(schema.Conf.Hosts) > 0 {
+			return schema.Conf.Hosts[0]
+		}
+	}
+	for _, node := range schema.Conf.Nodes {
 		// TODO:  implement real balancer
 		return node.Address
 	}

diff --git a/backends/mongo/mgo_source.go b/backends/mongo/mgo_source.go
@@ -145,10 +145,10 @@ func (m *MongoDataSource) loadSchema() error {
 	return nil
 }
 
-// TODO:  this is horrible, use mgo's built in gossip with mongo for cluster info
+// TODO:  this is horrible, should use mgo's built in gossip with mongo for cluster info
 func chooseBackend(source string, schema *schema.SchemaSource) string {
 	//u.Infof("check backends: %v", len(schema.Nodes))
-	for _, node := range schema.Nodes {
+	for _, node := range schema.Conf.Nodes {
 		//u.Debugf("check node:%q =? %+v", source, node)
 		if node.Source == source {
 			//u.Debugf("found node: %+v", node)

diff --git a/dataux.conf b/dataux.conf
@@ -54,11 +54,13 @@ sources : [
   {
     name : mgo_datauxtest
     type : mongo
+    hosts : ["localhost"]
   }
 
   {
-    name : es_test
-    type : elasticsearch
+    name  : es_test
+    type  : elasticsearch
+    hosts : ["http://localhost:9200"]
   }
 
   {
@@ -98,18 +100,4 @@ sources : [
       jwt       : "/home/aaron/google-datastore/googlejwt.json"
     }
   },
-]
-
-# List of nodes hosting data sources
-nodes : [
-  {
-    name    : estest1
-    source  : es_test
-    address : "http://localhost:9200"
-  },
-  {
-    name    : mgotest1
-    source  : mgo_datauxtest
-    address : "localhost"
-  }
 ]
diff --git a/models/config.go b/models/config.go
@@ -3,17 +3,11 @@ package models
 import (
 	"io/ioutil"
 
-	u "github.com/araddon/gou"
-	"github.com/lytics/confl"
-
 	"github.com/araddon/qlbridge/schema"
+	"github.com/lytics/confl"
 )
 
-var (
-	_ = u.EMPTY
-)
-
-// Read a Confl configured file
+// LoadConfigFromFile Read a Confl formatted config file from disk
 func LoadConfigFromFile(filename string) (*Config, error) {
 	var c Config
 	confBytes, err := ioutil.ReadFile(filename)
@@ -27,6 +21,8 @@ func LoadConfigFromFile(filename string) (*Config, error) {
 	return &c, nil
 }
 
+// LoadConfig load a confl formatted file from string (assumes came)
+//  from file or passed in
 func LoadConfig(conf string) (*Config, error) {
 	var c Config
 	if _, err := confl.Decode(conf, &c); err != nil {
@@ -35,10 +31,10 @@ func LoadConfig(conf string) (*Config, error) {
 	return &c, nil
 }
 
-// Overall DataUX Server config made up of blocks
+// Config for DataUX Server config made up of blocks
 //   1) Frontend Listeners (protocols)
 //   2) Sources (types of backends such as elasticsearch, mysql, mongo, ...)
-//   3) Virtual Schemas
+//   3) Schemas:  n number of sources can create a "Virtual Schema"
 //   4) list of server/nodes for Sources
 //   5) nats,etcd coordinators
 type Config struct {
@@ -50,20 +46,20 @@ type Config struct {
 	Sources        []*schema.ConfigSource `json:"sources"`         // backend servers/sources (es, mysql etc)
 	Schemas        []*schema.ConfigSchema `json:"schemas"`         // Schemas, each backend has 1 schema
 	Nodes          []*schema.ConfigNode   `json:"nodes"`           // list of nodes that host sources
-	Rules          *RulesConfig
+	Rules          *RulesConfig           `json:"rules"`           // rules for routing
 }
 
-// Frontend Listener to listen for inbound traffic on
-// specific protocola or transport
+// ListenerConfig Frontend Listener to listen for inbound
+// traffic on specific protocol aka transport
 type ListenerConfig struct {
-	Type     string `json:"type"`     // [mysql,mongo,mc,postgres,etc]
+	Type     string `json:"type"`     // named protocol type [mysql,mongo,mc,postgres,etc]
 	Addr     string `json:"address"`  // net.Conn compatible ip/dns address
 	User     string `json:"user"`     // user to talk to backend with
 	Password string `json:"password"` // optional pwd for backend
 }
 
 type RulesConfig struct {
-	Schema    string        `json:"default"`
+	Schema    string        `json:"schema"`
 	Default   string        `json:"default"`
 	ShardRule []ShardConfig `json:"shard"`
 }

diff --git a/models/serverctx.go b/models/serverctx.go
@@ -12,13 +12,13 @@ import (
 	"github.com/dataux/dataux/planner"
 )
 
-// Server Context for the DataUX Server giving access to the shared
+// ServerCtx Server Context for the DataUX Server giving access to the shared
 //  memory objects Config, Schemas, Grid runtime
 type ServerCtx struct {
 	// The dataux server config info on schema, backends, frontends, etc
 	Config *Config
 	// The underlying qlbridge schema holds info about the
-	//  available datasource Drivers/Adapters
+	//  available datasource's
 	Reg *datasource.Registry
 	// Grid is our real-time multi-node coordination and messaging system
 	Grid *planner.Server
@@ -34,13 +34,16 @@ func NewServerCtx(conf *Config) *ServerCtx {
 	return &svr
 }
 
-// Load all the config info for this context and start the grid servers
+// Init Load all the config info for this server and start the
+// grid/messaging/coordination systems
 func (m *ServerCtx) Init() error {
 
 	if err := m.loadConfig(); err != nil {
 		return err
 	}
 
+	// Copy over the nats, etcd info from config to
+	// Planner grid
 	planner.GridConf.NatsServers = m.Config.Nats
 	planner.GridConf.EtcdServers = m.Config.Etcd
 
@@ -108,11 +111,19 @@ func (m *ServerCtx) loadConfig() error {
 			ss.Schema = sch
 			//u.Infof("found sourceName: %q schema.Name=%q conf=%+v", sourceName, ss.Name, sourceConf)
 
-			for _, nc := range m.Config.Nodes {
-				if nc.Source == sourceConf.Name {
-					ss.Nodes = append(ss.Nodes, nc)
+			if len(m.Config.Nodes) == 0 {
+				for _, host := range sourceConf.Hosts {
+					nc := &schema.ConfigNode{Source: sourceName, Address: host}
+					//ss.Nodes = append(ss.Nodes, nc)
 					sourceConf.Nodes = append(sourceConf.Nodes, nc)
 				}
+			} else {
+				for _, nc := range m.Config.Nodes {
+					if nc.Source == sourceConf.Name {
+						//ss.Nodes = append(ss.Nodes, nc)
+						sourceConf.Nodes = append(sourceConf.Nodes, nc)
+					}
+				}
 			}
 
 			sch.AddSourceSchema(ss)

diff --git a/proxy/server.go b/proxy/server.go
@@ -26,14 +26,17 @@ var asciiIntro = `
 
 var Conf *models.Config
 
-func LoadConfig(configFile string) {
+// LoadConfig from @configFile (read from disk?)
+// also available is a
+func LoadConfig(configFile string) *models.Config {
 	// get config from file and exit if error
 	conf, err := models.LoadConfigFromFile(configFile)
 	if err != nil {
 		u.Errorf("Could not load config: %v", err)
 		os.Exit(1)
 	}
 	Conf = conf
+	return conf
 }
 func banner() string {
 	return strings.Replace(asciiIntro, "*", "`", -1)