Skip to content

Commit 16bd5ad

Browse files
author
Aaron Raddon
committed
Easier dataux get started, more default config
1 parent 6e4a39b commit 16bd5ad

File tree

13 files changed

+246
-107
lines changed

13 files changed

+246
-107
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11

22
aaron
33
dataux
4-
tools/importgithub/importgithub
54
archive
65
test.etcd
6+
tools/importgithub/importgithub
7+
backends/files/tables/

‎backends/kubernetes/README.md‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,10 @@ Provides SQL Access to Kubernetes Rest API's
66

77
![mysql_kube](https://cloud.githubusercontent.com/assets/7269/20697265/96e13c10-b5ac-11e6-944b-c588c6e7570e.png)
88

9-
* http://kubernetes.io/docs/user-guide/accessing-the-cluster/
10-
* https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx
11-
12-
**Types & Schema**
13-
* http://kubernetes.io/docs/api-reference/v1/definitions/
14-
159
**Try It Out**
1610

1711
* **Assumes minikube is running locally**
1812
* Install dataux https://github.com/dataux/dataux/releases
19-
* Install dataux via docker
2013

2114

2215
```sh
@@ -35,9 +28,26 @@ cd github.com/dataux/dataux
3528

3629
mysql -h 127.0.0.1 -P 4000 -Dkube
3730

31+
```
32+
33+
**SQL examples**
34+
35+
```sh
36+
37+
38+
mysql -h127.0.0.1 -P4000 -Dkube -e "describe pods;"
39+
mysql -h127.0.0.1 -P4000 -Dkube -e "describe nodes;"
40+
mysql -h127.0.0.1 -P4000 -Dkube -e "describe services;"
41+
42+
mysql -h127.0.0.1 -P4000 -Dkube -e "select name, creationtimestamp, hostip, podip, hostname from pods;"
43+
44+
mysql -h127.0.0.1 -P4000 -Dkube -e "select * from nodes;"
45+
3846

3947

4048
```
49+
50+
4151
Testing & Dev
4252
-------------------------------------
4353

@@ -48,6 +58,11 @@ or a google container engine cluster. Then we need to decide
4858
1. via kubectl proxy (easiest)
4959
2. via api
5060

61+
* http://kubernetes.io/docs/user-guide/accessing-the-cluster/
62+
* https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx
63+
64+
**Types & Schema**
65+
* http://kubernetes.io/docs/api-reference/v1/definitions/
5166

5267
**Setting up Minikube**
5368

@@ -73,24 +88,9 @@ kubectl get pods --all-namespaces
7388

7489
kubectl cluster-info
7590

91+
7692
minikube delete --v=10 --show-libmachine-logs --alsologtostderr
7793

7894
```
7995

8096

81-
**SQL examples**
82-
83-
```sh
84-
85-
86-
mysql -h127.0.0.1 -P4000 -Dkube -e "describe pods;"
87-
mysql -h127.0.0.1 -P4000 -Dkube -e "describe nodes;"
88-
mysql -h127.0.0.1 -P4000 -Dkube -e "describe services;"
89-
90-
mysql -h127.0.0.1 -P4000 -Dkube -e "select name, creationtimestamp, hostip, podip, hostname from pods;"
91-
92-
mysql -h127.0.0.1 -P4000 -Dkube -e "select * from nodes;"
93-
94-
95-
96-
```

‎backends/kubernetes/dataux.conf‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#
2+
# dataux configuration for kubernetes minikube local
3+
#
4+
# config format is nginx ish, it is quite lenient see https://github.com/lytics/confl
5+
# think of it as lenient json with support for comments
6+
7+
8+
# frontend is inbound tcp connection listener (only mysql currently supported)
9+
# we don't bind to 3306 in case a mysql server is running
10+
#
11+
# mysql -h127.0.0.1 -P4000 -Ddatauxtest
12+
#
13+
frontends : [
14+
{
15+
type : mysql
16+
address : "0.0.0.0:4000"
17+
}
18+
]
19+
20+
21+
# schemas: ie a virtual database made up of
22+
# combining tables from each source into a flattened table namespace
23+
schemas : [
24+
25+
{
26+
name : minikube
27+
sources : [ "minikube" ]
28+
}
29+
30+
]
31+
32+
# sources: Each source is a backend (files, cassandra, kubernetes, etc)
33+
sources : [
34+
{
35+
name : minikube
36+
type : kubernetes
37+
# since we have no connection info it is going
38+
# to assume kubectl .kube config info
39+
settings {
40+
namespace default
41+
hosts ["localhost"]
42+
}
43+
}
44+
]
45+
46+
47+
## Distributed Runtime Config Setup
48+
49+
50+
# etcd servers
51+
# etcd = ["http://localhost:2379"]
52+
53+
# Nats.io GnatsD servers
54+
# nats = [ "nats://localhost:4222" ]

‎backends/kubernetes/source.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ func (m *Source) DataSource() schema.Source { return m }
102102
func (m *Source) Tables() []string { return m.tables }
103103
func (m *Source) Table(table string) (*schema.Table, error) {
104104

105-
u.Debugf("Table(%q)", table)
105+
//u.Debugf("Table(%q)", table)
106106
if m.schema == nil {
107107
u.Warnf("no schema in use?")
108108
return nil, fmt.Errorf("no schema in use")

‎dataux.conf‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@
1414
# suppress recovery means panic, don't recover
1515
supress_recover: true
1616

17+
18+
## Distributed Runtime Config Setup
19+
1720
# etcd servers
1821
# [ "http://127.0.0.1:2379","http://127.0.0.1:2380"]
1922
etcd = ["http://localhost:2379"]
@@ -23,6 +26,8 @@ etcd = ["http://localhost:2379"]
2326
nats = [ "nats://localhost:4222" ]
2427

2528

29+
## Frontend
30+
2631
# frontend is inbound tcp connection listener (only mysql currently supported)
2732
# - we don't bind to 3306 because that is mysql's
2833
#

‎frontends/mysqlfe/mysql_handler.go‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import (
2121
)
2222

2323
func init() {
24-
// Load our Frontend Listener's
24+
// Register our Mysql Frontend Listener
2525
models.ListenerRegister(mysqlproxy.ListenerType, &MySqlConnCreator{})
2626
}
2727

@@ -49,11 +49,11 @@ type MySqlConnCreator struct {
4949
l models.Listener
5050
}
5151

52+
// Init is part of frontend interface to accept config and global server context at start
5253
func (m *MySqlConnCreator) Init(conf *models.ListenerConfig, svr *models.ServerCtx) error {
53-
//u.Debugf("mysql conn creator")
5454
l, err := mysqlproxy.ListenerInit(conf, svr.Config, m)
5555
if err != nil {
56-
u.Errorf("could not get mysql listener: %v", err)
56+
u.Errorf("could not init mysql listener: %v", err)
5757
return err
5858
}
5959
m.l = l
@@ -87,8 +87,11 @@ func (m *MySqlConnCreator) Close() error {
8787
return nil
8888
}
8989

90-
// MySql connection creator
91-
// - per connection, ie session specific
90+
func (m *MySqlConnCreator) String() string {
91+
return fmt.Sprintf("Mysql Frontend address:%v", m.conf.Addr)
92+
}
93+
94+
// MySql per connection, ie session specific
9295
type mySqlHandler struct {
9396
svr *models.ServerCtx
9497
sess expr.ContextReadWriter // session info
@@ -106,7 +109,7 @@ func (m *mySqlHandler) Close() error {
106109
return nil
107110
}
108111

109-
// Implement the Handle interface for frontends
112+
// Handle Implement the Handle interface for frontends that processes requests
110113
func (m *mySqlHandler) Handle(writer models.ResultWriter, req *models.Request) error {
111114
return m.chooseCommand(writer, req)
112115
}

‎frontends/mysqlfe/mysql_session.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
package mysqlfe
22

33
import (
4+
"fmt"
45
"time"
56

67
"github.com/araddon/qlbridge/datasource"
78
"github.com/araddon/qlbridge/expr"
89
"github.com/araddon/qlbridge/value"
10+
11+
"github.com/dataux/dataux/version"
912
)
1013

1114
// http://dev.mysql.com/doc/refman/5.6/en/server-system-variables.html
@@ -74,7 +77,7 @@ func NewMySqlGlobalVars() *datasource.ContextSimple {
7477
ctx.Data["@@system_time_zone"] = value.NewStringValue("UTC")
7578
ctx.Data["@@time_zone"] = value.NewStringValue("SYSTEM")
7679
ctx.Data["@@tx_isolation"] = value.NewStringValue("REPEATABLE-READ")
77-
ctx.Data["@@version_comment"] = value.NewStringValue("DataUX (MIT), Release .13")
80+
ctx.Data["@@version_comment"] = value.NewStringValue(fmt.Sprintf("DataUX (MIT), Release .%s", version.Version))
7881
ctx.Data["@@character_set_server"] = value.NewStringValue("utf8")
7982
return ctx
8083
}

‎main.go‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@ import (
88
"os"
99
"runtime"
1010

11-
// Backend Side-Effect imports
11+
// Backend Side-Effect imports, ie load the providers into registry but
12+
// config will determine if they get used.
13+
// if you are building custom daemon, you can cherry pick sources you care about
1214
_ "github.com/dataux/dataux/backends/bigtable"
1315
_ "github.com/dataux/dataux/backends/cassandra"
1416
_ "github.com/dataux/dataux/backends/datastore"
@@ -33,24 +35,29 @@ var (
3335

3436
func init() {
3537
flag.StringVar(&configFile, "config", "dataux.conf", "dataux proxy config file")
36-
flag.StringVar(&logLevel, "loglevel", "debug", "logging [ debug,info,warn,error ]")
38+
flag.StringVar(&logLevel, "loglevel", "info", "logging [ debug,info,warn,error ]")
3739
flag.StringVar(&pprofPort, "pprof", ":18008", "pprof and metrics port")
3840
flag.IntVar(&workerCt, "workerct", 3, "Number of worker nodes")
3941
flag.Parse()
4042
}
4143
func main() {
4244

4345
runtime.GOMAXPROCS(runtime.NumCPU())
44-
45-
if len(configFile) == 0 {
46-
u.Errorf("must use a config file")
47-
return
48-
}
4946
u.SetupLogging(logLevel)
5047
u.SetColorIfTerminal()
5148

52-
proxy.LoadConfig(configFile)
49+
// First try to look for dataux.conf or provided conf file
50+
// if that fails then use the empty default which means api's
51+
// etc can be used to dynamically define schema etc
52+
_, err := proxy.LoadConfig(configFile)
53+
if err != nil {
54+
_, err = proxy.LoadConfigString(DefaultConfig)
55+
if err != nil {
56+
os.Exit(1)
57+
}
58+
}
5359

60+
// go profiling
5461
if pprofPort != "" {
5562
conn, err := net.Listen("tcp", pprofPort)
5663
if err != nil {
@@ -67,3 +74,14 @@ func main() {
6774

6875
proxy.RunDaemon(true, workerCt)
6976
}
77+
78+
var DefaultConfig = `
79+
80+
frontends : [
81+
{
82+
type : mysql
83+
address : "0.0.0.0:4000"
84+
}
85+
]
86+
87+
`

0 commit comments

Comments
 (0)