#

# dataux configuration example

#

# config format is nginx ish, it is quite lenient see https://github.com/lytics/confl

# - think of it as lenient json with support for comments

# - name/value pairs may be separated by colons or equals [:,=,(whitespace)]

# - value's: for strings they don't have to be quoted

# - keys: left side keys must not have spaces in key name

# - arrays are json: ["a","b"] with entries allowed on new lines

# commas not required between items on new lines

# - objects are values so mobj = { name : "value", ....}

# - comments allowed

# suppress recovery means panic, don't recover

supress_recover: true

## Distributed Runtime Config Setup

# etcd servers

# [ "http://127.0.0.1:2379","http://127.0.0.1:2380"]

etcd = ["http://localhost:2379"]

# Nats.io GnatsD servers

# [ 'nats://10.240.134.116:4222', 'nats://10.145.147.112:4222' ]

nats = [ "nats://localhost:4222" ]

## Frontend

# frontend is inbound tcp connection listener (only mysql currently supported)

# - we don't bind to 3306 because that is mysql's

#

# mysql -h127.0.0.1 -P4000 -Ddatauxtest

#

frontends [

{

type : mysql

address : "0.0.0.0:4000"

}

]

# schemas: ie a virtual database made up of

# combining tables from each source into a flattened table namespace

schemas : [

{

name : datauxtest

sources : [ "mgo_datauxtest", "es_test" , "cass" ]

}

{

name : bt

sources : [ "bt"]

}

{

name : kube

sources : [ "minikube" ]

}

{

name : baseball

sources : [ "gcs_baseball_csv" ]

}

# {

# name : baseball_small

# sources : [ "gcs_baseball_small_csv" ]

# }

{

name : baseball_local

sources : [ "local_baseball_csv" ]

}

]

# sources

# - elasticsearch which uses example data from github, see tools/importgithub

# - csv files: cloudstorage (gcs, s3)

# - csv files: localstorage

# - mongo

#

sources : [

{

name : mgo_datauxtest

type : mongo

hosts : ["localhost"]

}

{

name : es_test

type : elasticsearch

hosts : ["http://localhost:9200"]

#tables_to_load : [ "github_commits" ]

}

{

# example of a cassandra source

name cass

type cassandra

settings {

keyspace datauxtest

numconns 100

hosts ["localhost"]

}

}

{

name : minikube

type : kubernetes

settings {

# locaction of kube config

# defaults to $HOME/.kube/config

#kube_conf "/etc/kube.conf"

# kubernetes namespace to use

#namespace default

}

}

{

# this section is a cloudstorage source for csv files

# stored on google-storage from

# http://seanlahman.com/baseball-archive/statistics/

name "gcs_baseball_csv"

type "cloudstore"

settings {

# types: "gcs","localfs"

type "gcs"

# google cloud storage bucket name, ie the gs://bucket-name

bucket "lytics-dataux-tests"

# restrict to looking for files inside this folder on bucket

path "baseball/"

# format, csv, json, custom

format "csv"

}

}

{

# this section is a cloudstorage source for csv files

# stored on google-storage from

# http://seanlahman.com/baseball-archive/statistics/

# only one table from previous set of data

# and that data is split across 2 partitions

name "gcs_baseball_small_csv"

type "cloudstore"

partition_count 2

settings {

# types: "gcs","localfs"

type "gcs"

# google cloud storage bucket name, ie the gs://bucket-name

bucket "lytics-dataux-tests"

# restrict to looking for files inside this folder on bucket

path "baseball2/"

# format, csv, json, custom

format "csv"

}

}

{

# same files as above baseball csv only copied locally

# mkdir -p /tmp/baseball

# gsutil rsync -d -r gs://lytics-dataux-tests/baseball /tmp/baseball

name "local_baseball_csv"

type "cloudstore"

settings {

type "localfs"

localpath "/tmp/baseball"

format "csv"

}

}

{

# example of using cloudstore using jwt tokens

name : "gcs1"

type : "cloudstore"

settings : {

type : "gcs"

project : "myproject"

jwt : "/home/aaron/.ssh/googlejwt.json"

}

}

{

name : bt

type : bigtable

tables_to_load : [ "datauxtest" , "article", "user", "event" ]

settings {

instance "bigtable0"

# project will be loaded from ENV $GCEPROJECT

}

}

]