Skip to content

Commit 5467dde

Browse files
tseavertseaver
committed
Merge pull request #1047 from tseaver/bigquery-system_tests
Initial BQ system tests
2 parents 6a47028 + 62509d2 commit 5467dde

File tree

3 files changed

+241
-0
lines changed

3 files changed

+241
-0
lines changed

‎scripts/run_system_tests.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,4 @@ fi
3838
python system_tests/run_system_test.py --package datastore
3939
python system_tests/run_system_test.py --package storage
4040
python system_tests/run_system_test.py --package pubsub
41+
python system_tests/run_system_test.py --package bigquery

‎system_tests/bigquery.py‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
# Copyright 2015 Google Inc. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import operator
16+
import time
17+
18+
import unittest2
19+
20+
from gcloud import _helpers
21+
from gcloud.environment_vars import TESTS_PROJECT
22+
from gcloud import bigquery
23+
24+
25+
_helpers.PROJECT = TESTS_PROJECT
26+
CLIENT = bigquery.Client()
27+
28+
29+
class TestBigQuery(unittest2.TestCase):
30+
31+
def setUp(self):
32+
self.to_delete = []
33+
34+
def tearDown(self):
35+
for doomed in self.to_delete:
36+
doomed.delete()
37+
38+
def test_create_dataset(self):
39+
DATASET_NAME = 'system_tests'
40+
dataset = CLIENT.dataset(DATASET_NAME)
41+
self.assertFalse(dataset.exists())
42+
dataset.create()
43+
self.to_delete.append(dataset)
44+
self.assertTrue(dataset.exists())
45+
self.assertEqual(dataset.name, DATASET_NAME)
46+
47+
def test_reload_dataset(self):
48+
DATASET_NAME = 'system_tests'
49+
dataset = CLIENT.dataset(DATASET_NAME)
50+
dataset.friendly_name = 'Friendly'
51+
dataset.description = 'Description'
52+
dataset.create()
53+
self.to_delete.append(dataset)
54+
other = CLIENT.dataset(DATASET_NAME)
55+
other.reload()
56+
self.assertEqual(other.friendly_name, 'Friendly')
57+
self.assertEqual(other.description, 'Description')
58+
59+
def test_patch_dataset(self):
60+
DATASET_NAME = 'system_tests'
61+
dataset = CLIENT.dataset(DATASET_NAME)
62+
self.assertFalse(dataset.exists())
63+
dataset.create()
64+
self.to_delete.append(dataset)
65+
self.assertTrue(dataset.exists())
66+
self.assertEqual(dataset.friendly_name, None)
67+
self.assertEqual(dataset.description, None)
68+
dataset.patch(friendly_name='Friendly', description='Description')
69+
self.assertEqual(dataset.friendly_name, 'Friendly')
70+
self.assertEqual(dataset.description, 'Description')
71+
72+
def test_update_dataset(self):
73+
DATASET_NAME = 'system_tests'
74+
dataset = CLIENT.dataset(DATASET_NAME)
75+
self.assertFalse(dataset.exists())
76+
dataset.create()
77+
self.to_delete.append(dataset)
78+
self.assertTrue(dataset.exists())
79+
after = [grant for grant in dataset.access_grants
80+
if grant.entity_id != 'projectWriters']
81+
dataset.access_grants = after
82+
dataset.update()
83+
self.assertEqual(len(dataset.access_grants), len(after))
84+
for found, expected in zip(dataset.access_grants, after):
85+
self.assertEqual(found.role, expected.role)
86+
self.assertEqual(found.entity_type, expected.entity_type)
87+
self.assertEqual(found.entity_id, expected.entity_id)
88+
89+
def test_list_datasets(self):
90+
datasets_to_create = [
91+
'new%d' % (1000 * time.time(),),
92+
'newer%d' % (1000 * time.time(),),
93+
'newest%d' % (1000 * time.time(),),
94+
]
95+
for dataset_name in datasets_to_create:
96+
dataset = CLIENT.dataset(dataset_name)
97+
dataset.create()
98+
self.to_delete.append(dataset)
99+
100+
# Retrieve the datasets.
101+
all_datasets, token = CLIENT.list_datasets()
102+
self.assertTrue(token is None)
103+
created = [dataset for dataset in all_datasets
104+
if dataset.name in datasets_to_create and
105+
dataset.project == CLIENT.project]
106+
self.assertEqual(len(created), len(datasets_to_create))
107+
108+
def test_create_table(self):
109+
DATASET_NAME = 'system_tests'
110+
dataset = CLIENT.dataset(DATASET_NAME)
111+
self.assertFalse(dataset.exists())
112+
dataset.create()
113+
self.to_delete.append(dataset)
114+
TABLE_NAME = 'test_table'
115+
full_name = bigquery.SchemaField('full_name', 'STRING',
116+
mode='REQUIRED')
117+
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
118+
table = dataset.table(TABLE_NAME, schema=[full_name, age])
119+
self.assertFalse(table.exists())
120+
table.create()
121+
self.to_delete.insert(0, table)
122+
self.assertTrue(table.exists())
123+
self.assertEqual(table.name, TABLE_NAME)
124+
self.assertTrue(table._dataset is dataset)
125+
126+
def test_list_tables(self):
127+
DATASET_NAME = 'system_tests'
128+
dataset = CLIENT.dataset(DATASET_NAME)
129+
self.assertFalse(dataset.exists())
130+
dataset.create()
131+
self.to_delete.append(dataset)
132+
tables_to_create = [
133+
'new%d' % (1000 * time.time(),),
134+
'newer%d' % (1000 * time.time(),),
135+
'newest%d' % (1000 * time.time(),),
136+
]
137+
full_name = bigquery.SchemaField('full_name', 'STRING',
138+
mode='REQUIRED')
139+
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
140+
for table_name in tables_to_create:
141+
table = dataset.table(table_name, schema=[full_name, age])
142+
table.create()
143+
self.to_delete.insert(0, table)
144+
145+
# Retrieve the tables.
146+
all_tables, token = dataset.list_tables()
147+
self.assertTrue(token is None)
148+
created = [table for table in all_tables
149+
if table.name in tables_to_create and
150+
table._dataset.name == DATASET_NAME]
151+
self.assertEqual(len(created), len(tables_to_create))
152+
153+
def test_patch_table(self):
154+
DATASET_NAME = 'system_tests'
155+
dataset = CLIENT.dataset(DATASET_NAME)
156+
self.assertFalse(dataset.exists())
157+
dataset.create()
158+
self.to_delete.append(dataset)
159+
TABLE_NAME = 'test_table'
160+
full_name = bigquery.SchemaField('full_name', 'STRING',
161+
mode='REQUIRED')
162+
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
163+
table = dataset.table(TABLE_NAME, schema=[full_name, age])
164+
self.assertFalse(table.exists())
165+
table.create()
166+
self.to_delete.insert(0, table)
167+
self.assertTrue(table.exists())
168+
self.assertEqual(table.friendly_name, None)
169+
self.assertEqual(table.description, None)
170+
table.patch(friendly_name='Friendly', description='Description')
171+
self.assertEqual(table.friendly_name, 'Friendly')
172+
self.assertEqual(table.description, 'Description')
173+
174+
def test_update_table(self):
175+
DATASET_NAME = 'system_tests'
176+
dataset = CLIENT.dataset(DATASET_NAME)
177+
self.assertFalse(dataset.exists())
178+
dataset.create()
179+
self.to_delete.append(dataset)
180+
TABLE_NAME = 'test_table'
181+
full_name = bigquery.SchemaField('full_name', 'STRING',
182+
mode='REQUIRED')
183+
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
184+
table = dataset.table(TABLE_NAME, schema=[full_name, age])
185+
self.assertFalse(table.exists())
186+
table.create()
187+
self.to_delete.insert(0, table)
188+
self.assertTrue(table.exists())
189+
voter = bigquery.SchemaField('voter', 'BOOLEAN', mode='NULLABLE')
190+
schema = table.schema
191+
schema.append(voter)
192+
table.schema = schema
193+
table.update()
194+
self.assertEqual(len(table.schema), len(schema))
195+
for found, expected in zip(table.schema, schema):
196+
self.assertEqual(found.name, expected.name)
197+
self.assertEqual(found.field_type, expected.field_type)
198+
self.assertEqual(found.mode, expected.mode)
199+
200+
def test_load_table_then_dump_table(self):
201+
ROWS = [
202+
('Phred Phlyntstone', 32),
203+
('Bharney Rhubble', 33),
204+
('Wylma Phlyntstone', 29),
205+
('Bhettye Rhubble', 27),
206+
]
207+
ROW_IDS = range(len(ROWS))
208+
DATASET_NAME = 'system_tests'
209+
dataset = CLIENT.dataset(DATASET_NAME)
210+
self.assertFalse(dataset.exists())
211+
dataset.create()
212+
self.to_delete.append(dataset)
213+
TABLE_NAME = 'test_table'
214+
full_name = bigquery.SchemaField('full_name', 'STRING',
215+
mode='REQUIRED')
216+
age = bigquery.SchemaField('age', 'INTEGER', mode='REQUIRED')
217+
table = dataset.table(TABLE_NAME, schema=[full_name, age])
218+
self.assertFalse(table.exists())
219+
table.create()
220+
self.to_delete.insert(0, table)
221+
self.assertTrue(table.exists())
222+
223+
errors = table.insert_data(ROWS, ROW_IDS)
224+
self.assertEqual(len(errors), 0)
225+
226+
rows = ()
227+
counter = 9
228+
# Allow for 90 seconds of "warm up" before rows visible. See:
229+
# https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataavailability
230+
231+
while len(rows) == 0 and counter > 0:
232+
counter -= 1
233+
rows, _, _ = table.fetch_data()
234+
if len(rows) == 0:
235+
time.sleep(10)
236+
237+
by_age = operator.itemgetter(1)
238+
self.assertEqual(sorted(rows, key=by_age),
239+
sorted(ROWS, key=by_age))

‎system_tests/run_system_test.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
'datastore': ['dataset_id', 'credentials'],
2525
'storage': ['project', 'credentials'],
2626
'pubsub': ['project', 'credentials'],
27+
'bigquery': ['project', 'credentials'],
2728
}
2829

2930

0 commit comments

Comments
 (0)