Skip to content

Commit 7960768

Browse files
aymericdamienaymericdamien
committed
update tf dataset
1 parent 12ed38e commit 7960768

File tree

6 files changed

+49
-96
lines changed

6 files changed

+49
-96
lines changed

‎examples/2_BasicModels/logistic_regression_eager_api.py‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@
2626
display_step = 100
2727

2828
dataset = tf.data.Dataset.from_tensor_slices(
29-
(mnist.train.images, mnist.train.labels)).batch(batch_size)
29+
(mnist.train.images, mnist.train.labels))
30+
dataset = dataset.repeat().batch(batch_size).prefetch(batch_size)
3031
dataset_iter = tfe.Iterator(dataset)
3132

3233
# Variables
@@ -64,12 +65,7 @@ def accuracy_fn(inference_fn, inputs, labels):
6465
for step in range(num_steps):
6566

6667
# Iterate through the dataset
67-
try:
68-
d = dataset_iter.next()
69-
except StopIteration:
70-
# Refill queue
71-
dataset_iter = tfe.Iterator(dataset)
72-
d = dataset_iter.next()
68+
d = dataset_iter.next()
7369

7470
# Images
7571
x_batch = d[0]

‎examples/3_NeuralNetworks/neural_network_eager_api.py‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@
3939

4040
# Using TF Dataset to split data into batches
4141
dataset = tf.data.Dataset.from_tensor_slices(
42-
(mnist.train.images, mnist.train.labels)).batch(batch_size)
42+
(mnist.train.images, mnist.train.labels))
43+
dataset = dataset.repeat().batch(batch_size).prefetch(batch_size)
4344
dataset_iter = tfe.Iterator(dataset)
4445

4546

@@ -92,12 +93,7 @@ def accuracy_fn(inference_fn, inputs, labels):
9293
for step in range(num_steps):
9394

9495
# Iterate through the dataset
95-
try:
96-
d = dataset_iter.next()
97-
except StopIteration:
98-
# Refill queue
99-
dataset_iter = tfe.Iterator(dataset)
100-
d = dataset_iter.next()
96+
d = dataset_iter.next()
10197

10298
# Images
10399
x_batch = d[0]

‎examples/5_DataManagement/tensorflow_dataset_api.py‎

Lines changed: 11 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -29,21 +29,21 @@
2929
sess = tf.Session()
3030

3131
# Create a dataset tensor from the images and the labels
32-
dataset = tf.contrib.data.Dataset.from_tensor_slices(
32+
dataset = tf.data.Dataset.from_tensor_slices(
3333
(mnist.train.images, mnist.train.labels))
34+
# Automatically refill the data queue when empty
35+
dataset = dataset.repeat()
3436
# Create batches of data
3537
dataset = dataset.batch(batch_size)
36-
# Create an iterator, to go over the dataset
38+
# Prefetch data for faster
39+
dataset = dataset.prefetch(batch_size)
40+
41+
# Create an iterator over the dataset
3742
iterator = dataset.make_initializable_iterator()
38-
# It is better to use 2 placeholders, to avoid to load all data into memory,
39-
# and avoid the 2Gb restriction length of a tensor.
40-
_data = tf.placeholder(tf.float32, [None, n_input])
41-
_labels = tf.placeholder(tf.float32, [None, n_classes])
4243
# Initialize the iterator
43-
sess.run(iterator.initializer, feed_dict={_data: mnist.train.images,
44-
_labels: mnist.train.labels})
44+
sess.run(iterator.initializer)
4545

46-
# Neural Net Input
46+
# Neural Net Input (images, labels)
4747
X, Y = iterator.get_next()
4848

4949

@@ -116,15 +116,8 @@ def conv_net(x, n_classes, dropout, reuse, is_training):
116116
# Training cycle
117117
for step in range(1, num_steps + 1):
118118

119-
try:
120-
# Run optimization
121-
sess.run(train_op)
122-
except tf.errors.OutOfRangeError:
123-
# Reload the iterator when it reaches the end of the dataset
124-
sess.run(iterator.initializer,
125-
feed_dict={_data: mnist.train.images,
126-
_labels: mnist.train.labels})
127-
sess.run(train_op)
119+
# Run optimization
120+
sess.run(train_op)
128121

129122
if step % display_step == 0 or step == 1:
130123
# Calculate batch loss and accuracy

‎notebooks/2_BasicModels/logistic_regression_eager_api.ipynb‎

Lines changed: 8 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,7 @@
5454
{
5555
"cell_type": "code",
5656
"execution_count": 3,
57-
"metadata": {
58-
"collapsed": false
59-
},
57+
"metadata": {},
6058
"outputs": [
6159
{
6260
"name": "stdout",
@@ -100,7 +98,8 @@
10098
"source": [
10199
"# Iterator for the dataset\n",
102100
"dataset = tf.data.Dataset.from_tensor_slices(\n",
103-
" (mnist.train.images, mnist.train.labels)).batch(batch_size)\n",
101+
" (mnist.train.images, mnist.train.labels))\n",
102+
"dataset = dataset.repeat().batch(batch_size).prefetch(batch_size)\n",
104103
"dataset_iter = tfe.Iterator(dataset)"
105104
]
106105
},
@@ -151,9 +150,7 @@
151150
{
152151
"cell_type": "code",
153152
"execution_count": 8,
154-
"metadata": {
155-
"collapsed": false
156-
},
153+
"metadata": {},
157154
"outputs": [
158155
{
159156
"name": "stdout",
@@ -181,12 +178,7 @@
181178
"for step in range(num_steps):\n",
182179
"\n",
183180
" # Iterate through the dataset\n",
184-
" try:\n",
185-
" d = dataset_iter.next()\n",
186-
" except StopIteration:\n",
187-
" # Refill queue\n",
188-
" dataset_iter = tfe.Iterator(dataset)\n",
189-
" d = dataset_iter.next()\n",
181+
" d = dataset_iter.next()\n",
190182
"\n",
191183
" # Images\n",
192184
" x_batch = d[0]\n",
@@ -222,9 +214,7 @@
222214
{
223215
"cell_type": "code",
224216
"execution_count": 9,
225-
"metadata": {
226-
"collapsed": false
227-
},
217+
"metadata": {},
228218
"outputs": [
229219
{
230220
"name": "stdout",
@@ -246,7 +236,7 @@
246236
],
247237
"metadata": {
248238
"kernelspec": {
249-
"display_name": "Python [default]",
239+
"display_name": "Python 2",
250240
"language": "python",
251241
"name": "python2"
252242
},
@@ -260,7 +250,7 @@
260250
"name": "python",
261251
"nbconvert_exporter": "python",
262252
"pygments_lexer": "ipython2",
263-
"version": "2.7.12"
253+
"version": "2.7.14"
264254
}
265255
},
266256
"nbformat": 4,

‎notebooks/3_NeuralNetworks/neural_network_eager_api.ipynb‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,7 @@
6060
{
6161
"cell_type": "code",
6262
"execution_count": 3,
63-
"metadata": {
64-
"collapsed": false
65-
},
63+
"metadata": {},
6664
"outputs": [
6765
{
6866
"name": "stdout",
@@ -112,7 +110,8 @@
112110
"source": [
113111
"# Using TF Dataset to split data into batches\n",
114112
"dataset = tf.data.Dataset.from_tensor_slices(\n",
115-
" (mnist.train.images, mnist.train.labels)).batch(batch_size)\n",
113+
" (mnist.train.images, mnist.train.labels))\n",
114+
"dataset = dataset.repeat().batch(batch_size).prefetch(batch_size)\n",
116115
"dataset_iter = tfe.Iterator(dataset)"
117116
]
118117
},
@@ -180,9 +179,7 @@
180179
{
181180
"cell_type": "code",
182181
"execution_count": 8,
183-
"metadata": {
184-
"collapsed": false
185-
},
182+
"metadata": {},
186183
"outputs": [
187184
{
188185
"name": "stdout",
@@ -210,13 +207,8 @@
210207
"for step in range(num_steps):\n",
211208
"\n",
212209
" # Iterate through the dataset\n",
213-
" try:\n",
214-
" d = dataset_iter.next()\n",
215-
" except StopIteration:\n",
216-
" # Refill queue\n",
217-
" dataset_iter = tfe.Iterator(dataset)\n",
218-
" d = dataset_iter.next()\n",
219-
"\n",
210+
" d = dataset_iter.next()\n",
211+
" \n",
220212
" # Images\n",
221213
" x_batch = d[0]\n",
222214
" # Labels\n",
@@ -251,9 +243,7 @@
251243
{
252244
"cell_type": "code",
253245
"execution_count": 9,
254-
"metadata": {
255-
"collapsed": false
256-
},
246+
"metadata": {},
257247
"outputs": [
258248
{
259249
"name": "stdout",
@@ -275,7 +265,7 @@
275265
],
276266
"metadata": {
277267
"kernelspec": {
278-
"display_name": "Python [default]",
268+
"display_name": "Python 2",
279269
"language": "python",
280270
"name": "python2"
281271
},
@@ -289,7 +279,7 @@
289279
"name": "python",
290280
"nbconvert_exporter": "python",
291281
"pygments_lexer": "ipython2",
292-
"version": "2.7.12"
282+
"version": "2.7.14"
293283
}
294284
},
295285
"nbformat": 4,

‎notebooks/5_DataManagement/tensorflow_dataset_api.ipynb‎

Lines changed: 15 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,7 @@
22
"cells": [
33
{
44
"cell_type": "markdown",
5-
"metadata": {
6-
"collapsed": false
7-
},
5+
"metadata": {},
86
"source": [
97
"# TensorFlow Dataset API\n",
108
"\n",
@@ -19,9 +17,7 @@
1917
{
2018
"cell_type": "code",
2119
"execution_count": 1,
22-
"metadata": {
23-
"collapsed": false
24-
},
20+
"metadata": {},
2521
"outputs": [
2622
{
2723
"name": "stdout",
@@ -64,21 +60,21 @@
6460
"sess = tf.Session()\n",
6561
"\n",
6662
"# Create a dataset tensor from the images and the labels\n",
67-
"dataset = tf.contrib.data.Dataset.from_tensor_slices(\n",
63+
"dataset = tf.data.Dataset.from_tensor_slices(\n",
6864
" (mnist.train.images, mnist.train.labels))\n",
65+
"# Automatically refill the data queue when empty\n",
66+
"dataset = dataset.repeat()\n",
6967
"# Create batches of data\n",
7068
"dataset = dataset.batch(batch_size)\n",
71-
"# Create an iterator, to go over the dataset\n",
69+
"# Prefetch data for faster\n",
70+
"dataset = dataset.prefetch(batch_size)\n",
71+
"\n",
72+
"# Create an iterator over the dataset\n",
7273
"iterator = dataset.make_initializable_iterator()\n",
73-
"# It is better to use 2 placeholders, to avoid to load all data into memory,\n",
74-
"# and avoid the 2Gb restriction length of a tensor.\n",
75-
"_data = tf.placeholder(tf.float32, [None, n_input])\n",
76-
"_labels = tf.placeholder(tf.float32, [None, n_classes])\n",
7774
"# Initialize the iterator\n",
78-
"sess.run(iterator.initializer, feed_dict={_data: mnist.train.images,\n",
79-
" _labels: mnist.train.labels})\n",
75+
"sess.run(iterator.initializer)\n",
8076
"\n",
81-
"# Neural Net Input\n",
77+
"# Neural Net Input (images, labels)\n",
8278
"X, Y = iterator.get_next()"
8379
]
8480
},
@@ -155,7 +151,6 @@
155151
"cell_type": "code",
156152
"execution_count": 4,
157153
"metadata": {
158-
"collapsed": false,
159154
"scrolled": false
160155
},
161156
"outputs": [
@@ -188,15 +183,8 @@
188183
"# Training cycle\n",
189184
"for step in range(1, num_steps + 1):\n",
190185
" \n",
191-
" try:\n",
192-
" # Run optimization\n",
193-
" sess.run(train_op)\n",
194-
" except tf.errors.OutOfRangeError:\n",
195-
" # Reload the iterator when it reaches the end of the dataset\n",
196-
" sess.run(iterator.initializer, \n",
197-
" feed_dict={_data: mnist.train.images,\n",
198-
" _labels: mnist.train.labels})\n",
199-
" sess.run(train_op)\n",
186+
" # Run optimization\n",
187+
" sess.run(train_op)\n",
200188
" \n",
201189
" if step % display_step == 0 or step == 1:\n",
202190
" # Calculate batch loss and accuracy\n",
@@ -212,7 +200,7 @@
212200
],
213201
"metadata": {
214202
"kernelspec": {
215-
"display_name": "Python [default]",
203+
"display_name": "Python 2",
216204
"language": "python",
217205
"name": "python2"
218206
},
@@ -226,7 +214,7 @@
226214
"name": "python",
227215
"nbconvert_exporter": "python",
228216
"pygments_lexer": "ipython2",
229-
"version": "2.7.12"
217+
"version": "2.7.14"
230218
}
231219
},
232220
"nbformat": 4,

0 commit comments

Comments
 (0)