From 373e9732eb2d0b5d05a6831c590c7eba4574c379 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 18 Jul 2017 12:26:58 +0200 Subject: [PATCH 01/24] new layerwise tf nets --- nideep/nets/abstract_net.py | 73 ++++++++++++++++++++++++++++++++++ nideep/nets/abstract_net_tf.py | 41 +++++++++++++++++++ nideep/nets/mlp_tf.py | 63 +++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 nideep/nets/abstract_net.py create mode 100644 nideep/nets/abstract_net_tf.py create mode 100644 nideep/nets/mlp_tf.py diff --git a/nideep/nets/abstract_net.py b/nideep/nets/abstract_net.py new file mode 100644 index 0000000..a9a3e43 --- /dev/null +++ b/nideep/nets/abstract_net.py @@ -0,0 +1,73 @@ +''' +Created on Jul 14, 2017 + +@author: kashefy +''' +from __future__ import division, print_function, absolute_import +from abc import ABCMeta, abstractmethod + +class AbstractNet(object): + ''' + classdocs + ''' + __metaclass__ = ABCMeta + + @abstractmethod + def _init_learning_params(self): + pass + + @abstractmethod + def build(self): + pass + + def _config_scopes(self): + self.var_scope = self.prefix + 'layer-%d' % self.depth + self.name_scope = self.var_scope + '/' + + def _depth(self, params): + if 'depth' not in params: + depth = 1 + else: + depth = params['depth'] + if depth < 1: + raise ValueError("Invalid depth (%s). Must be >= 1" % depth) + self.depth = depth + + @property + def x(self): + return self._x + + @x.setter + def x(self, value): + self._x = value + + @x.deleter + def x(self): + del self._x + + @property + def p(self): + return self._p + + @p.setter + def p(self, value): + self._p = value + + @p.deleter + def p(self): + del self._p + + def __init__(self, params): + ''' + Constructor + ''' + self._x = None + self._p = None + self.prefix = '' + if 'prefix' in params: + self.prefix = params['prefix'] + self._depth(params) + self._config_scopes() + self._init_learning_params() + self._cost_op = None + \ No newline at end of file diff --git a/nideep/nets/abstract_net_tf.py b/nideep/nets/abstract_net_tf.py new file mode 100644 index 0000000..cc4fd29 --- /dev/null +++ b/nideep/nets/abstract_net_tf.py @@ -0,0 +1,41 @@ +''' +Created on Jul 18, 2017 + +@author: kashefy +''' +import tensorflow as tf +from nideep.nets.abstract_net import AbstractNet +from abc import ABCMeta, abstractmethod + +class AbstractNetTF(AbstractNet): + ''' + classdocs + ''' + __metaclass__ = ABCMeta + + @staticmethod + def _init_weight_op(): + return tf.random_normal_initializer() + + @staticmethod + def _init_bias_op(value): + return tf.constant_initializer(value) + + def vars_new(self): + return tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, + scope=self.name_scope) + + @abstractmethod + def _init_learning_params_scoped(self): + pass + + def _init_learning_params(self): + with tf.variable_scope(self.var_scope): + self._init_learning_params_scoped() + pass + + def __init__(self, params): + ''' + Constructor + ''' + super(AbstractNetTF, self).__init__(params) \ No newline at end of file diff --git a/nideep/nets/mlp_tf.py b/nideep/nets/mlp_tf.py new file mode 100644 index 0000000..f04b9d4 --- /dev/null +++ b/nideep/nets/mlp_tf.py @@ -0,0 +1,63 @@ +''' +Created on Jul 14, 2017 + +@author: kashefy +''' +import tensorflow as tf +from nideep.nets.abstract_net_tf import AbstractNetTF + +class MLP(AbstractNetTF): + ''' + classdocs + ''' + def _init_learning_params_scoped(self): + fc_name_w = 'fc-%d/w' % self.depth + self.w = { + fc_name_w: tf.get_variable(fc_name_w, + [self.n_hidden[-1], self.n_outputs], + initializer=tf.random_normal_initializer(), + ) + } + self.b = {} + for key, value in self.w.iteritems(): + key_b = key.replace('/w', '/b').replace('_w', '_b').replace('-w', '-b') + self.b[key_b] = tf.get_variable(\ + key_b, + [int(value.get_shape()[-1])], + initializer=tf.constant_initializer(0.) + ) + + def _fc(self, x): + fc_name_w = 'fc-%d/w' % self.depth + fc_name_b = 'fc-%d/b' % self.depth + fc_op = tf.add(tf.matmul(x, self.w[fc_name_w]), + self.b[fc_name_b], + name='fc-%d' % self.depth) + self._y_logits = fc_op + self.y_pred = tf.nn.softmax(fc_op, name='a-%d' % self.depth) + return self.y_pred, self._y_logits + + def build(self): + with tf.name_scope(self.name_scope + 'fc'): + pred, logits = self._fc(self.x) + return pred, logits + + def cost(self, y_true, name=None): + with tf.name_scope(self.name_scope): + self._cost_op = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(\ + labels=y_true, logits=self._y_logits), + name=name) + return self._cost_op + + def __init__(self, params): + ''' + Constructor + ''' + # Network Parameters + self.n_hidden = params['n_hidden'] # 1st layer num features + self.n_outputs = params['n_outputs'] # 1st layer num features + super(MLP, self).__init__(params) + + self._cost_op = None + self._y_logits = None \ No newline at end of file From b39ef4e9747f5e2223416c70be96bef6249ff055 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Wed, 19 Jul 2017 17:49:49 +0200 Subject: [PATCH 02/24] mlp hidden should be input --- nideep/nets/mlp_tf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nideep/nets/mlp_tf.py b/nideep/nets/mlp_tf.py index f04b9d4..8119aa1 100644 --- a/nideep/nets/mlp_tf.py +++ b/nideep/nets/mlp_tf.py @@ -14,7 +14,7 @@ def _init_learning_params_scoped(self): fc_name_w = 'fc-%d/w' % self.depth self.w = { fc_name_w: tf.get_variable(fc_name_w, - [self.n_hidden[-1], self.n_outputs], + [self.n_input, self.n_outputs], initializer=tf.random_normal_initializer(), ) } @@ -55,7 +55,7 @@ def __init__(self, params): Constructor ''' # Network Parameters - self.n_hidden = params['n_hidden'] # 1st layer num features + self.n_input = params['n_input'] # 1st layer num features self.n_outputs = params['n_outputs'] # 1st layer num features super(MLP, self).__init__(params) From 9616caae5cef0b4f9a200caf61f4a38101927dca Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 20 Jul 2017 21:06:59 +0200 Subject: [PATCH 03/24] remove depth attribute --- nideep/nets/abstract_net.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/nideep/nets/abstract_net.py b/nideep/nets/abstract_net.py index a9a3e43..22894bc 100644 --- a/nideep/nets/abstract_net.py +++ b/nideep/nets/abstract_net.py @@ -21,17 +21,8 @@ def build(self): pass def _config_scopes(self): - self.var_scope = self.prefix + 'layer-%d' % self.depth + self.var_scope = self.prefix self.name_scope = self.var_scope + '/' - - def _depth(self, params): - if 'depth' not in params: - depth = 1 - else: - depth = params['depth'] - if depth < 1: - raise ValueError("Invalid depth (%s). Must be >= 1" % depth) - self.depth = depth @property def x(self): @@ -66,7 +57,6 @@ def __init__(self, params): self.prefix = '' if 'prefix' in params: self.prefix = params['prefix'] - self._depth(params) self._config_scopes() self._init_learning_params() self._cost_op = None From 339e428f7e0a13afe777e5a87861abb9478aa093 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Fri, 21 Jul 2017 07:18:25 +0200 Subject: [PATCH 04/24] init bias vars --- nideep/nets/abstract_net_tf.py | 8 ++++++ nideep/nets/mlp_tf.py | 45 +++++++++++++++++----------------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/nideep/nets/abstract_net_tf.py b/nideep/nets/abstract_net_tf.py index cc4fd29..d6918e4 100644 --- a/nideep/nets/abstract_net_tf.py +++ b/nideep/nets/abstract_net_tf.py @@ -25,6 +25,14 @@ def vars_new(self): return tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=self.name_scope) + def _init_bias_vars(self, bias_value=0.): + self.b = {} + for key, tensor in self.w.iteritems(): + key_b = key.replace('/w', '/b').replace('_w', '_b').replace('-w', '-b') + self.b[key_b] = tf.get_variable(key_b, + [tensor.get_shape()[-1].value], + initializer=self._init_bias_op(bias_value)) + @abstractmethod def _init_learning_params_scoped(self): pass diff --git a/nideep/nets/mlp_tf.py b/nideep/nets/mlp_tf.py index 8119aa1..606f5a8 100644 --- a/nideep/nets/mlp_tf.py +++ b/nideep/nets/mlp_tf.py @@ -11,30 +11,29 @@ class MLP(AbstractNetTF): classdocs ''' def _init_learning_params_scoped(self): - fc_name_w = 'fc-%d/w' % self.depth - self.w = { - fc_name_w: tf.get_variable(fc_name_w, - [self.n_input, self.n_outputs], - initializer=tf.random_normal_initializer(), - ) - } - self.b = {} - for key, value in self.w.iteritems(): - key_b = key.replace('/w', '/b').replace('_w', '_b').replace('-w', '-b') - self.b[key_b] = tf.get_variable(\ - key_b, - [int(value.get_shape()[-1])], - initializer=tf.constant_initializer(0.) - ) + self.w = {} + for idx, dim in enumerate(self.n_nodes): + input_dim = self.n_nodes[idx-1] + if idx == 0: + input_dim = self.n_input + fc_name_w = 'fc-%d/w' % idx + self.w[fc_name_w] = tf.get_variable(fc_name_w, + [input_dim, dim], + initializer=self._init_weight_op() + ) + self._init_bias_vars() def _fc(self, x): - fc_name_w = 'fc-%d/w' % self.depth - fc_name_b = 'fc-%d/b' % self.depth - fc_op = tf.add(tf.matmul(x, self.w[fc_name_w]), - self.b[fc_name_b], - name='fc-%d' % self.depth) - self._y_logits = fc_op - self.y_pred = tf.nn.softmax(fc_op, name='a-%d' % self.depth) + in_op = x + for idx in xrange(len(self.n_nodes)): + fc_name_w = 'fc-%d/w' % idx + fc_name_b = 'fc-%d/b' % idx + fc_op = tf.add(tf.matmul(in_op, self.w[fc_name_w]), + self.b[fc_name_b], + name='fc-%d' % idx) + self._y_logits = fc_op + self.y_pred = tf.nn.softmax(fc_op, name='a-%d' % idx) + in_op = self.y_pred return self.y_pred, self._y_logits def build(self): @@ -56,7 +55,7 @@ def __init__(self, params): ''' # Network Parameters self.n_input = params['n_input'] # 1st layer num features - self.n_outputs = params['n_outputs'] # 1st layer num features + self.n_nodes = params['n_nodes'] # 1st layer num features super(MLP, self).__init__(params) self._cost_op = None From ba47b501b4685fa34d496e992c53a53279b82b65 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Mon, 14 Aug 2017 09:55:39 +0200 Subject: [PATCH 05/24] wrapper around mnist tf Dataset --- .gitignore | 4 +- nideep/datasets/mnist/__init__.py | 0 nideep/datasets/mnist/mnist_tf.py | 65 ++++++++++++++++++++++++++ nideep/datasets/mnist/test_mnist_tf.py | 54 +++++++++++++++++++++ 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 nideep/datasets/mnist/__init__.py create mode 100644 nideep/datasets/mnist/mnist_tf.py create mode 100644 nideep/datasets/mnist/test_mnist_tf.py diff --git a/.gitignore b/.gitignore index e60c133..22b9e2b 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,8 @@ target/ *.solverstate.h5 *.binaryproto *leveldb +MNIST_data/ +t*-ubyte.gz ##--------------------------------------------------- ## Remove autosaves generated by the Matlab editor @@ -99,4 +101,4 @@ octave-workspace .idea/ # Merge files -*.orig \ No newline at end of file +*.orig diff --git a/nideep/datasets/mnist/__init__.py b/nideep/datasets/mnist/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nideep/datasets/mnist/mnist_tf.py b/nideep/datasets/mnist/mnist_tf.py new file mode 100644 index 0000000..de2c069 --- /dev/null +++ b/nideep/datasets/mnist/mnist_tf.py @@ -0,0 +1,65 @@ +''' +Created on Jul 31, 2017 + +@author: kashefy +''' +import numpy as np +from tensorflow.python.framework import dtypes, random_seed +from tensorflow.contrib.learn.python.learn.datasets.base import Datasets +from tensorflow.contrib.learn.python.learn.datasets.mnist import DataSet, read_data_sets +# imported for mocking +from tensorflow.contrib.learn.python.learn.datasets.mnist import extract_images + +class MNIST(object): + ''' + classdocs + ''' + @classmethod + def read_data_sets(cls, + train_dir, + fake_data=False, + one_hot=False, + dtype=dtypes.float32, + reshape=True, + validation_size=5000, + seed=None): + + seed1, seed2 = random_seed.get_seed(seed) + # If op level seed is not set, use whatever graph level seed is returned + np.random.seed(seed1 if seed is None else seed2) + + ds = read_data_sets(\ + train_dir, + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=reshape, + validation_size=0, + seed=seed) + perm0 = np.arange(ds.train.num_examples) + np.random.shuffle(perm0) + train_idxs = perm0[validation_size:] + val_idxs = perm0[:validation_size] + + train = DataSet(\ + ds.train.images[train_idxs], + ds.train.labels[train_idxs], + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=False, # already reshaped + seed=seed) + validation = DataSet(\ + ds.train.images[val_idxs], + ds.train.labels[val_idxs], + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=False, # already reshaped + seed=seed) + return Datasets(train=train, validation=validation, test=ds.test) + + def __init__(self, params): + ''' + Constructor + ''' diff --git a/nideep/datasets/mnist/test_mnist_tf.py b/nideep/datasets/mnist/test_mnist_tf.py new file mode 100644 index 0000000..efaee48 --- /dev/null +++ b/nideep/datasets/mnist/test_mnist_tf.py @@ -0,0 +1,54 @@ +''' +Created on Aug 11, 2017 + +@author: kashefy +''' +import os +import tempfile +import shutil +import numpy as np +from nose.tools import assert_equals, assert_true, assert_false, \ + assert_list_equal +from mock import patch +from mnist_tf import MNIST + +class TestMNISTTF: + @classmethod + def setup_class(self): + self.dir_tmp = tempfile.mkdtemp() + + @classmethod + def teardown_class(self): + shutil.rmtree(self.dir_tmp) + + @patch('nideep.datasets.mnist.mnist_tf.extract_images') + def test_read(self, mock_extract): + mock_extract.return_value = np.empty_like((100000, 28, 28)) + d = MNIST.read_data_sets(self.dir_tmp) + assert_equals(d.train.num_examples, 55000) + assert_equals(d.validation.num_examples, 5000) + assert_equals(d.test.num_examples, 10000) + + @patch('nideep.datasets.mnist.mnist_tf.extract_images') + def test_validation_size(self, mock_extract): + mock_extract.return_value = np.empty_like((100000, 28, 28)) + for sz in range(5000, 55000, 5000): + yield self.check_validation_size, sz + + def check_validation_size(self, sz): + d = MNIST.read_data_sets(self.dir_tmp, validation_size=sz) + assert_equals(d.train.num_examples, 60000-sz) + assert_equals(d.validation.num_examples, sz) + assert_equals(d.test.num_examples, 10000) + + def test_shuffle(self): + d1 = MNIST.read_data_sets(self.dir_tmp) + d2 = MNIST.read_data_sets(self.dir_tmp) + assert_false(np.array_equal(d1.train.images, d2.train.images)) + assert_false(np.array_equal(d1.train.labels, d2.train.labels)) + assert_false(np.array_equal(d1.validation.images, d2.validation.images)) + assert_false(np.array_equal(d1.validation.labels, d2.validation.labels)) + assert_true(np.array_equal(d1.test.images, d2.test.images)) + assert_true(np.array_equal(d1.test.labels, d2.test.labels)) +# + \ No newline at end of file From b122923edf6b44ae35f5dc2811048f562ac557d6 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Mon, 14 Aug 2017 11:01:20 +0200 Subject: [PATCH 06/24] test different seeds and add exmple config --- nideep/datasets/mnist/test_mnist_tf.py | 19 ++++++++++++++++++- nideep/test_data/config0.yaml | 2 ++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 nideep/test_data/config0.yaml diff --git a/nideep/datasets/mnist/test_mnist_tf.py b/nideep/datasets/mnist/test_mnist_tf.py index efaee48..cecd97f 100644 --- a/nideep/datasets/mnist/test_mnist_tf.py +++ b/nideep/datasets/mnist/test_mnist_tf.py @@ -50,5 +50,22 @@ def test_shuffle(self): assert_false(np.array_equal(d1.validation.labels, d2.validation.labels)) assert_true(np.array_equal(d1.test.images, d2.test.images)) assert_true(np.array_equal(d1.test.labels, d2.test.labels)) -# + + def test_seed(self): + d1 = MNIST.read_data_sets(self.dir_tmp, seed=123) + d2 = MNIST.read_data_sets(self.dir_tmp, seed=999) + d3 = MNIST.read_data_sets(self.dir_tmp, seed=123) + assert_false(np.array_equal(d1.train.images, d2.train.images)) + assert_false(np.array_equal(d1.train.labels, d2.train.labels)) + assert_false(np.array_equal(d1.validation.images, d2.validation.images)) + assert_false(np.array_equal(d1.validation.labels, d2.validation.labels)) + assert_true(np.array_equal(d1.test.images, d2.test.images)) + assert_true(np.array_equal(d1.test.labels, d2.test.labels)) + # same seed + assert_true(np.array_equal(d1.train.images, d3.train.images)) + assert_true(np.array_equal(d1.train.labels, d3.train.labels)) + assert_true(np.array_equal(d1.validation.images, d3.validation.images)) + assert_true(np.array_equal(d1.validation.labels, d3.validation.labels)) + assert_true(np.array_equal(d1.test.images, d3.test.images)) + assert_true(np.array_equal(d1.test.labels, d3.test.labels)) \ No newline at end of file diff --git a/nideep/test_data/config0.yaml b/nideep/test_data/config0.yaml new file mode 100644 index 0000000..086d8cd --- /dev/null +++ b/nideep/test_data/config0.yaml @@ -0,0 +1,2 @@ +learning_rate: 0.01 +batch_size: 16 From 1f13732b4e3207a4d42f976fde58fdf06e03d53e Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Thu, 17 Aug 2017 23:14:25 +0200 Subject: [PATCH 07/24] use property, istall bunch, edit config --- .travis.yml | 2 +- nideep/nets/mlp_tf.py | 6 +++--- nideep/test_data/config0.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index cdb3f5e..7d88f44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,7 +68,7 @@ before_install: install: # Packages that need to be installed via pip - - travis_retry pip install lmdb tensorflow + - travis_retry pip install lmdb bunch tensorflow - if [[ $WITH_COVERAGE == ON ]]; then echo "Installing Coveralls" travis_retry pip install coveralls; diff --git a/nideep/nets/mlp_tf.py b/nideep/nets/mlp_tf.py index 606f5a8..86118fc 100644 --- a/nideep/nets/mlp_tf.py +++ b/nideep/nets/mlp_tf.py @@ -32,9 +32,9 @@ def _fc(self, x): self.b[fc_name_b], name='fc-%d' % idx) self._y_logits = fc_op - self.y_pred = tf.nn.softmax(fc_op, name='a-%d' % idx) - in_op = self.y_pred - return self.y_pred, self._y_logits + self.p = tf.nn.softmax(fc_op, name='a-%d' % idx) + in_op = self.p + return self.p, self._y_logits def build(self): with tf.name_scope(self.name_scope + 'fc'): diff --git a/nideep/test_data/config0.yaml b/nideep/test_data/config0.yaml index 086d8cd..dcb2981 100644 --- a/nideep/test_data/config0.yaml +++ b/nideep/test_data/config0.yaml @@ -1,2 +1,2 @@ -learning_rate: 0.01 +learning_rate: 0.001 batch_size: 16 From 6eeff952e1633b8ec0a0feb86d17796e6f78dcef Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Sat, 19 Aug 2017 20:13:49 +0200 Subject: [PATCH 08/24] fix scaling --- nideep/datasets/mnist/mnist_tf.py | 5 ++--- nideep/datasets/mnist/test_mnist_tf.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/nideep/datasets/mnist/mnist_tf.py b/nideep/datasets/mnist/mnist_tf.py index de2c069..1df8137 100644 --- a/nideep/datasets/mnist/mnist_tf.py +++ b/nideep/datasets/mnist/mnist_tf.py @@ -40,9 +40,8 @@ def read_data_sets(cls, np.random.shuffle(perm0) train_idxs = perm0[validation_size:] val_idxs = perm0[:validation_size] - train = DataSet(\ - ds.train.images[train_idxs], + np.multiply(ds.train.images[train_idxs], 255.), # will rescale to [0,1] inside ds.train.labels[train_idxs], fake_data=fake_data, one_hot=one_hot, @@ -50,7 +49,7 @@ def read_data_sets(cls, reshape=False, # already reshaped seed=seed) validation = DataSet(\ - ds.train.images[val_idxs], + np.multiply(ds.train.images[val_idxs], 255.), # will rescale to [0,1] inside ds.train.labels[val_idxs], fake_data=fake_data, one_hot=one_hot, diff --git a/nideep/datasets/mnist/test_mnist_tf.py b/nideep/datasets/mnist/test_mnist_tf.py index cecd97f..b0fce9b 100644 --- a/nideep/datasets/mnist/test_mnist_tf.py +++ b/nideep/datasets/mnist/test_mnist_tf.py @@ -68,4 +68,21 @@ def test_seed(self): assert_true(np.array_equal(d1.validation.labels, d3.validation.labels)) assert_true(np.array_equal(d1.test.images, d3.test.images)) assert_true(np.array_equal(d1.test.labels, d3.test.labels)) + + def test_data_scale(self): + from tensorflow.examples.tutorials.mnist import input_data + d0 = input_data.read_data_sets(self.dir_tmp, one_hot=True) + d1 = MNIST.read_data_sets(self.dir_tmp, one_hot=True) + assert_equals(d0.train.images.min(), 0) + assert_equals(d0.train.images.max(), 1) + assert_equals(d0.train.images.min(), d1.train.images.min()) + assert_equals(d0.train.images.max(), d1.train.images.max()) + assert_equals(d0.validation.images.min(), 0) + assert_equals(d0.validation.images.max(), 1) + assert_equals(d0.validation.images.min(), d1.validation.images.min()) + assert_equals(d0.validation.images.max(), d1.validation.images.max()) + assert_equals(d0.test.images.min(), 0) + assert_equals(d0.test.images.max(), 1) + assert_equals(d0.test.images.min(), d1.test.images.min()) + assert_equals(d0.test.images.max(), d1.test.images.max()) \ No newline at end of file From a3754f4f3304894a5ff073ef2fb323b9f15b4ea8 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Sat, 19 Aug 2017 20:14:15 +0200 Subject: [PATCH 09/24] fix weight range --- nideep/nets/abstract_net_tf.py | 5 +++-- nideep/nets/mlp_tf.py | 11 +++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/nideep/nets/abstract_net_tf.py b/nideep/nets/abstract_net_tf.py index d6918e4..e2e3ac8 100644 --- a/nideep/nets/abstract_net_tf.py +++ b/nideep/nets/abstract_net_tf.py @@ -14,8 +14,9 @@ class AbstractNetTF(AbstractNet): __metaclass__ = ABCMeta @staticmethod - def _init_weight_op(): - return tf.random_normal_initializer() + def _init_weight_op(mean=0.0, stddev=0.1): + return tf.random_normal_initializer(mean=mean, + stddev=stddev) @staticmethod def _init_bias_op(value): diff --git a/nideep/nets/mlp_tf.py b/nideep/nets/mlp_tf.py index 86118fc..b20e960 100644 --- a/nideep/nets/mlp_tf.py +++ b/nideep/nets/mlp_tf.py @@ -21,7 +21,7 @@ def _init_learning_params_scoped(self): [input_dim, dim], initializer=self._init_weight_op() ) - self._init_bias_vars() + self._init_bias_vars(bias_value=0.1) def _fc(self, x): in_op = x @@ -31,9 +31,12 @@ def _fc(self, x): fc_op = tf.add(tf.matmul(in_op, self.w[fc_name_w]), self.b[fc_name_b], name='fc-%d' % idx) - self._y_logits = fc_op - self.p = tf.nn.softmax(fc_op, name='a-%d' % idx) - in_op = self.p + if idx < len(self.n_nodes)-1: + a = tf.sigmoid(fc_op, name='a-%d' % idx) + in_op = a + else: + self._y_logits = fc_op + self.p = tf.nn.softmax(fc_op, name='a-%d' % idx) return self.p, self._y_logits def build(self): From 5270cf18ec721df67a2dbc350163d5bc68a80285 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Mon, 18 Sep 2017 08:25:04 +0200 Subject: [PATCH 10/24] fix scaling --- nideep/datasets/mnist/mnist_tf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nideep/datasets/mnist/mnist_tf.py b/nideep/datasets/mnist/mnist_tf.py index 1df8137..5ac0e73 100644 --- a/nideep/datasets/mnist/mnist_tf.py +++ b/nideep/datasets/mnist/mnist_tf.py @@ -41,7 +41,7 @@ def read_data_sets(cls, train_idxs = perm0[validation_size:] val_idxs = perm0[:validation_size] train = DataSet(\ - np.multiply(ds.train.images[train_idxs], 255.), # will rescale to [0,1] inside + np.multiply(ds.train.images[train_idxs], [1., 255.][dtype == dtypes.float32]), # will rescale to [0,1] inside ds.train.labels[train_idxs], fake_data=fake_data, one_hot=one_hot, @@ -49,7 +49,7 @@ def read_data_sets(cls, reshape=False, # already reshaped seed=seed) validation = DataSet(\ - np.multiply(ds.train.images[val_idxs], 255.), # will rescale to [0,1] inside + np.multiply(ds.train.images[val_idxs], [1., 255.][dtype == dtypes.float32]), # will rescale to [0,1] inside ds.train.labels[val_idxs], fake_data=fake_data, one_hot=one_hot, From 91a95d19b540b52488d42e7cf513a14feeb2d517 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Mon, 18 Sep 2017 08:26:27 +0200 Subject: [PATCH 11/24] fixes: property names,variable reuse,abstract methods --- nideep/nets/abstract_net.py | 39 ++++++++++++++++++++++++++++------ nideep/nets/abstract_net_tf.py | 32 +++++++++++++++++++++++----- nideep/nets/mlp_tf.py | 6 +++--- 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/nideep/nets/abstract_net.py b/nideep/nets/abstract_net.py index 22894bc..31c6c60 100644 --- a/nideep/nets/abstract_net.py +++ b/nideep/nets/abstract_net.py @@ -14,11 +14,16 @@ class AbstractNet(object): @abstractmethod def _init_learning_params(self): - pass + pass @abstractmethod + def _init_ops(self): + pass # return prediction_op, logit_op + def build(self): - pass + self._init_learning_params() + self._init_ops() + return self.p, self.logits def _config_scopes(self): self.var_scope = self.prefix @@ -47,6 +52,30 @@ def p(self, value): @p.deleter def p(self): del self._p + + @property + def logits(self): + return self._logits + + @logits.setter + def logits(self, value): + self._logits = value + + @logits.deleter + def logits(self): + del self._logits + + @property + def vars_restored(self): + return self._vars_restored + + @vars_restored.setter + def vars_restored(self, value): + self._vars_restored = value + + @vars_restored.deleter + def vars_restored(self): + del self._vars_restored def __init__(self, params): ''' @@ -54,10 +83,8 @@ def __init__(self, params): ''' self._x = None self._p = None - self.prefix = '' - if 'prefix' in params: - self.prefix = params['prefix'] + self.prefix = params.get('prefix', '') self._config_scopes() - self._init_learning_params() + self._vars_restored = [] self._cost_op = None \ No newline at end of file diff --git a/nideep/nets/abstract_net_tf.py b/nideep/nets/abstract_net_tf.py index e2e3ac8..3da5d5c 100644 --- a/nideep/nets/abstract_net_tf.py +++ b/nideep/nets/abstract_net_tf.py @@ -30,21 +30,43 @@ def _init_bias_vars(self, bias_value=0.): self.b = {} for key, tensor in self.w.iteritems(): key_b = key.replace('/w', '/b').replace('_w', '_b').replace('-w', '-b') - self.b[key_b] = tf.get_variable(key_b, - [tensor.get_shape()[-1].value], - initializer=self._init_bias_op(bias_value)) - + if self.reuse: + self.b[key_b] = self._restore_variable(self.var_scope + '/' + key_b) + else: + self.b[key_b] = tf.get_variable(key_b, + [tensor.get_shape()[-1].value], + initializer=self._init_bias_op(bias_value)) + @abstractmethod def _init_learning_params_scoped(self): pass def _init_learning_params(self): - with tf.variable_scope(self.var_scope): + with tf.variable_scope(self.var_scope, reuse=self.reuse): self._init_learning_params_scoped() pass + def _restore_variable(self, var_name): + v = self.get_tensor_by_name(var_name) + self.vars_restored.append(v) + return v + + def get_tensor_by_name(self, tensor_name, graph=tf.get_default_graph(), index=-1): + name_with_index = self.get_tensor_names(tensor_name, graph=graph)[index] + return graph.get_tensor_by_name(name_with_index) + + def get_tensor_names(self, tensor_name, graph=tf.get_default_graph()): +# op_names = [output.name for op in graph.get_operations() +# if op.op_def and 'Variable' in op.op_def.name and +# tensor_name == op.name for output in op.outputs] + op_names = [output.name for op in graph.get_operations() + if op.op_def and + tensor_name == op.name for output in op.outputs] + return op_names + def __init__(self, params): ''' Constructor ''' + self.reuse = params.get('reuse', None) super(AbstractNetTF, self).__init__(params) \ No newline at end of file diff --git a/nideep/nets/mlp_tf.py b/nideep/nets/mlp_tf.py index b20e960..12ad23a 100644 --- a/nideep/nets/mlp_tf.py +++ b/nideep/nets/mlp_tf.py @@ -5,6 +5,7 @@ ''' import tensorflow as tf from nideep.nets.abstract_net_tf import AbstractNetTF +from networkx.algorithms.shortest_paths.unweighted import predecessor class MLP(AbstractNetTF): ''' @@ -39,10 +40,9 @@ def _fc(self, x): self.p = tf.nn.softmax(fc_op, name='a-%d' % idx) return self.p, self._y_logits - def build(self): + def _init_ops(self): with tf.name_scope(self.name_scope + 'fc'): - pred, logits = self._fc(self.x) - return pred, logits + self.p, self.logits = self._fc(self.x) def cost(self, y_true, name=None): with tf.name_scope(self.name_scope): From 726bb062e10c19224c34adf1383e612b4feae094 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Mon, 18 Sep 2017 20:50:26 +0200 Subject: [PATCH 12/24] try tfrecords --- nideep/datasets/mnist/mnist_tf.py | 61 ++++++++++++++++++++++++ nideep/datasets/mnist/test_mnist_tf.py | 65 +++++++++++++++++++++++++- 2 files changed, 125 insertions(+), 1 deletion(-) diff --git a/nideep/datasets/mnist/mnist_tf.py b/nideep/datasets/mnist/mnist_tf.py index 5ac0e73..24a1fed 100644 --- a/nideep/datasets/mnist/mnist_tf.py +++ b/nideep/datasets/mnist/mnist_tf.py @@ -7,9 +7,16 @@ from tensorflow.python.framework import dtypes, random_seed from tensorflow.contrib.learn.python.learn.datasets.base import Datasets from tensorflow.contrib.learn.python.learn.datasets.mnist import DataSet, read_data_sets +import tensorflow as tf # imported for mocking from tensorflow.contrib.learn.python.learn.datasets.mnist import extract_images +def _int64_feature(value): + return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) + +def _bytes_feature(value): + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + class MNIST(object): ''' classdocs @@ -57,6 +64,60 @@ def read_data_sets(cls, reshape=False, # already reshaped seed=seed) return Datasets(train=train, validation=validation, test=ds.test) + + @classmethod + def to_tf_record(cls, fpath, one_hot=False): + mnist = MNIST.read_data_sets("MNIST_data", one_hot=one_hot, validation_size=5000) + with tf.python_io.TFRecordWriter(fpath) as writer: + for img, label in zip(mnist.train.images, mnist.train.labels): + img_raw = img.tobytes() + ## Convert the bytes back to image as follow: + # image = Image.frombytes('RGB', (224,224), img_raw) + # tl.visualize.frame(I=image, second=1, saveable=False, name='frame', fig_idx=1236) + ## Write the data into TF format + # image : Feature + BytesList + # label : Feature + Int64List or FloatList + # sentence : FeatureList + Int64List , see Google's im2txt example + if one_hot: + label_raw = label.astype(np.float32).tobytes() + example = tf.train.Example(features=tf.train.Features(feature={ + "label" : _bytes_feature(label_raw), + 'img_raw' : _bytes_feature(img_raw), + })) + else: + example = tf.train.Example(features=tf.train.Features(feature={ + "label" : _int64_feature(int(label)), + 'img_raw' : _bytes_feature(img_raw), + })) + writer.write(example.SerializeToString()) # Serialize To String + + @classmethod + def read_and_decode_ops(cls, fpath, one_hot=False): + # generate a queue with a given file name + filename_queue = tf.train.string_input_producer([fpath]) + reader = tf.TFRecordReader() + _, serialized_example = reader.read(filename_queue) # return the file and the name of file + if one_hot: + features = tf.parse_single_example(serialized_example, # see parse_single_sequence_example for sequence example + features={ + 'label': tf.FixedLenFeature([], tf.string), + 'img_raw' : tf.FixedLenFeature([], tf.string), + }) + label = tf.decode_raw(features['label'], tf.float32) +# label = tf.reshape(label, [10]) + label.set_shape((10,)) + else: + features = tf.parse_single_example(serialized_example, # see parse_single_sequence_example for sequence example + features={ + 'label': tf.FixedLenFeature([], tf.int64), + 'img_raw' : tf.FixedLenFeature([], tf.string), + }) + label = tf.cast(features['label'], tf.int32) + # You can do more image distortion here for training data + img = tf.decode_raw(features['img_raw'], tf.float32) +# img = tf.reshape(img, [28 * 28 * 1]) + img.set_shape((784,)) + return img, label def __init__(self, params): ''' diff --git a/nideep/datasets/mnist/test_mnist_tf.py b/nideep/datasets/mnist/test_mnist_tf.py index b0fce9b..3589fb1 100644 --- a/nideep/datasets/mnist/test_mnist_tf.py +++ b/nideep/datasets/mnist/test_mnist_tf.py @@ -20,7 +20,70 @@ def setup_class(self): @classmethod def teardown_class(self): shutil.rmtree(self.dir_tmp) - + +# def test_to_tf_record(self): +# import tensorflow as tf +# x = tf.placeholder(tf.float32, [None, 784]) +# name_w = 'W' +# with tf.variable_scope("var_scope", reuse=None): +# W = tf.get_variable(name_w, shape=[784, 10], +# initializer=tf.random_normal_initializer(stddev=0.1)) +# b = tf.get_variable('b', shape=[10], +# initializer=tf.constant_initializer(0.1)) +# logits = tf.matmul(x, W) + b +# y = tf.nn.softmax(logits) +# y_ = tf.placeholder(tf.float32, [None, 10]) +# cross_entropy = tf.reduce_mean( +# tf.nn.softmax_cross_entropy_with_logits(\ +# labels=y_, logits=logits)) +# train_step = tf.train.GradientDescentOptimizer(0.9).minimize(cross_entropy) +# init_op = tf.global_variables_initializer() +# one_hot = True +# mnist = MNIST.read_data_sets('MNIST_data', one_hot=True) +# MNIST.to_tf_record(os.path.join('MNIST_data', 'train.tfrecords'), +# one_hot=one_hot) +# +# +# img, label = MNIST.read_and_decode_ops(os.path.join('MNIST_data', 'train.tfrecords'), +# one_hot=one_hot) +## import tensorflow as tf +# batch_xs_op, batch_ys_op = tf.train.batch([img, label], +# batch_size=128, +# capacity=2000, +## min_after_dequeue=1000, +# num_threads=8 +# ) +# +# import matplotlib.pyplot as plt +# f, a = plt.subplots(3, 4, figsize=(10, 2)) +# with tf.Session() as sess: +# +# coord = tf.train.Coordinator() +# threads = tf.train.start_queue_runners(sess=sess, coord=coord) +## for i in range(3): +## val, l = sess.run([batch_xs_op, batch_ys_op]) +## print(val.shape, l) +## for j in range(4): +## a[i][j].imshow(np.reshape(val[j], (28, 28)), clim=(0.0, 1.0)) +# +## plt.show() +# sess.run(init_op) +# for itr in range(1000): +# batch_xs, batch_ys = sess.run([batch_xs_op, batch_ys_op]) +# _, c = sess.run([train_step, cross_entropy], +# feed_dict={x: batch_xs, y_: batch_ys}) +## print(c) +# print +# correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) +# accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) +# print(sess.run(accuracy, +# feed_dict={x: mnist.test.images, y_: mnist.test.labels})) +# print(sess.run(accuracy, +# feed_dict={x: mnist.validation.images, y_: mnist.validation.labels})) +# +# coord.request_stop() +# coord.join(threads) + @patch('nideep.datasets.mnist.mnist_tf.extract_images') def test_read(self, mock_extract): mock_extract.return_value = np.empty_like((100000, 28, 28)) From d41793a3815039dde2436ff00c77fd92dce4148f Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Sun, 29 Oct 2017 15:27:46 +0100 Subject: [PATCH 13/24] tf records with rotations --- nideep/datasets/mnist/mnist_tf.py | 234 +++++++++++++++++---- nideep/datasets/mnist/test_mnist_tf.py | 269 ++++++++++++++----------- 2 files changed, 340 insertions(+), 163 deletions(-) diff --git a/nideep/datasets/mnist/mnist_tf.py b/nideep/datasets/mnist/mnist_tf.py index 24a1fed..df7f420 100644 --- a/nideep/datasets/mnist/mnist_tf.py +++ b/nideep/datasets/mnist/mnist_tf.py @@ -3,7 +3,9 @@ @author: kashefy ''' +import os import numpy as np +import skimage.transform as transform from tensorflow.python.framework import dtypes, random_seed from tensorflow.contrib.learn.python.learn.datasets.base import Datasets from tensorflow.contrib.learn.python.learn.datasets.mnist import DataSet, read_data_sets @@ -34,7 +36,6 @@ def read_data_sets(cls, seed1, seed2 = random_seed.get_seed(seed) # If op level seed is not set, use whatever graph level seed is returned np.random.seed(seed1 if seed is None else seed2) - ds = read_data_sets(\ train_dir, fake_data=fake_data, @@ -66,58 +67,211 @@ def read_data_sets(cls, return Datasets(train=train, validation=validation, test=ds.test) @classmethod - def to_tf_record(cls, fpath, one_hot=False): - mnist = MNIST.read_data_sets("MNIST_data", one_hot=one_hot, validation_size=5000) - with tf.python_io.TFRecordWriter(fpath) as writer: - for img, label in zip(mnist.train.images, mnist.train.labels): - img_raw = img.tobytes() - ## Convert the bytes back to image as follow: - # image = Image.frombytes('RGB', (224,224), img_raw) - # tl.visualize.frame(I=image, second=1, saveable=False, name='frame', fig_idx=1236) - ## Write the data into TF format - # image : Feature + BytesList - # label : Feature + Int64List or FloatList - # sentence : FeatureList + Int64List , see Google's im2txt example - if one_hot: - label_raw = label.astype(np.float32).tobytes() - example = tf.train.Example(features=tf.train.Features(feature={ - "label" : _bytes_feature(label_raw), - 'img_raw' : _bytes_feature(img_raw), - })) - else: - example = tf.train.Example(features=tf.train.Features(feature={ - "label" : _int64_feature(int(label)), - 'img_raw' : _bytes_feature(img_raw), - })) - writer.write(example.SerializeToString()) # Serialize To String - + def to_tf_record(cls, + fpath, + data_dir, + fake_data=False, + one_hot=False, + dtype=dtypes.float32, + reshape=True, + validation_size=5000, + seed=None, + orientations=np.linspace(-90, 90, 180/(12+1), endpoint=True).tolist(), + ): + """ + Get dataset with orientations stored in tf.records + """ + mnist = MNIST.read_data_sets( + data_dir, + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=False, + validation_size=validation_size, + seed=seed, + ) + fname0, ext = os.path.splitext(fpath) + fpath_list_out = [] + for phase in ['train', 'validation', 'test']: + if fname0.endswith(phase): + fpath_phase = fpath + else: + fpath_phase = ''.join([fname0, '_', phase, ext]) + if not os.path.isfile(fpath_phase): + # Skip if it already exists + with tf.python_io.TFRecordWriter(fpath_phase) as writer: + split = getattr(mnist, 'train') + for img, label in zip(split.images, split.labels): + if img.ndim < 2: + raise AttributeError("Rotation need 2-dims of images resolved, found shape %s" % img.shape) + img = np.expand_dims(img, axis=0) + imgs_orient_all, labels_orient_all = \ + MNIST.rotate(img, orientations, one_hot) + for img_orient, label_orient in zip(imgs_orient_all, labels_orient_all): + img_raw = img_orient.tobytes() + ## Convert the bytes back to image as follow: + # image = Image.frombytes('RGB', (224,224), img_raw) + # tl.visualize.frame(I=image, second=1, saveable=False, name='frame', fig_idx=1236) + ## Write the data into TF format + # image : Feature + BytesList + # label : Feature + Int64List or FloatList + # sentence : FeatureList + Int64List , see Google's im2txt example + if one_hot: + label_raw = label.astype(np.float32).tobytes() + label_orient_raw = label_orient.astype(np.float32).tobytes() + example = tf.train.Example(features=tf.train.Features(feature={ + "label" : _bytes_feature(label_raw), + "label_orient" : _bytes_feature(label_orient_raw), + 'img_raw' : _bytes_feature(img_raw), + })) + else: + example = tf.train.Example(features=tf.train.Features(feature={ + "label" : _int64_feature(int(label)), + "label_orient" : _int64_feature(int(label_orient)), + 'img_raw' : _bytes_feature(img_raw), + })) + writer.write(example.SerializeToString()) # Serialize To String + fpath_list_out.append(fpath_phase) + return fpath_list_out + @classmethod - def read_and_decode_ops(cls, fpath, one_hot=False): + def read_and_decode_ops(cls, fpath, one_hot=False, + img_shape=(784,), # or (28, 28, 1) + num_classes=10, + num_orientations=0, + ): # generate a queue with a given file name filename_queue = tf.train.string_input_producer([fpath]) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) # return the file and the name of file + feat_dict = { + 'label' : tf.FixedLenFeature([], [tf.int64, tf.string][one_hot]), + 'img_raw' : tf.FixedLenFeature([], tf.string), + } + if num_orientations > 0: + feat_dict['label_orient'] = tf.FixedLenFeature([], [tf.int64, tf.string][one_hot]) + features = tf.parse_single_example(serialized_example, # see parse_single_sequence_example for sequence example + features=feat_dict) if one_hot: - features = tf.parse_single_example(serialized_example, # see parse_single_sequence_example for sequence example - features={ - 'label': tf.FixedLenFeature([], tf.string), - 'img_raw' : tf.FixedLenFeature([], tf.string), - }) label = tf.decode_raw(features['label'], tf.float32) -# label = tf.reshape(label, [10]) - label.set_shape((10,)) +# label = tf.reshape(label, [10]) + label.set_shape((num_classes,)) + if num_orientations > 0: + label_orient = tf.decode_raw(features['label_orient'], tf.float32) + # label = tf.reshape(label, [10]) + label_orient.set_shape((num_orientations,)) else: - features = tf.parse_single_example(serialized_example, # see parse_single_sequence_example for sequence example - features={ - 'label': tf.FixedLenFeature([], tf.int64), - 'img_raw' : tf.FixedLenFeature([], tf.string), - }) label = tf.cast(features['label'], tf.int32) + if num_orientations > 0: + label_orient = tf.cast(features['label_orient'], tf.int32) # You can do more image distortion here for training data img = tf.decode_raw(features['img_raw'], tf.float32) # img = tf.reshape(img, [28 * 28 * 1]) img.set_shape((784,)) - return img, label + if num_orientations > 0: + return img, label, label_orient + else: + return img, label + + @staticmethod + def rotate(imgs, orientations, one_hot): + num_examples_in = len(imgs) + labels_orient_all = None + imgs_orient_all = None + for angle_idx, angle in enumerate(orientations): + imgs_orient = np.zeros_like(imgs) + for img_idx, img in enumerate(imgs): + img_rot = transform.rotate(img, angle, + resize=False, center=None, + order=1, + mode='symmetric', cval=0, + clip=True, preserve_range=False) + imgs_orient[img_idx] = img_rot + if one_hot: + labels_orient = np.zeros((num_examples_in, len(orientations))) + labels_orient[:, angle_idx] = 1 + else: + labels_orient = np.zeros((num_examples_in,)) + angle_idx + if labels_orient_all is None: + labels_orient_all = labels_orient + imgs_orient_all = imgs_orient + else: + labels_orient_all = np.vstack((labels_orient_all, labels_orient)) + imgs_orient_all = np.vstack((imgs_orient_all, imgs_orient)) + return imgs_orient_all, labels_orient_all + + @classmethod + def read_data_sets_orient(cls, + train_dir, + fake_data=False, + one_hot=False, + dtype=dtypes.float32, + reshape=True, + validation_size=5000, + seed=None, + orientations=np.linspace(-90, 90, 180/(12+1), endpoint=True).tolist()): + + seed1, seed2 = random_seed.get_seed(seed) + # If op level seed is not set, use whatever graph level seed is returned + np.random.seed(seed1 if seed is None else seed2) + ds = read_data_sets(\ + train_dir, + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=False, # preseve image dimensions for rotation + validation_size=0, + seed=seed) + imgs_orient_all, labels_orient_all = MNIST.rotate(ds.train.images, orientations, one_hot) + print(labels_orient_all.shape, imgs_orient_all.shape) + # An image and all its orientation either fall into train or validation + perm0 = np.arange(ds.train.num_examples) + np.random.shuffle(perm0) + train_idxs0 = perm0[validation_size:] + val_idxs0 = perm0[:validation_size] + train_idxs = train_idxs0 + val_idxs = val_idxs0 + for orient_idx in range(len(orientations)): + train_idxs_orient = train_idxs0 + (orient_idx+1)*ds.train.num_examples + val_idxs_orient = val_idxs0 + (orient_idx+1)*ds.train.num_examples + train_idxs = np.vstack((train_idxs, train_idxs_orient)) + val_idxs = np.vstack((val_idxs, val_idxs_orient)) + np.random.shuffle(train_idxs) + np.random.shuffle(val_idxs) + train = DataSet(\ + np.multiply(imgs_orient_all[train_idxs], [1., 255.][dtype == dtypes.float32]), # will rescale to [0,1] inside + labels_orient_all[train_idxs], + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=reshape, # reshape here for the first time + seed=seed) + validation = DataSet(\ + np.multiply(imgs_orient_all[val_idxs], [1., 255.][dtype == dtypes.float32]), # will rescale to [0,1] inside + labels_orient_all[val_idxs], + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=reshape, # reshape here for the first time + seed=seed) + validation = DataSet(\ + np.multiply(imgs_orient_all[val_idxs], [1., 255.][dtype == dtypes.float32]), # will rescale to [0,1] inside + labels_orient_all[val_idxs], + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=reshape, # reshape here for the first time + seed=seed) + labels_orient_all, imgs_orient_all = MNIST.rotate(ds.test.images, orientations, one_hot) + test = DataSet(\ + np.multiply(imgs_orient_all, [1., 255.][dtype == dtypes.float32]), # will rescale to [0,1] inside + labels_orient_all, + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=reshape, # reshape here for the first time + seed=seed) + return Datasets(train=train, validation=validation, test=ds.test) def __init__(self, params): ''' diff --git a/nideep/datasets/mnist/test_mnist_tf.py b/nideep/datasets/mnist/test_mnist_tf.py index 3589fb1..ef6d3b2 100644 --- a/nideep/datasets/mnist/test_mnist_tf.py +++ b/nideep/datasets/mnist/test_mnist_tf.py @@ -21,131 +21,154 @@ def setup_class(self): def teardown_class(self): shutil.rmtree(self.dir_tmp) -# def test_to_tf_record(self): -# import tensorflow as tf -# x = tf.placeholder(tf.float32, [None, 784]) -# name_w = 'W' -# with tf.variable_scope("var_scope", reuse=None): -# W = tf.get_variable(name_w, shape=[784, 10], -# initializer=tf.random_normal_initializer(stddev=0.1)) -# b = tf.get_variable('b', shape=[10], -# initializer=tf.constant_initializer(0.1)) -# logits = tf.matmul(x, W) + b -# y = tf.nn.softmax(logits) -# y_ = tf.placeholder(tf.float32, [None, 10]) -# cross_entropy = tf.reduce_mean( -# tf.nn.softmax_cross_entropy_with_logits(\ -# labels=y_, logits=logits)) -# train_step = tf.train.GradientDescentOptimizer(0.9).minimize(cross_entropy) -# init_op = tf.global_variables_initializer() -# one_hot = True + def test_to_tf_record(self): + import tensorflow as tf + x = tf.placeholder(tf.float32, [None, 784]) + name_w = 'W' + with tf.variable_scope("var_scope", reuse=None): + W = tf.get_variable(name_w, shape=[784, 10], + initializer=tf.random_normal_initializer(stddev=0.1)) + b = tf.get_variable('b', shape=[10], + initializer=tf.constant_initializer(0.1)) + logits = tf.matmul(x, W) + b + y = tf.nn.softmax(logits) + y_ = tf.placeholder(tf.float32, [None, 10]) + cross_entropy = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(\ + labels=y_, logits=logits)) + train_step = tf.train.GradientDescentOptimizer(0.9).minimize(cross_entropy) + init_op = tf.global_variables_initializer() + one_hot = True + shuffle = True + orientations = range(-60,75,15) + print(orientations) # mnist = MNIST.read_data_sets('MNIST_data', one_hot=True) -# MNIST.to_tf_record(os.path.join('MNIST_data', 'train.tfrecords'), -# one_hot=one_hot) -# -# -# img, label = MNIST.read_and_decode_ops(os.path.join('MNIST_data', 'train.tfrecords'), -# one_hot=one_hot) -## import tensorflow as tf -# batch_xs_op, batch_ys_op = tf.train.batch([img, label], -# batch_size=128, -# capacity=2000, -## min_after_dequeue=1000, -# num_threads=8 -# ) -# -# import matplotlib.pyplot as plt -# f, a = plt.subplots(3, 4, figsize=(10, 2)) -# with tf.Session() as sess: -# -# coord = tf.train.Coordinator() -# threads = tf.train.start_queue_runners(sess=sess, coord=coord) -## for i in range(3): -## val, l = sess.run([batch_xs_op, batch_ys_op]) -## print(val.shape, l) -## for j in range(4): -## a[i][j].imshow(np.reshape(val[j], (28, 28)), clim=(0.0, 1.0)) -# -## plt.show() -# sess.run(init_op) -# for itr in range(1000): -# batch_xs, batch_ys = sess.run([batch_xs_op, batch_ys_op]) -# _, c = sess.run([train_step, cross_entropy], -# feed_dict={x: batch_xs, y_: batch_ys}) -## print(c) -# print -# correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) -# accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) -# print(sess.run(accuracy, -# feed_dict={x: mnist.test.images, y_: mnist.test.labels})) -# print(sess.run(accuracy, -# feed_dict={x: mnist.validation.images, y_: mnist.validation.labels})) -# -# coord.request_stop() -# coord.join(threads) - - @patch('nideep.datasets.mnist.mnist_tf.extract_images') - def test_read(self, mock_extract): - mock_extract.return_value = np.empty_like((100000, 28, 28)) - d = MNIST.read_data_sets(self.dir_tmp) - assert_equals(d.train.num_examples, 55000) - assert_equals(d.validation.num_examples, 5000) - assert_equals(d.test.num_examples, 10000) - - @patch('nideep.datasets.mnist.mnist_tf.extract_images') - def test_validation_size(self, mock_extract): - mock_extract.return_value = np.empty_like((100000, 28, 28)) - for sz in range(5000, 55000, 5000): - yield self.check_validation_size, sz + fpath_list = MNIST.to_tf_record(os.path.join('MNIST_data', 'train.tfrecords'), + 'MNIST_data', + one_hot=one_hot, + orientations=orientations) + print fpath_list + return - def check_validation_size(self, sz): - d = MNIST.read_data_sets(self.dir_tmp, validation_size=sz) - assert_equals(d.train.num_examples, 60000-sz) - assert_equals(d.validation.num_examples, sz) - assert_equals(d.test.num_examples, 10000) + img, label, label_orient = MNIST.read_and_decode_ops(\ + os.path.join('MNIST_data', 'train.tfrecords'), + one_hot=one_hot, + num_orientations=len(orientations)) +# import tensorflow as tf + if shuffle: + batch_xs_op, batch_ys_op, batch_os_op = tf.train.shuffle_batch([img, label, label_orient], + batch_size=64, + capacity=2000, + min_after_dequeue=1000, + num_threads=8 + ) + else: + batch_xs_op, batch_ys_op, batch_os_op = tf.train.batch([img, label, label_orient], + batch_size=64, + capacity=2000, + # min_after_dequeue=1000, + num_threads=8 + ) + + import matplotlib.pyplot as plt + f, a = plt.subplots(3, 12, figsize=(10, 2)) + with tf.Session() as sess: + + coord = tf.train.Coordinator() + threads = tf.train.start_queue_runners(sess=sess, coord=coord) + for i in range(3): + val, l, orient = sess.run([batch_xs_op, batch_ys_op, batch_os_op]) + print(val.shape, l) + for j in range(12): + a[i][j].imshow(np.reshape(val[j], (28, 28)), clim=(0.0, 1.0)) + if one_hot: + a[i][j].set_title('%d at %.2f deg' % (np.argmax(l[j]), orientations[np.argmax(orient[j])])) + else: + a[i][j].set_title('%d at %.2f deg' % (l[j], orientations[orient[j]])) - def test_shuffle(self): - d1 = MNIST.read_data_sets(self.dir_tmp) - d2 = MNIST.read_data_sets(self.dir_tmp) - assert_false(np.array_equal(d1.train.images, d2.train.images)) - assert_false(np.array_equal(d1.train.labels, d2.train.labels)) - assert_false(np.array_equal(d1.validation.images, d2.validation.images)) - assert_false(np.array_equal(d1.validation.labels, d2.validation.labels)) - assert_true(np.array_equal(d1.test.images, d2.test.images)) - assert_true(np.array_equal(d1.test.labels, d2.test.labels)) + plt.show() + return + sess.run(init_op) + for itr in range(1000): + batch_xs, batch_ys = sess.run([batch_xs_op, batch_ys_op]) + _, c = sess.run([train_step, cross_entropy], + feed_dict={x: batch_xs, y_: batch_ys}) +# print(c) + correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) + accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) + print(sess.run(accuracy, + feed_dict={x: mnist.test.images, y_: mnist.test.labels})) + print(sess.run(accuracy, + feed_dict={x: mnist.validation.images, y_: mnist.validation.labels})) + + coord.request_stop() + coord.join(threads) - def test_seed(self): - d1 = MNIST.read_data_sets(self.dir_tmp, seed=123) - d2 = MNIST.read_data_sets(self.dir_tmp, seed=999) - d3 = MNIST.read_data_sets(self.dir_tmp, seed=123) - assert_false(np.array_equal(d1.train.images, d2.train.images)) - assert_false(np.array_equal(d1.train.labels, d2.train.labels)) - assert_false(np.array_equal(d1.validation.images, d2.validation.images)) - assert_false(np.array_equal(d1.validation.labels, d2.validation.labels)) - assert_true(np.array_equal(d1.test.images, d2.test.images)) - assert_true(np.array_equal(d1.test.labels, d2.test.labels)) - # same seed - assert_true(np.array_equal(d1.train.images, d3.train.images)) - assert_true(np.array_equal(d1.train.labels, d3.train.labels)) - assert_true(np.array_equal(d1.validation.images, d3.validation.images)) - assert_true(np.array_equal(d1.validation.labels, d3.validation.labels)) - assert_true(np.array_equal(d1.test.images, d3.test.images)) - assert_true(np.array_equal(d1.test.labels, d3.test.labels)) - - def test_data_scale(self): - from tensorflow.examples.tutorials.mnist import input_data - d0 = input_data.read_data_sets(self.dir_tmp, one_hot=True) - d1 = MNIST.read_data_sets(self.dir_tmp, one_hot=True) - assert_equals(d0.train.images.min(), 0) - assert_equals(d0.train.images.max(), 1) - assert_equals(d0.train.images.min(), d1.train.images.min()) - assert_equals(d0.train.images.max(), d1.train.images.max()) - assert_equals(d0.validation.images.min(), 0) - assert_equals(d0.validation.images.max(), 1) - assert_equals(d0.validation.images.min(), d1.validation.images.min()) - assert_equals(d0.validation.images.max(), d1.validation.images.max()) - assert_equals(d0.test.images.min(), 0) - assert_equals(d0.test.images.max(), 1) - assert_equals(d0.test.images.min(), d1.test.images.min()) - assert_equals(d0.test.images.max(), d1.test.images.max()) +# def test_orient(self): +# d1 = MNIST.read_data_sets_orient(self.dir_tmp, seed=123, orientations=range(-30,60,30)) + +# @patch('nideep.datasets.mnist.mnist_tf.extract_images') +# def test_read(self, mock_extract): +# mock_extract.return_value = np.empty_like((100000, 28, 28)) +# d = MNIST.read_data_sets(self.dir_tmp) +# assert_equals(d.train.num_examples, 55000) +# assert_equals(d.validation.num_examples, 5000) +# assert_equals(d.test.num_examples, 10000) +# +# @patch('nideep.datasets.mnist.mnist_tf.extract_images') +# def test_validation_size(self, mock_extract): +# mock_extract.return_value = np.empty_like((100000, 28, 28)) +# for sz in range(5000, 55000, 5000): +# yield self.check_validation_size, sz +# +# def check_validation_size(self, sz): +# d = MNIST.read_data_sets(self.dir_tmp, validation_size=sz) +# assert_equals(d.train.num_examples, 60000-sz) +# assert_equals(d.validation.num_examples, sz) +# assert_equals(d.test.num_examples, 10000) +# +# def test_shuffle(self): +# d1 = MNIST.read_data_sets(self.dir_tmp) +# d2 = MNIST.read_data_sets(self.dir_tmp) +# assert_false(np.array_equal(d1.train.images, d2.train.images)) +# assert_false(np.array_equal(d1.train.labels, d2.train.labels)) +# assert_false(np.array_equal(d1.validation.images, d2.validation.images)) +# assert_false(np.array_equal(d1.validation.labels, d2.validation.labels)) +# assert_true(np.array_equal(d1.test.images, d2.test.images)) +# assert_true(np.array_equal(d1.test.labels, d2.test.labels)) +# +# def test_seed(self): +# d1 = MNIST.read_data_sets(self.dir_tmp, seed=123) +# d2 = MNIST.read_data_sets(self.dir_tmp, seed=999) +# d3 = MNIST.read_data_sets(self.dir_tmp, seed=123) +# assert_false(np.array_equal(d1.train.images, d2.train.images)) +# assert_false(np.array_equal(d1.train.labels, d2.train.labels)) +# assert_false(np.array_equal(d1.validation.images, d2.validation.images)) +# assert_false(np.array_equal(d1.validation.labels, d2.validation.labels)) +# assert_true(np.array_equal(d1.test.images, d2.test.images)) +# assert_true(np.array_equal(d1.test.labels, d2.test.labels)) +# # same seed +# assert_true(np.array_equal(d1.train.images, d3.train.images)) +# assert_true(np.array_equal(d1.train.labels, d3.train.labels)) +# assert_true(np.array_equal(d1.validation.images, d3.validation.images)) +# assert_true(np.array_equal(d1.validation.labels, d3.validation.labels)) +# assert_true(np.array_equal(d1.test.images, d3.test.images)) +# assert_true(np.array_equal(d1.test.labels, d3.test.labels)) +# +# def test_data_scale(self): +# from tensorflow.examples.tutorials.mnist import input_data +# d0 = input_data.read_data_sets(self.dir_tmp, one_hot=True) +# d1 = MNIST.read_data_sets(self.dir_tmp, one_hot=True) +# assert_equals(d0.train.images.min(), 0) +# assert_equals(d0.train.images.max(), 1) +# assert_equals(d0.train.images.min(), d1.train.images.min()) +# assert_equals(d0.train.images.max(), d1.train.images.max()) +# assert_equals(d0.validation.images.min(), 0) +# assert_equals(d0.validation.images.max(), 1) +# assert_equals(d0.validation.images.min(), d1.validation.images.min()) +# assert_equals(d0.validation.images.max(), d1.validation.images.max()) +# assert_equals(d0.test.images.min(), 0) +# assert_equals(d0.test.images.max(), 1) +# assert_equals(d0.test.images.min(), d1.test.images.min()) +# assert_equals(d0.test.images.max(), d1.test.images.max()) \ No newline at end of file From 10613b978cdb75259634f4310ab315dbda72d5e6 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Mon, 30 Oct 2017 16:59:05 +0100 Subject: [PATCH 14/24] tf records cont'd --- nideep/datasets/mnist/mnist_tf.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/nideep/datasets/mnist/mnist_tf.py b/nideep/datasets/mnist/mnist_tf.py index df7f420..3031dc5 100644 --- a/nideep/datasets/mnist/mnist_tf.py +++ b/nideep/datasets/mnist/mnist_tf.py @@ -4,6 +4,7 @@ @author: kashefy ''' import os +from collections import namedtuple import numpy as np import skimage.transform as transform from tensorflow.python.framework import dtypes, random_seed @@ -12,6 +13,7 @@ import tensorflow as tf # imported for mocking from tensorflow.contrib.learn.python.learn.datasets.mnist import extract_images +from _ast import Num def _int64_feature(value): return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) @@ -91,8 +93,15 @@ def to_tf_record(cls, seed=seed, ) fname0, ext = os.path.splitext(fpath) - fpath_list_out = [] - for phase in ['train', 'validation', 'test']: + DatasetTFRecords = namedtuple('DatasetTFRecords', + ['num_examples', + 'path', + 'one_hot', + 'orientations', + 'phase']) + dsets = Datasets(train=None, validation=None, test=None) + for phase in dsets._fields:#['train', 'validation', 'test']: + num_examples = 0 if fname0.endswith(phase): fpath_phase = fpath else: @@ -100,10 +109,10 @@ def to_tf_record(cls, if not os.path.isfile(fpath_phase): # Skip if it already exists with tf.python_io.TFRecordWriter(fpath_phase) as writer: - split = getattr(mnist, 'train') + split = getattr(mnist, phase) for img, label in zip(split.images, split.labels): if img.ndim < 2: - raise AttributeError("Rotation need 2-dims of images resolved, found shape %s" % img.shape) + raise AttributeError("Rotation needs both height and width images resolved, found shape %s" % img.shape) img = np.expand_dims(img, axis=0) imgs_orient_all, labels_orient_all = \ MNIST.rotate(img, orientations, one_hot) @@ -131,8 +140,14 @@ def to_tf_record(cls, 'img_raw' : _bytes_feature(img_raw), })) writer.write(example.SerializeToString()) # Serialize To String - fpath_list_out.append(fpath_phase) - return fpath_list_out + num_examples += 1 + dsets.train = DatasetTFRecords( + num_examples=num_examples, + path=fpath_phase, + one_hot=one_hot, + orientations=orientations, + phase=phase) + return dsets @classmethod def read_and_decode_ops(cls, fpath, one_hot=False, From b1a3862fde58a3cd8a1c9229deb2892989234283 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 31 Oct 2017 20:56:04 +0100 Subject: [PATCH 15/24] disguise Datasets --- nideep/datasets/mnist/mnist_tf.py | 36 ++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/nideep/datasets/mnist/mnist_tf.py b/nideep/datasets/mnist/mnist_tf.py index 3031dc5..bfcaef5 100644 --- a/nideep/datasets/mnist/mnist_tf.py +++ b/nideep/datasets/mnist/mnist_tf.py @@ -98,18 +98,23 @@ def to_tf_record(cls, 'path', 'one_hot', 'orientations', - 'phase']) - dsets = Datasets(train=None, validation=None, test=None) - for phase in dsets._fields:#['train', 'validation', 'test']: + 'phase', + 'images', + 'labels']) + dsets = {} + for phase in Datasets._fields:#['train', 'validation', 'test']: + split = getattr(mnist, phase) num_examples = 0 if fname0.endswith(phase): fpath_phase = fpath else: fpath_phase = ''.join([fname0, '_', phase, ext]) - if not os.path.isfile(fpath_phase): + if os.path.isfile(fpath_phase): # Skip if it already exists + print(fpath_phase) + num_examples = sum(1 for _ in tf.python_io.tf_record_iterator(fpath_phase)) + else: with tf.python_io.TFRecordWriter(fpath_phase) as writer: - split = getattr(mnist, phase) for img, label in zip(split.images, split.labels): if img.ndim < 2: raise AttributeError("Rotation needs both height and width images resolved, found shape %s" % img.shape) @@ -141,13 +146,20 @@ def to_tf_record(cls, })) writer.write(example.SerializeToString()) # Serialize To String num_examples += 1 - dsets.train = DatasetTFRecords( + dsets[phase] = DatasetTFRecords( num_examples=num_examples, path=fpath_phase, one_hot=one_hot, orientations=orientations, - phase=phase) - return dsets + phase=phase, + images=np.empty((num_examples,) + split.images.shape[1:], + dtype=split.images.dtype), + labels=np.empty((num_examples,) + split.labels.shape[1:], + dtype=split.labels.dtype), + ) + return Datasets(train=dsets['train'], + validation=dsets['validation'], + test=dsets['test']) @classmethod def read_and_decode_ops(cls, fpath, one_hot=False, @@ -189,7 +201,11 @@ def read_and_decode_ops(cls, fpath, one_hot=False, return img, label @staticmethod - def rotate(imgs, orientations, one_hot): + def rotate(imgs, + orientations, + one_hot, + mode='symmetric', + cval=0): num_examples_in = len(imgs) labels_orient_all = None imgs_orient_all = None @@ -199,7 +215,7 @@ def rotate(imgs, orientations, one_hot): img_rot = transform.rotate(img, angle, resize=False, center=None, order=1, - mode='symmetric', cval=0, + mode=mode, cval=cval, clip=True, preserve_range=False) imgs_orient[img_idx] = img_rot if one_hot: From a9cf2d6ddea45096c82d45c9cc954122f96ccfb9 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Fri, 3 Nov 2017 21:59:18 +0100 Subject: [PATCH 16/24] remove print msg --- nideep/datasets/mnist/mnist_tf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nideep/datasets/mnist/mnist_tf.py b/nideep/datasets/mnist/mnist_tf.py index bfcaef5..0b581d1 100644 --- a/nideep/datasets/mnist/mnist_tf.py +++ b/nideep/datasets/mnist/mnist_tf.py @@ -111,7 +111,6 @@ def to_tf_record(cls, fpath_phase = ''.join([fname0, '_', phase, ext]) if os.path.isfile(fpath_phase): # Skip if it already exists - print(fpath_phase) num_examples = sum(1 for _ in tf.python_io.tf_record_iterator(fpath_phase)) else: with tf.python_io.TFRecordWriter(fpath_phase) as writer: From dc9f071d91b43f4c6abaa1478557d98823ffa029 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 7 Nov 2017 22:13:52 +0100 Subject: [PATCH 17/24] remove unused import --- nideep/datasets/mnist/mnist_tf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nideep/datasets/mnist/mnist_tf.py b/nideep/datasets/mnist/mnist_tf.py index 0b581d1..fd5a837 100644 --- a/nideep/datasets/mnist/mnist_tf.py +++ b/nideep/datasets/mnist/mnist_tf.py @@ -13,7 +13,6 @@ import tensorflow as tf # imported for mocking from tensorflow.contrib.learn.python.learn.datasets.mnist import extract_images -from _ast import Num def _int64_feature(value): return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) From 01d064cb3a19628a04526c81c7f3d17e1284b6e3 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Tue, 7 Nov 2017 22:14:58 +0100 Subject: [PATCH 18/24] WIP: second task --- nideep/nets/mlp_tf.py | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/nideep/nets/mlp_tf.py b/nideep/nets/mlp_tf.py index 12ad23a..0d8b83d 100644 --- a/nideep/nets/mlp_tf.py +++ b/nideep/nets/mlp_tf.py @@ -5,7 +5,6 @@ ''' import tensorflow as tf from nideep.nets.abstract_net_tf import AbstractNetTF -from networkx.algorithms.shortest_paths.unweighted import predecessor class MLP(AbstractNetTF): ''' @@ -22,6 +21,12 @@ def _init_learning_params_scoped(self): [input_dim, dim], initializer=self._init_weight_op() ) + if idx == len(self.n_nodes)-1: + fc_name_w = 'fc-%d-aux/w' % idx + self.w[fc_name_w] = tf.get_variable(fc_name_w, + [input_dim, 9], + initializer=self._init_weight_op() + ) self._init_bias_vars(bias_value=0.1) def _fc(self, x): @@ -38,6 +43,18 @@ def _fc(self, x): else: self._y_logits = fc_op self.p = tf.nn.softmax(fc_op, name='a-%d' % idx) + + fc_name_w = 'fc-%d-aux/w' % idx + fc_name_b = 'fc-%d-aux/b' % idx + if len(self.n_nodes) == 1: + in_op = x + else: + in_op = a + fc_op = tf.add(tf.matmul(in_op, self.w[fc_name_w]), + self.b[fc_name_b], + name='fc-%d' % idx) + self._y_logits_aux = fc_op + self.p_aux = tf.nn.softmax(fc_op, name='a-%d-aux' % idx) return self.p, self._y_logits def _init_ops(self): @@ -46,11 +63,25 @@ def _init_ops(self): def cost(self, y_true, name=None): with tf.name_scope(self.name_scope): - self._cost_op = tf.reduce_mean( + c = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(\ labels=y_true, logits=self._y_logits), name=name) - return self._cost_op + self._cost_ops.append(c) + return c + + def cost_o(self, y_true, name=None): + with tf.name_scope(self.name_scope): + c = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(\ + labels=y_true, logits=self._y_logits_aux), + name=name) + self._cost_ops.append(c) + return c + + @property + def cost_ops(self): + return self._cost_ops def __init__(self, params): ''' @@ -61,5 +92,5 @@ def __init__(self, params): self.n_nodes = params['n_nodes'] # 1st layer num features super(MLP, self).__init__(params) - self._cost_op = None + self._cost_ops = [] self._y_logits = None \ No newline at end of file From 1188f80e545b840398d49afb2e9c2200fc9ecdc1 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Sat, 30 Dec 2017 10:34:49 +0100 Subject: [PATCH 19/24] WIP: second task, branching, logging --- nideep/nets/abstract_net.py | 5 +++++ nideep/nets/mlp_tf.py | 22 ++++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/nideep/nets/abstract_net.py b/nideep/nets/abstract_net.py index 31c6c60..a02b976 100644 --- a/nideep/nets/abstract_net.py +++ b/nideep/nets/abstract_net.py @@ -5,6 +5,7 @@ ''' from __future__ import division, print_function, absolute_import from abc import ABCMeta, abstractmethod +import logging class AbstractNet(object): ''' @@ -81,6 +82,10 @@ def __init__(self, params): ''' Constructor ''' + logger_name = self.__class__.__name__ + if params.get('logger_name', None) is not None: + logger_name = '.'.join([params.get('logger_name', None), logger_name]) + self.logger = logging.getLogger(logger_name) self._x = None self._p = None self.prefix = params.get('prefix', '') diff --git a/nideep/nets/mlp_tf.py b/nideep/nets/mlp_tf.py index 0d8b83d..7e353c5 100644 --- a/nideep/nets/mlp_tf.py +++ b/nideep/nets/mlp_tf.py @@ -21,16 +21,17 @@ def _init_learning_params_scoped(self): [input_dim, dim], initializer=self._init_weight_op() ) - if idx == len(self.n_nodes)-1: + if idx == self.branch: fc_name_w = 'fc-%d-aux/w' % idx self.w[fc_name_w] = tf.get_variable(fc_name_w, - [input_dim, 9], + [dim, 9], initializer=self._init_weight_op() ) self._init_bias_vars(bias_value=0.1) def _fc(self, x): in_op = x + branch_op = x for idx in xrange(len(self.n_nodes)): fc_name_w = 'fc-%d/w' % idx fc_name_b = 'fc-%d/b' % idx @@ -43,16 +44,15 @@ def _fc(self, x): else: self._y_logits = fc_op self.p = tf.nn.softmax(fc_op, name='a-%d' % idx) - + if idx == self.branch: + branch_op = a + idx = self.branch fc_name_w = 'fc-%d-aux/w' % idx fc_name_b = 'fc-%d-aux/b' % idx - if len(self.n_nodes) == 1: - in_op = x - else: - in_op = a + in_op = branch_op fc_op = tf.add(tf.matmul(in_op, self.w[fc_name_w]), self.b[fc_name_b], - name='fc-%d' % idx) + name='fc-%d-aux' % idx) self._y_logits_aux = fc_op self.p_aux = tf.nn.softmax(fc_op, name='a-%d-aux' % idx) return self.p, self._y_logits @@ -70,7 +70,7 @@ def cost(self, y_true, name=None): self._cost_ops.append(c) return c - def cost_o(self, y_true, name=None): + def cost_aux(self, y_true, name=None): with tf.name_scope(self.name_scope): c = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(\ @@ -93,4 +93,6 @@ def __init__(self, params): super(MLP, self).__init__(params) self._cost_ops = [] - self._y_logits = None \ No newline at end of file + self._y_logits = None + self.branch = params.get('branch', len(self.n_nodes)-1) + self.logger.debug("Branch aux. task from layer: %d" % self.branch) \ No newline at end of file From 8ae8cf9140898feb1fe8be023acf57239217538a Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Fri, 9 Mar 2018 12:02:56 +0100 Subject: [PATCH 20/24] fix input dim at branching --- nideep/nets/mlp_tf.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nideep/nets/mlp_tf.py b/nideep/nets/mlp_tf.py index 7e353c5..6403404 100644 --- a/nideep/nets/mlp_tf.py +++ b/nideep/nets/mlp_tf.py @@ -24,20 +24,20 @@ def _init_learning_params_scoped(self): if idx == self.branch: fc_name_w = 'fc-%d-aux/w' % idx self.w[fc_name_w] = tf.get_variable(fc_name_w, - [dim, 9], + [input_dim, 9], initializer=self._init_weight_op() ) self._init_bias_vars(bias_value=0.1) def _fc(self, x): - in_op = x + in_op = a = x branch_op = x for idx in xrange(len(self.n_nodes)): fc_name_w = 'fc-%d/w' % idx fc_name_b = 'fc-%d/b' % idx fc_op = tf.add(tf.matmul(in_op, self.w[fc_name_w]), - self.b[fc_name_b], - name='fc-%d' % idx) + self.b[fc_name_b], + name='fc-%d' % idx) if idx < len(self.n_nodes)-1: a = tf.sigmoid(fc_op, name='a-%d' % idx) in_op = a @@ -88,10 +88,9 @@ def __init__(self, params): Constructor ''' # Network Parameters + super(MLP, self).__init__(params) self.n_input = params['n_input'] # 1st layer num features self.n_nodes = params['n_nodes'] # 1st layer num features - super(MLP, self).__init__(params) - self._cost_ops = [] self._y_logits = None self.branch = params.get('branch', len(self.n_nodes)-1) From 399caadc0de61487ffa51a0976badb0f97c91601 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Fri, 9 Mar 2018 15:04:58 +0100 Subject: [PATCH 21/24] fix input op to branching --- nideep/nets/mlp_tf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nideep/nets/mlp_tf.py b/nideep/nets/mlp_tf.py index 6403404..93e90f4 100644 --- a/nideep/nets/mlp_tf.py +++ b/nideep/nets/mlp_tf.py @@ -33,6 +33,8 @@ def _fc(self, x): in_op = a = x branch_op = x for idx in xrange(len(self.n_nodes)): + if idx == self.branch: + branch_op = a fc_name_w = 'fc-%d/w' % idx fc_name_b = 'fc-%d/b' % idx fc_op = tf.add(tf.matmul(in_op, self.w[fc_name_w]), @@ -44,8 +46,6 @@ def _fc(self, x): else: self._y_logits = fc_op self.p = tf.nn.softmax(fc_op, name='a-%d' % idx) - if idx == self.branch: - branch_op = a idx = self.branch fc_name_w = 'fc-%d-aux/w' % idx fc_name_b = 'fc-%d-aux/b' % idx From 9ab68c51760f76ee035b868160f75abb20d96149 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Wed, 28 Mar 2018 16:13:28 +0200 Subject: [PATCH 22/24] add init for package --- __init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 __init__.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 From 4733a6310732ba92b780a03a0d2b186c47e336c0 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Wed, 28 Mar 2018 16:36:11 +0200 Subject: [PATCH 23/24] remove init --- __init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 __init__.py diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29..0000000 From a5f2141f5a0534f87b977b55ce4a2f6c7e9073b4 Mon Sep 17 00:00:00 2001 From: Youssef Kashef Date: Mon, 6 Jan 2020 12:50:21 +0100 Subject: [PATCH 24/24] pip conda install switch --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 62a5045..4824537 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ before_install: - conda update -q conda # Useful for debugging any issues with conda - conda info -a - - conda create -q -n $CONDA_ENV python=$TRAVIS_PYTHON_VERSION pip numpy scipy nose mock coverage h5py opencv PIL protobuf scikit-learn scikit-image pandas + - conda create -q -n $CONDA_ENV python=$TRAVIS_PYTHON_VERSION pip numpy scipy nose mock coverage h5py opencv PIL protobuf scikit-learn tensorflow pandas - source activate $CONDA_ENV # Download caffe protobuf message definitions - mkdir -p $CAFFE_ROOT/src/caffe/proto @@ -69,7 +69,8 @@ before_install: install: # Packages that need to be installed via pip - - travis_retry pip install lmdb bunch tensorflow + - travis_retry pip install lmdb bunch + - travis_retry pip install -U scikit-image - if [[ $WITH_COVERAGE == ON ]]; then echo "Installing Coveralls" travis_retry pip install coveralls;