diff --git a/.gitignore b/.gitignore index b4d81e5..8827518 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,8 @@ target/ *.solverstate.h5 *.binaryproto *leveldb +MNIST_data/ +t*-ubyte.gz ##--------------------------------------------------- ## Remove autosaves generated by the Matlab editor diff --git a/.travis.yml b/.travis.yml index 086d9e4..4824537 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,7 +54,7 @@ before_install: - conda update -q conda # Useful for debugging any issues with conda - conda info -a - - conda create -q -n $CONDA_ENV python=$TRAVIS_PYTHON_VERSION pip numpy scipy nose mock coverage h5py opencv PIL protobuf scikit-learn scikit-image pandas + - conda create -q -n $CONDA_ENV python=$TRAVIS_PYTHON_VERSION pip numpy scipy nose mock coverage h5py opencv PIL protobuf scikit-learn tensorflow pandas - source activate $CONDA_ENV # Download caffe protobuf message definitions - mkdir -p $CAFFE_ROOT/src/caffe/proto @@ -69,7 +69,8 @@ before_install: install: # Packages that need to be installed via pip - - travis_retry pip install lmdb tensorflow + - travis_retry pip install lmdb bunch + - travis_retry pip install -U scikit-image - if [[ $WITH_COVERAGE == ON ]]; then echo "Installing Coveralls" travis_retry pip install coveralls; diff --git a/nideep/datasets/mnist/__init__.py b/nideep/datasets/mnist/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nideep/datasets/mnist/mnist_tf.py b/nideep/datasets/mnist/mnist_tf.py new file mode 100644 index 0000000..fd5a837 --- /dev/null +++ b/nideep/datasets/mnist/mnist_tf.py @@ -0,0 +1,308 @@ +''' +Created on Jul 31, 2017 + +@author: kashefy +''' +import os +from collections import namedtuple +import numpy as np +import skimage.transform as transform +from tensorflow.python.framework import dtypes, random_seed +from tensorflow.contrib.learn.python.learn.datasets.base import Datasets +from tensorflow.contrib.learn.python.learn.datasets.mnist import DataSet, read_data_sets +import tensorflow as tf +# imported for mocking +from tensorflow.contrib.learn.python.learn.datasets.mnist import extract_images + +def _int64_feature(value): + return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) + +def _bytes_feature(value): + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + +class MNIST(object): + ''' + classdocs + ''' + @classmethod + def read_data_sets(cls, + train_dir, + fake_data=False, + one_hot=False, + dtype=dtypes.float32, + reshape=True, + validation_size=5000, + seed=None): + + seed1, seed2 = random_seed.get_seed(seed) + # If op level seed is not set, use whatever graph level seed is returned + np.random.seed(seed1 if seed is None else seed2) + ds = read_data_sets(\ + train_dir, + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=reshape, + validation_size=0, + seed=seed) + perm0 = np.arange(ds.train.num_examples) + np.random.shuffle(perm0) + train_idxs = perm0[validation_size:] + val_idxs = perm0[:validation_size] + train = DataSet(\ + np.multiply(ds.train.images[train_idxs], [1., 255.][dtype == dtypes.float32]), # will rescale to [0,1] inside + ds.train.labels[train_idxs], + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=False, # already reshaped + seed=seed) + validation = DataSet(\ + np.multiply(ds.train.images[val_idxs], [1., 255.][dtype == dtypes.float32]), # will rescale to [0,1] inside + ds.train.labels[val_idxs], + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=False, # already reshaped + seed=seed) + return Datasets(train=train, validation=validation, test=ds.test) + + @classmethod + def to_tf_record(cls, + fpath, + data_dir, + fake_data=False, + one_hot=False, + dtype=dtypes.float32, + reshape=True, + validation_size=5000, + seed=None, + orientations=np.linspace(-90, 90, 180/(12+1), endpoint=True).tolist(), + ): + """ + Get dataset with orientations stored in tf.records + """ + mnist = MNIST.read_data_sets( + data_dir, + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=False, + validation_size=validation_size, + seed=seed, + ) + fname0, ext = os.path.splitext(fpath) + DatasetTFRecords = namedtuple('DatasetTFRecords', + ['num_examples', + 'path', + 'one_hot', + 'orientations', + 'phase', + 'images', + 'labels']) + dsets = {} + for phase in Datasets._fields:#['train', 'validation', 'test']: + split = getattr(mnist, phase) + num_examples = 0 + if fname0.endswith(phase): + fpath_phase = fpath + else: + fpath_phase = ''.join([fname0, '_', phase, ext]) + if os.path.isfile(fpath_phase): + # Skip if it already exists + num_examples = sum(1 for _ in tf.python_io.tf_record_iterator(fpath_phase)) + else: + with tf.python_io.TFRecordWriter(fpath_phase) as writer: + for img, label in zip(split.images, split.labels): + if img.ndim < 2: + raise AttributeError("Rotation needs both height and width images resolved, found shape %s" % img.shape) + img = np.expand_dims(img, axis=0) + imgs_orient_all, labels_orient_all = \ + MNIST.rotate(img, orientations, one_hot) + for img_orient, label_orient in zip(imgs_orient_all, labels_orient_all): + img_raw = img_orient.tobytes() + ## Convert the bytes back to image as follow: + # image = Image.frombytes('RGB', (224,224), img_raw) + # tl.visualize.frame(I=image, second=1, saveable=False, name='frame', fig_idx=1236) + ## Write the data into TF format + # image : Feature + BytesList + # label : Feature + Int64List or FloatList + # sentence : FeatureList + Int64List , see Google's im2txt example + if one_hot: + label_raw = label.astype(np.float32).tobytes() + label_orient_raw = label_orient.astype(np.float32).tobytes() + example = tf.train.Example(features=tf.train.Features(feature={ + "label" : _bytes_feature(label_raw), + "label_orient" : _bytes_feature(label_orient_raw), + 'img_raw' : _bytes_feature(img_raw), + })) + else: + example = tf.train.Example(features=tf.train.Features(feature={ + "label" : _int64_feature(int(label)), + "label_orient" : _int64_feature(int(label_orient)), + 'img_raw' : _bytes_feature(img_raw), + })) + writer.write(example.SerializeToString()) # Serialize To String + num_examples += 1 + dsets[phase] = DatasetTFRecords( + num_examples=num_examples, + path=fpath_phase, + one_hot=one_hot, + orientations=orientations, + phase=phase, + images=np.empty((num_examples,) + split.images.shape[1:], + dtype=split.images.dtype), + labels=np.empty((num_examples,) + split.labels.shape[1:], + dtype=split.labels.dtype), + ) + return Datasets(train=dsets['train'], + validation=dsets['validation'], + test=dsets['test']) + + @classmethod + def read_and_decode_ops(cls, fpath, one_hot=False, + img_shape=(784,), # or (28, 28, 1) + num_classes=10, + num_orientations=0, + ): + # generate a queue with a given file name + filename_queue = tf.train.string_input_producer([fpath]) + reader = tf.TFRecordReader() + _, serialized_example = reader.read(filename_queue) # return the file and the name of file + feat_dict = { + 'label' : tf.FixedLenFeature([], [tf.int64, tf.string][one_hot]), + 'img_raw' : tf.FixedLenFeature([], tf.string), + } + if num_orientations > 0: + feat_dict['label_orient'] = tf.FixedLenFeature([], [tf.int64, tf.string][one_hot]) + features = tf.parse_single_example(serialized_example, # see parse_single_sequence_example for sequence example + features=feat_dict) + if one_hot: + label = tf.decode_raw(features['label'], tf.float32) +# label = tf.reshape(label, [10]) + label.set_shape((num_classes,)) + if num_orientations > 0: + label_orient = tf.decode_raw(features['label_orient'], tf.float32) + # label = tf.reshape(label, [10]) + label_orient.set_shape((num_orientations,)) + else: + label = tf.cast(features['label'], tf.int32) + if num_orientations > 0: + label_orient = tf.cast(features['label_orient'], tf.int32) + # You can do more image distortion here for training data + img = tf.decode_raw(features['img_raw'], tf.float32) +# img = tf.reshape(img, [28 * 28 * 1]) + img.set_shape((784,)) + if num_orientations > 0: + return img, label, label_orient + else: + return img, label + + @staticmethod + def rotate(imgs, + orientations, + one_hot, + mode='symmetric', + cval=0): + num_examples_in = len(imgs) + labels_orient_all = None + imgs_orient_all = None + for angle_idx, angle in enumerate(orientations): + imgs_orient = np.zeros_like(imgs) + for img_idx, img in enumerate(imgs): + img_rot = transform.rotate(img, angle, + resize=False, center=None, + order=1, + mode=mode, cval=cval, + clip=True, preserve_range=False) + imgs_orient[img_idx] = img_rot + if one_hot: + labels_orient = np.zeros((num_examples_in, len(orientations))) + labels_orient[:, angle_idx] = 1 + else: + labels_orient = np.zeros((num_examples_in,)) + angle_idx + if labels_orient_all is None: + labels_orient_all = labels_orient + imgs_orient_all = imgs_orient + else: + labels_orient_all = np.vstack((labels_orient_all, labels_orient)) + imgs_orient_all = np.vstack((imgs_orient_all, imgs_orient)) + return imgs_orient_all, labels_orient_all + + @classmethod + def read_data_sets_orient(cls, + train_dir, + fake_data=False, + one_hot=False, + dtype=dtypes.float32, + reshape=True, + validation_size=5000, + seed=None, + orientations=np.linspace(-90, 90, 180/(12+1), endpoint=True).tolist()): + + seed1, seed2 = random_seed.get_seed(seed) + # If op level seed is not set, use whatever graph level seed is returned + np.random.seed(seed1 if seed is None else seed2) + ds = read_data_sets(\ + train_dir, + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=False, # preseve image dimensions for rotation + validation_size=0, + seed=seed) + imgs_orient_all, labels_orient_all = MNIST.rotate(ds.train.images, orientations, one_hot) + print(labels_orient_all.shape, imgs_orient_all.shape) + # An image and all its orientation either fall into train or validation + perm0 = np.arange(ds.train.num_examples) + np.random.shuffle(perm0) + train_idxs0 = perm0[validation_size:] + val_idxs0 = perm0[:validation_size] + train_idxs = train_idxs0 + val_idxs = val_idxs0 + for orient_idx in range(len(orientations)): + train_idxs_orient = train_idxs0 + (orient_idx+1)*ds.train.num_examples + val_idxs_orient = val_idxs0 + (orient_idx+1)*ds.train.num_examples + train_idxs = np.vstack((train_idxs, train_idxs_orient)) + val_idxs = np.vstack((val_idxs, val_idxs_orient)) + np.random.shuffle(train_idxs) + np.random.shuffle(val_idxs) + train = DataSet(\ + np.multiply(imgs_orient_all[train_idxs], [1., 255.][dtype == dtypes.float32]), # will rescale to [0,1] inside + labels_orient_all[train_idxs], + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=reshape, # reshape here for the first time + seed=seed) + validation = DataSet(\ + np.multiply(imgs_orient_all[val_idxs], [1., 255.][dtype == dtypes.float32]), # will rescale to [0,1] inside + labels_orient_all[val_idxs], + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=reshape, # reshape here for the first time + seed=seed) + validation = DataSet(\ + np.multiply(imgs_orient_all[val_idxs], [1., 255.][dtype == dtypes.float32]), # will rescale to [0,1] inside + labels_orient_all[val_idxs], + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=reshape, # reshape here for the first time + seed=seed) + labels_orient_all, imgs_orient_all = MNIST.rotate(ds.test.images, orientations, one_hot) + test = DataSet(\ + np.multiply(imgs_orient_all, [1., 255.][dtype == dtypes.float32]), # will rescale to [0,1] inside + labels_orient_all, + fake_data=fake_data, + one_hot=one_hot, + dtype=dtype, + reshape=reshape, # reshape here for the first time + seed=seed) + return Datasets(train=train, validation=validation, test=ds.test) + + def __init__(self, params): + ''' + Constructor + ''' diff --git a/nideep/datasets/mnist/test_mnist_tf.py b/nideep/datasets/mnist/test_mnist_tf.py new file mode 100644 index 0000000..ef6d3b2 --- /dev/null +++ b/nideep/datasets/mnist/test_mnist_tf.py @@ -0,0 +1,174 @@ +''' +Created on Aug 11, 2017 + +@author: kashefy +''' +import os +import tempfile +import shutil +import numpy as np +from nose.tools import assert_equals, assert_true, assert_false, \ + assert_list_equal +from mock import patch +from mnist_tf import MNIST + +class TestMNISTTF: + @classmethod + def setup_class(self): + self.dir_tmp = tempfile.mkdtemp() + + @classmethod + def teardown_class(self): + shutil.rmtree(self.dir_tmp) + + def test_to_tf_record(self): + import tensorflow as tf + x = tf.placeholder(tf.float32, [None, 784]) + name_w = 'W' + with tf.variable_scope("var_scope", reuse=None): + W = tf.get_variable(name_w, shape=[784, 10], + initializer=tf.random_normal_initializer(stddev=0.1)) + b = tf.get_variable('b', shape=[10], + initializer=tf.constant_initializer(0.1)) + logits = tf.matmul(x, W) + b + y = tf.nn.softmax(logits) + y_ = tf.placeholder(tf.float32, [None, 10]) + cross_entropy = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(\ + labels=y_, logits=logits)) + train_step = tf.train.GradientDescentOptimizer(0.9).minimize(cross_entropy) + init_op = tf.global_variables_initializer() + one_hot = True + shuffle = True + orientations = range(-60,75,15) + print(orientations) +# mnist = MNIST.read_data_sets('MNIST_data', one_hot=True) + fpath_list = MNIST.to_tf_record(os.path.join('MNIST_data', 'train.tfrecords'), + 'MNIST_data', + one_hot=one_hot, + orientations=orientations) + print fpath_list + return + + img, label, label_orient = MNIST.read_and_decode_ops(\ + os.path.join('MNIST_data', 'train.tfrecords'), + one_hot=one_hot, + num_orientations=len(orientations)) +# import tensorflow as tf + if shuffle: + batch_xs_op, batch_ys_op, batch_os_op = tf.train.shuffle_batch([img, label, label_orient], + batch_size=64, + capacity=2000, + min_after_dequeue=1000, + num_threads=8 + ) + else: + batch_xs_op, batch_ys_op, batch_os_op = tf.train.batch([img, label, label_orient], + batch_size=64, + capacity=2000, + # min_after_dequeue=1000, + num_threads=8 + ) + + import matplotlib.pyplot as plt + f, a = plt.subplots(3, 12, figsize=(10, 2)) + with tf.Session() as sess: + + coord = tf.train.Coordinator() + threads = tf.train.start_queue_runners(sess=sess, coord=coord) + for i in range(3): + val, l, orient = sess.run([batch_xs_op, batch_ys_op, batch_os_op]) + print(val.shape, l) + for j in range(12): + a[i][j].imshow(np.reshape(val[j], (28, 28)), clim=(0.0, 1.0)) + if one_hot: + a[i][j].set_title('%d at %.2f deg' % (np.argmax(l[j]), orientations[np.argmax(orient[j])])) + else: + a[i][j].set_title('%d at %.2f deg' % (l[j], orientations[orient[j]])) + + plt.show() + return + sess.run(init_op) + for itr in range(1000): + batch_xs, batch_ys = sess.run([batch_xs_op, batch_ys_op]) + _, c = sess.run([train_step, cross_entropy], + feed_dict={x: batch_xs, y_: batch_ys}) +# print(c) + correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) + accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) + print(sess.run(accuracy, + feed_dict={x: mnist.test.images, y_: mnist.test.labels})) + print(sess.run(accuracy, + feed_dict={x: mnist.validation.images, y_: mnist.validation.labels})) + + coord.request_stop() + coord.join(threads) + +# def test_orient(self): +# d1 = MNIST.read_data_sets_orient(self.dir_tmp, seed=123, orientations=range(-30,60,30)) + +# @patch('nideep.datasets.mnist.mnist_tf.extract_images') +# def test_read(self, mock_extract): +# mock_extract.return_value = np.empty_like((100000, 28, 28)) +# d = MNIST.read_data_sets(self.dir_tmp) +# assert_equals(d.train.num_examples, 55000) +# assert_equals(d.validation.num_examples, 5000) +# assert_equals(d.test.num_examples, 10000) +# +# @patch('nideep.datasets.mnist.mnist_tf.extract_images') +# def test_validation_size(self, mock_extract): +# mock_extract.return_value = np.empty_like((100000, 28, 28)) +# for sz in range(5000, 55000, 5000): +# yield self.check_validation_size, sz +# +# def check_validation_size(self, sz): +# d = MNIST.read_data_sets(self.dir_tmp, validation_size=sz) +# assert_equals(d.train.num_examples, 60000-sz) +# assert_equals(d.validation.num_examples, sz) +# assert_equals(d.test.num_examples, 10000) +# +# def test_shuffle(self): +# d1 = MNIST.read_data_sets(self.dir_tmp) +# d2 = MNIST.read_data_sets(self.dir_tmp) +# assert_false(np.array_equal(d1.train.images, d2.train.images)) +# assert_false(np.array_equal(d1.train.labels, d2.train.labels)) +# assert_false(np.array_equal(d1.validation.images, d2.validation.images)) +# assert_false(np.array_equal(d1.validation.labels, d2.validation.labels)) +# assert_true(np.array_equal(d1.test.images, d2.test.images)) +# assert_true(np.array_equal(d1.test.labels, d2.test.labels)) +# +# def test_seed(self): +# d1 = MNIST.read_data_sets(self.dir_tmp, seed=123) +# d2 = MNIST.read_data_sets(self.dir_tmp, seed=999) +# d3 = MNIST.read_data_sets(self.dir_tmp, seed=123) +# assert_false(np.array_equal(d1.train.images, d2.train.images)) +# assert_false(np.array_equal(d1.train.labels, d2.train.labels)) +# assert_false(np.array_equal(d1.validation.images, d2.validation.images)) +# assert_false(np.array_equal(d1.validation.labels, d2.validation.labels)) +# assert_true(np.array_equal(d1.test.images, d2.test.images)) +# assert_true(np.array_equal(d1.test.labels, d2.test.labels)) +# # same seed +# assert_true(np.array_equal(d1.train.images, d3.train.images)) +# assert_true(np.array_equal(d1.train.labels, d3.train.labels)) +# assert_true(np.array_equal(d1.validation.images, d3.validation.images)) +# assert_true(np.array_equal(d1.validation.labels, d3.validation.labels)) +# assert_true(np.array_equal(d1.test.images, d3.test.images)) +# assert_true(np.array_equal(d1.test.labels, d3.test.labels)) +# +# def test_data_scale(self): +# from tensorflow.examples.tutorials.mnist import input_data +# d0 = input_data.read_data_sets(self.dir_tmp, one_hot=True) +# d1 = MNIST.read_data_sets(self.dir_tmp, one_hot=True) +# assert_equals(d0.train.images.min(), 0) +# assert_equals(d0.train.images.max(), 1) +# assert_equals(d0.train.images.min(), d1.train.images.min()) +# assert_equals(d0.train.images.max(), d1.train.images.max()) +# assert_equals(d0.validation.images.min(), 0) +# assert_equals(d0.validation.images.max(), 1) +# assert_equals(d0.validation.images.min(), d1.validation.images.min()) +# assert_equals(d0.validation.images.max(), d1.validation.images.max()) +# assert_equals(d0.test.images.min(), 0) +# assert_equals(d0.test.images.max(), 1) +# assert_equals(d0.test.images.min(), d1.test.images.min()) +# assert_equals(d0.test.images.max(), d1.test.images.max()) + \ No newline at end of file diff --git a/nideep/nets/abstract_net.py b/nideep/nets/abstract_net.py new file mode 100644 index 0000000..a02b976 --- /dev/null +++ b/nideep/nets/abstract_net.py @@ -0,0 +1,95 @@ +''' +Created on Jul 14, 2017 + +@author: kashefy +''' +from __future__ import division, print_function, absolute_import +from abc import ABCMeta, abstractmethod +import logging + +class AbstractNet(object): + ''' + classdocs + ''' + __metaclass__ = ABCMeta + + @abstractmethod + def _init_learning_params(self): + pass + + @abstractmethod + def _init_ops(self): + pass # return prediction_op, logit_op + + def build(self): + self._init_learning_params() + self._init_ops() + return self.p, self.logits + + def _config_scopes(self): + self.var_scope = self.prefix + self.name_scope = self.var_scope + '/' + + @property + def x(self): + return self._x + + @x.setter + def x(self, value): + self._x = value + + @x.deleter + def x(self): + del self._x + + @property + def p(self): + return self._p + + @p.setter + def p(self, value): + self._p = value + + @p.deleter + def p(self): + del self._p + + @property + def logits(self): + return self._logits + + @logits.setter + def logits(self, value): + self._logits = value + + @logits.deleter + def logits(self): + del self._logits + + @property + def vars_restored(self): + return self._vars_restored + + @vars_restored.setter + def vars_restored(self, value): + self._vars_restored = value + + @vars_restored.deleter + def vars_restored(self): + del self._vars_restored + + def __init__(self, params): + ''' + Constructor + ''' + logger_name = self.__class__.__name__ + if params.get('logger_name', None) is not None: + logger_name = '.'.join([params.get('logger_name', None), logger_name]) + self.logger = logging.getLogger(logger_name) + self._x = None + self._p = None + self.prefix = params.get('prefix', '') + self._config_scopes() + self._vars_restored = [] + self._cost_op = None + \ No newline at end of file diff --git a/nideep/nets/abstract_net_tf.py b/nideep/nets/abstract_net_tf.py new file mode 100644 index 0000000..3da5d5c --- /dev/null +++ b/nideep/nets/abstract_net_tf.py @@ -0,0 +1,72 @@ +''' +Created on Jul 18, 2017 + +@author: kashefy +''' +import tensorflow as tf +from nideep.nets.abstract_net import AbstractNet +from abc import ABCMeta, abstractmethod + +class AbstractNetTF(AbstractNet): + ''' + classdocs + ''' + __metaclass__ = ABCMeta + + @staticmethod + def _init_weight_op(mean=0.0, stddev=0.1): + return tf.random_normal_initializer(mean=mean, + stddev=stddev) + + @staticmethod + def _init_bias_op(value): + return tf.constant_initializer(value) + + def vars_new(self): + return tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, + scope=self.name_scope) + + def _init_bias_vars(self, bias_value=0.): + self.b = {} + for key, tensor in self.w.iteritems(): + key_b = key.replace('/w', '/b').replace('_w', '_b').replace('-w', '-b') + if self.reuse: + self.b[key_b] = self._restore_variable(self.var_scope + '/' + key_b) + else: + self.b[key_b] = tf.get_variable(key_b, + [tensor.get_shape()[-1].value], + initializer=self._init_bias_op(bias_value)) + + @abstractmethod + def _init_learning_params_scoped(self): + pass + + def _init_learning_params(self): + with tf.variable_scope(self.var_scope, reuse=self.reuse): + self._init_learning_params_scoped() + pass + + def _restore_variable(self, var_name): + v = self.get_tensor_by_name(var_name) + self.vars_restored.append(v) + return v + + def get_tensor_by_name(self, tensor_name, graph=tf.get_default_graph(), index=-1): + name_with_index = self.get_tensor_names(tensor_name, graph=graph)[index] + return graph.get_tensor_by_name(name_with_index) + + def get_tensor_names(self, tensor_name, graph=tf.get_default_graph()): +# op_names = [output.name for op in graph.get_operations() +# if op.op_def and 'Variable' in op.op_def.name and +# tensor_name == op.name for output in op.outputs] + op_names = [output.name for op in graph.get_operations() + if op.op_def and + tensor_name == op.name for output in op.outputs] + return op_names + + def __init__(self, params): + ''' + Constructor + ''' + self.reuse = params.get('reuse', None) + super(AbstractNetTF, self).__init__(params) \ No newline at end of file diff --git a/nideep/nets/mlp_tf.py b/nideep/nets/mlp_tf.py new file mode 100644 index 0000000..93e90f4 --- /dev/null +++ b/nideep/nets/mlp_tf.py @@ -0,0 +1,97 @@ +''' +Created on Jul 14, 2017 + +@author: kashefy +''' +import tensorflow as tf +from nideep.nets.abstract_net_tf import AbstractNetTF + +class MLP(AbstractNetTF): + ''' + classdocs + ''' + def _init_learning_params_scoped(self): + self.w = {} + for idx, dim in enumerate(self.n_nodes): + input_dim = self.n_nodes[idx-1] + if idx == 0: + input_dim = self.n_input + fc_name_w = 'fc-%d/w' % idx + self.w[fc_name_w] = tf.get_variable(fc_name_w, + [input_dim, dim], + initializer=self._init_weight_op() + ) + if idx == self.branch: + fc_name_w = 'fc-%d-aux/w' % idx + self.w[fc_name_w] = tf.get_variable(fc_name_w, + [input_dim, 9], + initializer=self._init_weight_op() + ) + self._init_bias_vars(bias_value=0.1) + + def _fc(self, x): + in_op = a = x + branch_op = x + for idx in xrange(len(self.n_nodes)): + if idx == self.branch: + branch_op = a + fc_name_w = 'fc-%d/w' % idx + fc_name_b = 'fc-%d/b' % idx + fc_op = tf.add(tf.matmul(in_op, self.w[fc_name_w]), + self.b[fc_name_b], + name='fc-%d' % idx) + if idx < len(self.n_nodes)-1: + a = tf.sigmoid(fc_op, name='a-%d' % idx) + in_op = a + else: + self._y_logits = fc_op + self.p = tf.nn.softmax(fc_op, name='a-%d' % idx) + idx = self.branch + fc_name_w = 'fc-%d-aux/w' % idx + fc_name_b = 'fc-%d-aux/b' % idx + in_op = branch_op + fc_op = tf.add(tf.matmul(in_op, self.w[fc_name_w]), + self.b[fc_name_b], + name='fc-%d-aux' % idx) + self._y_logits_aux = fc_op + self.p_aux = tf.nn.softmax(fc_op, name='a-%d-aux' % idx) + return self.p, self._y_logits + + def _init_ops(self): + with tf.name_scope(self.name_scope + 'fc'): + self.p, self.logits = self._fc(self.x) + + def cost(self, y_true, name=None): + with tf.name_scope(self.name_scope): + c = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(\ + labels=y_true, logits=self._y_logits), + name=name) + self._cost_ops.append(c) + return c + + def cost_aux(self, y_true, name=None): + with tf.name_scope(self.name_scope): + c = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(\ + labels=y_true, logits=self._y_logits_aux), + name=name) + self._cost_ops.append(c) + return c + + @property + def cost_ops(self): + return self._cost_ops + + def __init__(self, params): + ''' + Constructor + ''' + # Network Parameters + super(MLP, self).__init__(params) + self.n_input = params['n_input'] # 1st layer num features + self.n_nodes = params['n_nodes'] # 1st layer num features + self._cost_ops = [] + self._y_logits = None + self.branch = params.get('branch', len(self.n_nodes)-1) + self.logger.debug("Branch aux. task from layer: %d" % self.branch) \ No newline at end of file diff --git a/nideep/test_data/config0.yaml b/nideep/test_data/config0.yaml new file mode 100644 index 0000000..dcb2981 --- /dev/null +++ b/nideep/test_data/config0.yaml @@ -0,0 +1,2 @@ +learning_rate: 0.001 +batch_size: 16