git clone https://github.com/TengdaHan/GAN-TensorFlow

def xavier_init(size):
  in_dim = size[0]
  xavier_stddev = 1\. / tf.sqrt(in_dim / 2.)
  return xavier_stddev

def generator(X):
  with tf.variable_scope('generator'): 
    K = 128
    L = 784
    W1 = tf.get_variable('G_W1', [100, K],
  initializer=tf.random_normal_initializer(stddev=xavier_init([100,   K])))
    B1 = tf.get_variable('G_B1', [K],       initializer=tf.constant_initializer())
    W2 = tf.get_variable('G_W2', [K, L],
    initializer=tf.random_normal_initializer(stddev=xavier_init([K,   L])))
    B2 = tf.get_variable('G_B2', [L],    initializer=tf.constant_initializer())
    # summary
    tf.summary.histogram('weight1', W1)
    tf.summary.histogram('weight2', W2)
    tf.summary.histogram('biases1', B1)
    tf.summary.histogram('biases2', B2)
    fc1 = tf.nn.relu((tf.matmul(X, W1) + B1))
    fc2 = tf.matmul(fc1, W2) + B2
    prob = tf.nn.sigmoid(fc2)
    return prob

def discriminator(X, reuse=False):
  with tf.variable_scope('discriminator'):
    if reuse:
      tf.get_variable_scope().reuse_variables()
    J = 784
    K = 128
    L = 1
    W1 = tf.get_variable('D_W1', [J, K],
    initializer=tf.random_normal_initializer(stddev=xavier_init([J,  K])))
    B1 = tf.get_variable('D_B1', [K],    initializer=tf.constant_initializer())
    W2 = tf.get_variable('D_W2', [K, L],
initializer=tf.random_normal_initializer(stddev=xavier_init([K, L])))
    B2 = tf.get_variable('D_B2', [L],  initializer=tf.constant_initializer())
    # summary
    tf.summary.histogram('weight1', W1)
    tf.summary.histogram('weight2', W2)
    tf.summary.histogram('biases1', B1)
    tf.summary.histogram('biases2', B2)
    fc1 = tf.nn.relu((tf.matmul(X, W1) + B1))
    logits = tf.matmul(fc1, W2) + B2
    prob = tf.nn.sigmoid(logits)
    return prob, logits

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import os
import argparse

def read_data():
  from tensorflow.examples.tutorials.mnist import input_data
  mnist = input_data.read_data_sets("../MNIST_data/", one_hot=True)
  return mnist

def plot(samples):
  fig = plt.figure(figsize=(8, 8))
  gs = gridspec.GridSpec(8, 8)
  gs.update(wspace=0.05, hspace=0.05)
  for i, sample in enumerate(samples):
    ax = plt.subplot(gs[i])
    plt.axis('off')
    ax.set_xticklabels([])
    ax.set_yticklabels([])
    ax.set_aspect('equal')
    plt.imshow(sample.reshape(28, 28), cmap='Greys_r')
  return fig

def train(logdir, batch_size):
  from model_fc import discriminator, generator
  mnist = read_data()
  with tf.variable_scope('placeholder'):
    # Raw image
    X = tf.placeholder(tf.float32, [None, 784])
    tf.summary.image('raw image', tf.reshape(X, [-1, 28, 28, 1]), 3)
   # Noise
   z = tf.placeholder(tf.float32, [None, 100]) # noise
   tf.summary.histogram('Noise', z)

  with tf.variable_scope('GAN'):
    G = generator(z)
    D_real, D_real_logits = discriminator(X, reuse=False)
    D_fake, D_fake_logits = discriminator(G, reuse=True)
    tf.summary.image('generated image', tf.reshape(G, [-1, 28, 28, 1]), 3)

  with tf.variable_scope('Prediction'):
    tf.summary.histogram('real', D_real) 
    tf.summary.histogram('fake', D_fake)

  with tf.variable_scope('D_loss'):
    d_loss_real = tf.reduce_mean(
    tf.nn.sigmoid_cross_entropy_with_logits(
    logits=D_real_logits, labels=tf.ones_like(D_real_logits)))
    d_loss_fake = tf.reduce_mean(
    tf.nn.sigmoid_cross_entropy_with_logits(
     logits=D_fake_logits, labels=tf.zeros_like(D_fake_logits)))
    d_loss = d_loss_real + d_loss_fake

  tf.summary.scalar('d_loss_real', d_loss_real)
  tf.summary.scalar('d_loss_fake', d_loss_fake)
  tf.summary.scalar('d_loss', d_loss)

  with tf.name_scope('G_loss'):
    g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits
   (logits=D_fake_logits, labels=tf.ones_like(D_fake_logits)))
    tf.summary.scalar('g_loss', g_loss)
    tvar = tf.trainable_variables()
    dvar = [var for var in tvar if 'discriminator' in var.name]
    gvar = [var for var in tvar if 'generator' in var.name]

  with tf.name_scope('train'):
    d_train_step = tf.train.AdamOptimizer().minimize(d_loss,   var_list=dvar)
    g_train_step = tf.train.AdamOptimizer().minimize(g_loss,  var_list=gvar)

  sess = tf.Session()
  init = tf.global_variables_initializer()
  sess.run(init)
  merged_summary = tf.summary.merge_all()
  writer = tf.summary.FileWriter('tmp/mnist/'+logdir)
  writer.add_graph(sess.graph)
  num_img = 0
  if not os.path.exists('output/'):
    os.makedirs('output/')

  for i in range(100000):
    batch_X, _ = mnist.train.next_batch(batch_size)
    batch_noise = np.random.uniform(-1., 1., [batch_size, 100])
    if i % 500 == 0:
     samples = sess.run(G, feed_dict={z: np.random.uniform(-1., 1., [64, 100])})
     fig = plot(samples)
     plt.savefig('output/%s.png' % str(num_img).zfill(3),    bbox_inches='tight')
     num_img += 1
    plt.close(fig)

  _, d_loss_print = sess.run([d_train_step, d_loss],
feed_dict={X: batch_X, z: batch_noise})
  _, g_loss_print = sess.run([g_train_step, g_loss],
  feed_dict={z: batch_noise})
  if i % 100 == 0:
    s = sess.run(merged_summary, feed_dict={X: batch_X, z: batch_noise})
    writer.add_summary(s, i)
    print('epoch:%d g_loss:%f d_loss:%f' % (i, g_loss_print, d_loss_print))

  if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Train vanila GAN using fully-connected layers networks')
    parser.add_argument('--logdir', type=str, default='1', help='logdir for Tensorboard, give a string')
    parser.add_argument('--batch_size', type=int, default=64, help='batch size: give a int')
    args = parser.parse_args()
    train(logdir=args.logdir, batch_size=args.batch_size)

git clone https://github.com/TengdaHan/GAN-TensorFlow

def xavier_init(size):
  in_dim = size[0]
  xavier_stddev = 1\. / tf.sqrt(in_dim / 2.)
  # return tf.random_normal(shape=size, stddev=xavier_stddev)
  return xavier_stddev

def conv(x, w, b, stride, name):
  with tf.variable_scope('conv'):
    tf.summary.histogram('weight', w)
    tf.summary.histogram('biases', b)
    return tf.nn.conv2d(x,
      filter=w,
      strides=[1, stride, stride, 1],
      padding='SAME',
      name=name) + b

def deconv(x, w, b, shape, stride, name):
  with tf.variable_scope('deconv'):
    tf.summary.histogram('weight', w)
    tf.summary.histogram('biases', b)
    return tf.nn.conv2d_transpose(x,
      filter=w,
      output_shape=shape,
      strides=[1, stride, stride, 1],
      padding='SAME',
      name=name) + b

def lrelu(x, alpha=0.2):
  with tf.variable_scope('leakyReLU'):
    return tf.maximum(x, alpha * x)

def generator(X, batch_size=64):
  with tf.variable_scope('generator'):
    K = 256
    L = 128
    M = 64
    W1 = tf.get_variable('G_W1', [100, 7*7*K],    initializer=tf.random_normal_initializer(stddev=0.1))
    B1 = tf.get_variable('G_B1', [7*7*K], initializer=tf.constant_initializer())
    W2 = tf.get_variable('G_W2', [4, 4, M, K], initializer=tf.random_normal_initializer(stddev=0.1))
    B2 = tf.get_variable('G_B2', [M], initializer=tf.constant_initializer())
    W3 = tf.get_variable('G_W3', [4, 4, 1, M], initializer=tf.random_normal_initializer(stddev=0.1))
    B3 = tf.get_variable('G_B3', [1], initializer=tf.constant_initializer())
    X = lrelu(tf.matmul(X, W1) + B1)
    X = tf.reshape(X, [batch_size, 7, 7, K])
    deconv1 = deconv(X, W2, B2, shape=[batch_size, 14, 14, M], stride=2, name='deconv1')
    bn1 = tf.contrib.layers.batch_norm(deconv1)
    deconv2 = deconv(tf.nn.dropout(lrelu(bn1), 0.4), W3, B3, shape=[batch_size, 28, 28, 1], stride=2, name='deconv2')
    XX = tf.reshape(deconv2, [-1, 28*28], 'reshape')
    return tf.nn.sigmoid(XX)

def discriminator(X, reuse=False):
  with tf.variable_scope('discriminator'):
    if reuse:
      tf.get_variable_scope().reuse_variables()
    K = 64
    M = 128
    N = 256
    W1 = tf.get_variable('D_W1', [4, 4, 1, K],   initializer=tf.random_normal_initializer(stddev=0.1))
    B1 = tf.get_variable('D_B1', [K], initializer=tf.constant_initializer())
    W2 = tf.get_variable('D_W2', [4, 4, K, M], initializer=tf.random_normal_initializer(stddev=0.1))
    B2 = tf.get_variable('D_B2', [M], initializer=tf.constant_initializer())
    W3 = tf.get_variable('D_W3', [7*7*M, N], initializer=tf.random_normal_initializer(stddev=0.1))
    B3 = tf.get_variable('D_B3', [N], initializer=tf.constant_initializer())
    W4 = tf.get_variable('D_W4', [N, 1], initializer=tf.random_normal_initializer(stddev=0.1))
    B4 = tf.get_variable('D_B4', [1], initializer=tf.constant_initializer())
    X = tf.reshape(X, [-1, 28, 28, 1], 'reshape')
    conv1 = conv(X, W1, B1, stride=2, name='conv1')
    bn1 = tf.contrib.layers.batch_norm(conv1)
    conv2 = conv(tf.nn.dropout(lrelu(bn1), 0.4), W2, B2, stride=2, name='conv2')
    bn2 = tf.contrib.layers.batch_norm(conv2)
    flat = tf.reshape(tf.nn.dropout(lrelu(bn2), 0.4), [-1, 7*7*M], name='flat')
    dense = lrelu(tf.matmul(flat, W3) + B3)
    logits = tf.matmul(dense, W4) + B4
    prob = tf.nn.sigmoid(logits)
    return prob, logits

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import os
import argparse

def read_data():
  from tensorflow.examples.tutorials.mnist import input_data
  mnist = input_data.read_data_sets("../MNIST_data/", one_hot=True)
  return mnist

def plot(samples):
  fig = plt.figure(figsize=(8, 8))
  gs = gridspec.GridSpec(8, 8)
  gs.update(wspace=0.05, hspace=0.05)
  for i, sample in enumerate(samples):
    ax = plt.subplot(gs[i])
    plt.axis('off')
    ax.set_xticklabels([])
    ax.set_yticklabels([])
    ax.set_aspect('equal')
    plt.imshow(sample.reshape(28, 28), cmap='Greys_r')
    return fig

def train(logdir, batch_size):
  from model_conv import discriminator, generator
  mnist = read_data()

  with tf.variable_scope('placeholder'):
    # Raw image
    X = tf.placeholder(tf.float32, [None, 784])
    tf.summary.image('raw image', tf.reshape(X, [-1, 28, 28, 1]), 3)
    # Noise
    z = tf.placeholder(tf.float32, [None, 100]) # noise
    tf.summary.histogram('Noise', z)

  with tf.variable_scope('GAN'):
    G = generator(z, batch_size)
    D_real, D_real_logits = discriminator(X, reuse=False)
    D_fake, D_fake_logits = discriminator(G, reuse=True)
    tf.summary.image('generated image', tf.reshape(G, [-1, 28, 28, 1]), 3)

  with tf.variable_scope('Prediction'):
    tf.summary.histogram('real', D_real)
    tf.summary.histogram('fake', D_fake)

  with tf.variable_scope('D_loss'):
    d_loss_real = tf.reduce_mean(
    tf.nn.sigmoid_cross_entropy_with_logits(
logits=D_real_logits, labels=tf.ones_like(D_real_logits)))

    d_loss_fake = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=D_fake_logits, labels=tf.zeros_like(D_fake_logits)))
    d_loss = d_loss_real + d_loss_fake
    tf.summary.scalar('d_loss_real', d_loss_real)
    tf.summary.scalar('d_loss_fake', d_loss_fake)
    tf.summary.scalar('d_loss', d_loss)

  with tf.name_scope('G_loss'):
    g_loss =   tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits
(logits=D_fake_logits, labels=tf.ones_like(D_fake_logits)))
    tf.summary.scalar('g_loss', g_loss)
    tvar = tf.trainable_variables()
    dvar = [var for var in tvar if 'discriminator' in var.name]
    gvar = [var for var in tvar if 'generator' in var.name]

  with tf.name_scope('train'):
    d_train_step = tf.train.AdamOptimizer().minimize(d_loss, var_list=dvar)
    g_train_step = tf.train.AdamOptimizer().minimize(g_loss, var_list=gvar)

  sess = tf.Session()
  init = tf.global_variables_initializer()

  sess.run(init)
  merged_summary = tf.summary.merge_all()
  writer = tf.summary.FileWriter('tmp/'+'gan_conv_'+logdir)
  writer.add_graph(sess.graph)
  num_img = 0

  if not os.path.exists('output/'):
    os.makedirs('output/')
  for i in range(100000):
    batch_X, _ = mnist.train.next_batch(batch_size)
    batch_noise = np.random.uniform(-1., 1., [batch_size, 100])
    if i % 500 == 0:
      samples = sess.run(G, feed_dict={z: np.random.uniform(-1., 1., [64, 100])})
    fig = plot(samples)
    plt.savefig('output/%s.png' % str(num_img).zfill(3), bbox_inches='tight')
    num_img += 1
    plt.close(fig)

  _, d_loss_print = sess.run([d_train_step, d_loss],
feed_dict={X: batch_X, z: batch_noise})
  _, g_loss_print = sess.run([g_train_step, g_loss],
feed_dict={z: batch_noise})

  if i % 100 == 0:
    s = sess.run(merged_summary, feed_dict={X: batch_X, z: batch_noise})
    writer.add_summary(s, i)
    print('epoch:%d g_loss:%f d_loss:%f' % (i, g_loss_print, d_loss_print))

  if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Train vanila GAN using convolutional networks')
    parser.add_argument('--logdir', type=str, default='1', help='logdir for Tensorboard, give a string')
    parser.add_argument('--batch_size', type=int, default=64, help='batch size: give a int')
    args = parser.parse_args()
    train(logdir=args.logdir, batch_size=args.batch_size)

git clone https://github.com/carpedm20/DCGAN-tensorflow

python download.py mnist celebA

python main.py --dataset celebA --input_height=108 --train --crop

python main.py --dataset celebA --input_height=108 --crop

$ mkdir data/DATASET_NAME
 ... add images to data/DATASET_NAME ...
 $ python main.py --dataset DATASET_NAME --train
 $ python main.py --dataset DATASET_NAME
 $ # example
 $ python main.py --dataset=eyes --input_fname_pattern="*_cropped.png" --train

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
%matplotlib inline

class VariationalAutoencoder(object):
  def __init__(self, network_architecture,   transfer_fct=tf.nn.softplus,
learning_rate=0.001, batch_size=100):
      self.network_architecture = network_architecture
      self.transfer_fct = transfer_fct
      self.learning_rate = learning_rate
      self.batch_size = batch_size
      # Place holder for the input
      self.x = tf.placeholder(tf.float32, [None,    network_architecture["n_input"]])
      # Define weights and biases
      network_weights = self._initialize_weights(**self.network_architecture)
      # Create autoencoder network
      # Use Encoder Network to determine mean and
      # (log) variance of Gaussian distribution in latent
      # space
      self.z_mean, self.z_log_sigma_sq = \
      self._encoder_network(network_weights["weights_encoder"],
    network_weights["biases_encoder"])
      # Draw one sample z from Gaussian distribution
      n_z = self.network_architecture["n_z"]
      eps = tf.random_normal((self.batch_size, n_z), 0, 1, dtype=tf.float32)
      # z = mu + sigma*epsilon
      self.z =       tf.add(self.z_mean,tf.multiply(tf.sqrt(tf.exp(self.z_log_sigma_sq)), eps))

      # Use Decoder network to determine mean of
      # Bernoulli distribution of reconstructed input
      self.x_reconstr_mean = \
      self._decoder_network(network_weights["weights_decoder"],
   network_weights["biases_decoder"])
      # Define loss function based variational upper-bound and 
      # corresponding optimizer
      # define generation loss
      generation_loss = \
  -tf.reduce_sum(self.x * tf.log(1e-10 + self.x_reconstr_mean)
+ (1-self.x) * tf.log(1e-10 + 1 - self.x_reconstr_mean), 1)
      latent_loss = -0.5 * tf.reduce_sum(1 + self.z_log_sigma_sq
- tf.square(self.z_mean)- tf.exp(self.z_log_sigma_sq), 1)
      self.cost = tf.reduce_mean(generation_loss + latent_loss)       #    average over batch
      # Define the optimizer
      self.optimizer = \
 tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(self.cost)
      # Initializing the tensor flow variables
      init = tf.global_variables_initializer()
  # Launch the session
      self.sess = tf.InteractiveSession()
      self.sess.run(init)

def _initialize_weights(self, n_hidden_recog_1, n_hidden_recog_2,
n_hidden_gener_1, n_hidden_gener_2,
n_input, n_z):
   initializer = tf.contrib.layers.xavier_initializer()
   all_weights = dict()
   all_weights['weights_encoder'] = {
   'h1': tf.Variable(initializer(shape=(n_input, n_hidden_recog_1))),
   'h2': tf.Variable(initializer(shape=(n_hidden_recog_1, n_hidden_recog_2))),
   'out_mean': tf.Variable(initializer(shape=(n_hidden_recog_2, n_z))),
   'out_log_sigma': tf.Variable(initializer(shape=(n_hidden_recog_2, n_z)))}
   all_weights['biases_encoder'] = {
   'b1': tf.Variable(tf.zeros([n_hidden_recog_1], dtype=tf.float32)),
   'b2': tf.Variable(tf.zeros([n_hidden_recog_2], dtype=tf.float32)),
   'out_mean': tf.Variable(tf.zeros([n_z], dtype=tf.float32)),
   'out_log_sigma': tf.Variable(tf.zeros([n_z], dtype=tf.float32))}

   all_weights['weights_decoder'] = {
   'h1': tf.Variable(initializer(shape=(n_z, n_hidden_gener_1))),
   'h2': tf.Variable(initializer(shape=(n_hidden_gener_1, n_hidden_gener_2))),
   'out_mean': tf.Variable(initializer(shape=(n_hidden_gener_2, n_input))),
   'out_log_sigma': tf.Variable(initializer(shape=(n_hidden_gener_2, n_input)))}

    all_weights['biases_decoder'] = {
   'b1': tf.Variable(tf.zeros([n_hidden_gener_1],    dtype=tf.float32)),
   'b2': tf.Variable(tf.zeros([n_hidden_gener_2], dtype=tf.float32)),'out_mean': tf.Variable(tf.zeros([n_input], dtype=tf.float32)),
   'out_log_sigma': tf.Variable(tf.zeros([n_input], dtype=tf.float32))}
   return all_weights

def _encoder_network(self, weights, biases):
  # Generate probabilistic encoder (recognition network), which
  # maps inputs onto a normal distribution in latent space.
  # The transformation is parametrized and can be learned.
  layer_1 = self.transfer_fct(tf.add(tf.matmul(self.x,     weights['h1']),
biases['b1']))
  layer_2 = self.transfer_fct(tf.add(tf.matmul(layer_1,   weights['h2']),
biases['b2']))
  z_mean = tf.add(tf.matmul(layer_2, weights['out_mean']),
biases['out_mean'])
  z_log_sigma_sq = \
tf.add(tf.matmul(layer_2, weights['out_log_sigma']),
biases['out_log_sigma'])
  return (z_mean, z_log_sigma_sq)

def _decoder_network(self, weights, biases):
  # Generate probabilistic decoder (decoder network), which
  # maps points in latent space onto a Bernoulli distribution in data space.
  # The transformation is parametrized and can be learned.
  layer_1 = self.transfer_fct(tf.add(tf.matmul(self.z, weights['h1']),
biases['b1']))
  layer_2 = self.transfer_fct(tf.add(tf.matmul(layer_1, weights['h2']),
biases['b2']))
  x_reconstr_mean = \
tf.nn.sigmoid(tf.add(tf.matmul(layer_2, weights['out_mean']),
  biases['out_mean']))
  return x_reconstr_mean

def fit(self, X):
  opt, cost = self.sess.run((self.optimizer, self.cost),
  feed_dict={self.x: X})
  return cost

def generate(self, z_mu=None):
""" Generate data by sampling from latent space.
If z_mu is not None, data for this point in latent space is
generated. Otherwise, z_mu is drawn from prior in latent
space.
"""
  if z_mu is None:
    z_mu = np.random.normal(size=self.network_architecture["n_z"])
# Note: This maps to mean of distribution, we could alternatively
# sample from Gaussian distribution
  return self.sess.run(self.x_reconstr_mean,
      feed_dict={self.z: z_mu})

def reconstruct(self, X):
""" Use VAE to reconstruct given data. """
  return self.sess.run(self.x_reconstr_mean,
    feed_dict={self.x: X})

def train(network_architecture, learning_rate=0.001,
batch_size=100, training_epochs=10, display_step=5):
  vae = VariationalAutoencoder(network_architecture,
  learning_rate=learning_rate,
  batch_size=batch_size)
  # Training cycle
  for epoch in range(training_epochs):
    avg_cost = 0.
    total_batch = int(n_samples / batch_size)
    # Loop over all batches
    for i in range(total_batch):
      batch_xs, _ = mnist.train.next_batch(batch_size)
      # Fit training using batch data
      cost = vae.fit(batch_xs)
      # Compute average loss
      avg_cost += cost / n_samples * batch_size
      # Display logs per epoch step
     if epoch % display_step == 0:
       print("Epoch:", '%04d' % (epoch+1), 
           "cost=", "{:.9f}".format(avg_cost))
  return vae

# Load MNIST data in a format suited for tensorflow.
# The script input_data is available under this URL:
#https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/examples/tutorials/mnist/input_data.py

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
n_samples = mnist.train.num_examples

network_architecture = \
dict(n_hidden_recog_1=500, # 1st layer encoder neurons
n_hidden_recog_2=500, # 2nd layer encoder neurons
n_hidden_gener_1=500, # 1st layer decoder neurons
n_hidden_gener_2=500, # 2nd layer decoder neurons
n_input=784, # MNIST data input (img shape: 28*28)
n_z=2) # dimensionality of latent space
vae = train(network_architecture, training_epochs=75)

x_sample = mnist.test.next_batch(100)[0]
x_reconstruct = vae.reconstruct(x_sample)
plt.figure(figsize=(8, 12))
for i in range(5):
  plt.subplot(5, 2, 2*i + 1)
  plt.imshow(x_sample[i].reshape(28, 28),  vmin=0, vmax=1, cmap="gray")
  plt.title("Test input")
  plt.colorbar()
  plt.subplot(5, 2, 2*i + 2)
  plt.imshow(x_reconstruct[i].reshape(28, 28), vmin=0, vmax=1, cmap="gray")
  plt.title("Reconstruction")
  plt.colorbar()
  plt.tight_layout()

nx = ny = 20
x_values = np.linspace(-3, 3, nx)
y_values = np.linspace(-3, 3, ny)
canvas = np.empty((28*ny, 28*nx))
for i, yi in enumerate(x_values):
  for j, xi in enumerate(y_values):
    z_mu = np.array([[xi, yi]]*vae.batch_size)
    x_mean = vae.generate(z_mu)
    canvas[(nx-i-1)*28:(nx-i)*28, j*28:(j+1)*28] = x_mean[0].reshape(28, 28)
plt.figure(figsize=(8, 10))
Xi, Yi = np.meshgrid(x_values, y_values)
plt.imshow(canvas, origin="upper", cmap="gray")
plt.tight_layout()

def transform(self, X):
    """Transform data by mapping it into the latent space."""
    # Note: This maps to mean of distribution, we could alternatively sample from Gaussian distribution
    return self.sess.run(self.z_mean,   feed_dict={self.x: X})

git clone https://github.com/naturomics/CapsNet-Tensorflow.git
 $ cd CapsNet-Tensorflow

mkdir -p data/mnist
wget -c -P data/mnist \\
http://yann.lecun.com/exdb/mnist/{train-images-idx3-ubyte.gz,train-labels-idx1-ubyte.gz,t10k-images-idx3-ubyte.gz,t10k-labels-idx1-ubyte.gz}
gunzip data/mnist/*.gz

python main.py

# capsLayer.py
#
import numpy as np
import tensorflow as tf
from config import cfg
epsilon = 1e-9
class CapsLayer(object):
''' Capsule layer.
Args:
input: A 4-D tensor.
num_outputs: the number of capsule in this layer.
vec_len: integer, the length of the output vector of a capsule.
layer_type: string, one of 'FC' or "CONV", the type of this layer,
fully connected or convolution, for the future expansion capability
with_routing: boolean, this capsule is routing with the
lower-level layer capsule.
Returns:
A 4-D tensor.
'''
def __init__(self, num_outputs, vec_len, with_routing=True, layer_type='FC'):
  self.num_outputs = num_outputs
  self.vec_len = vec_len
  self.with_routing = with_routing
  self.layer_type = layer_type

def __call__(self, input, kernel_size=None, stride=None):
'''
The parameters 'kernel_size' and 'stride' will be used while 'layer_type' equal 'CONV'
'''
  if self.layer_type == 'CONV':
    self.kernel_size = kernel_size
    self.stride = stride

    if not self.with_routing:
    # the PrimaryCaps layer, a convolutional layer
    # input: [batch_size, 20, 20, 256]
      assert input.get_shape() ==  [cfg.batch_size, 20, 20, 256]
      capsules = []
      for i in range(self.vec_len):
        # each capsule i: [batch_size, 6, 6, 32]
        with tf.variable_scope('ConvUnit_' + str(i)):
          caps_i = tf.contrib.layers.conv2d(input,      self.num_outputs,
self.kernel_size, self.stride,
padding="VALID")
          caps_i = tf.reshape(caps_i, shape=(cfg.batch_size, -1, 1, 1))
          capsules.append(caps_i)

      assert capsules[0].get_shape() == [cfg.batch_size, 1152, 1, 1]
# [batch_size, 1152, 8, 1]
      capsules = tf.concat(capsules, axis=2)
      capsules = squash(capsules)
      assert capsules.get_shape() == [cfg.batch_size, 1152, 8, 1]
      return(capsules)

  if self.layer_type == 'FC':
    if self.with_routing:
      # the DigitCaps layer, a fully connected layer
      # Reshape the input into [batch_size, 1152, 1, 8, 1]
      self.input = tf.reshape(input, shape=(cfg.batch_size, -1, 1, input.shape[-2].value, 1))
    with tf.variable_scope('routing'):
      # b_IJ: [1, num_caps_l, num_caps_l_plus_1, 1, 1]
      b_IJ = tf.constant(np.zeros([1, input.shape[1].value,    self.num_outputs, 1, 1], dtype=np.float32))
      capsules = routing(self.input, b_IJ)
      capsules = tf.squeeze(capsules, axis=1)
    return(capsules)

def routing(input, b_IJ):
''' The routing algorithm.
Args:
input: A Tensor with [batch_size, num_caps_l=1152, 1, length(u_i)=8, 1]
shape, num_caps_l meaning the number of capsule in the layer l.
Returns:
A Tensor of shape [batch_size, num_caps_l_plus_1, length(v_j)=16, 1]
representing the vector output `v_j` in the layer l+1
Notes:
u_i represents the vector output of capsule i in the layer l, and
v_j the vector output of capsule j in the layer l+1.
'''
  # W: [num_caps_j, num_caps_i, len_u_i, len_v_j]
  W = tf.get_variable('Weight', shape=(1, 1152, 10, 8, 16), dtype=tf.float32,
  initializer=tf.random_normal_initializer(stddev=cfg.stddev))
  # Eq.2, calc u_hat
  # do tiling for input and W before matmul
  # input => [batch_size, 1152, 10, 8, 1]
  # W => [batch_size, 1152, 10, 8, 16]
  input = tf.tile(input, [1, 1, 10, 1, 1])
  W = tf.tile(W, [cfg.batch_size, 1, 1, 1, 1])
  assert input.get_shape() == [cfg.batch_size, 1152, 10, 8, 1]

  # in last 2 dims:
  # [8, 16].T x [8, 1] => [16, 1] => [batch_size, 1152, 10, 16, 1]
  u_hat = tf.matmul(W, input, transpose_a=True)
  assert u_hat.get_shape() == [cfg.batch_size, 1152, 10, 16, 1]

  # line 3,for r iterations do
  for r_iter in range(cfg.iter_routing):
    with tf.variable_scope('iter_' + str(r_iter)):
      # line 4:
      # => [1, 1152, 10, 1, 1]
      c_IJ = tf.nn.softmax(b_IJ, dim=2)
      c_IJ = tf.tile(c_IJ, [cfg.batch_size, 1, 1, 1, 1])
      assert c_IJ.get_shape() == [cfg.batch_size, 1152, 10, 1, 1]
      # line 5:
      # weighting u_hat with c_IJ, element-wise in the last two dims
      # => [batch_size, 1152, 10, 16, 1]
      s_J = tf.multiply(c_IJ, u_hat)
      # then sum in the second dim, resulting in [batch_size, 1, 10, 16, 1]
      s_J = tf.reduce_sum(s_J, axis=1, keep_dims=True)
      assert s_J.get_shape() == [cfg.batch_size, 1, 10, 16, 16
      # line 6:
      # squash using Eq.1,
      v_J = squash(s_J)
      assert v_J.get_shape() == [cfg.batch_size, 1, 10, 16, 1]
      # line 7:
      # reshape & tile v_j from [batch_size ,1, 10, 16, 1] to [batch_size, 10, 1152, 16, 1]
      # then matmul in the last tow dim: [16, 1].T x [16, 1] => [1, 1], reduce mean in the
      # batch_size dim, resulting in [1, 1152, 10, 1, 1]
      v_J_tiled = tf.tile(v_J, [1, 1152, 1, 1, 1])
      u_produce_v = tf.matmul(u_hat, v_J_tiled, transpose_a=True)
      assert u_produce_v.get_shape() == [cfg.batch_size, 1152, 10, 1, 1]
      b_IJ += tf.reduce_sum(u_produce_v, axis=0, keep_dims=True)
  return(v_J)

def squash(vector):
'''Squashing function corresponding to Eq. 1
Args:
vector: A 5-D tensor with shape [batch_size, 1, num_caps, vec_len, 1],
Returns:
A 5-D tensor with the same shape as vector but squashed in 4rd and 5th dimensions.
'''
  vec_squared_norm = tf.reduce_sum(tf.square(vector), -2, keep_dims=True)
  scalar_factor = vec_squared_norm / (1 + vec_squared_norm) / tf.sqrt(vec_squared_norm + epsilon)
  vec_squashed = scalar_factor * vector # element-wise
return(vec_squashed)

#capsNet.py
#
import tensorflow as tf
from config import cfg
from utils import get_batch_data
from capsLayer import CapsLayer
epsilon = 1e-9

class CapsNet(object):
  def __init__(self, is_training=True):
    self.graph = tf.Graph()
    with self.graph.as_default():
      if is_training:
        self.X, self.labels = get_batch_data()
        self.Y = tf.one_hot(self.labels, depth=10, axis=1, dtype=tf.float32)
        self.build_arch()
        self.loss()
        self._summary()

        # t_vars = tf.trainable_variables()
        self.global_step = tf.Variable(0, name='global_step', trainable=False)
        self.optimizer = tf.train.AdamOptimizer()
        self.train_op =    self.optimizer.minimize(self.total_loss, global_step=self.global_step) # var_list=t_vars)

      elif cfg.mask_with_y:
        self.X = tf.placeholder(tf.float32,
          shape=(cfg.batch_size, 28, 28, 1))
        self.Y = tf.placeholder(tf.float32, shape=(cfg.batch_size, 10, 1))
        self.build_arch()
      else:
        self.X = tf.placeholder(tf.float32,
        shape=(cfg.batch_size, 28, 28, 1))
        self.build_arch()
      tf.logging.info('Setting up the main structure')

def build_arch(self):

  with tf.variable_scope('Conv1_layer'):
    # Conv1, [batch_size, 20, 20, 256]
    conv1 = tf.contrib.layers.conv2d(self.X, num_outputs=256,
       kernel_size=9, stride=1, 
       padding='VALID')
    assert conv1.get_shape() == [cfg.batch_size, 20, 20, 256]# Primary Capsules layer, return [batch_size, 1152, 8, 1]

  with tf.variable_scope('PrimaryCaps_layer'):
    primaryCaps = CapsLayer(num_outputs=32, vec_len=8,   with_routing=False, layer_type='CONV')
    caps1 = primaryCaps(conv1, kernel_size=9, stride=2)
    assert caps1.get_shape() == [cfg.batch_size, 1152, 8, 1]

  # DigitCaps layer, return [batch_size, 10, 16, 1]
  with tf.variable_scope('DigitCaps_layer'):
    digitCaps = CapsLayer(num_outputs=10, vec_len=16,   with_routing=True, layer_type='FC')
    self.caps2 = digitCaps(caps1)

  # Decoder structure in Fig. 2
  # 1\. Do masking, how:
  with tf.variable_scope('Masking'):
    # a). calc ||v_c||, then do softmax(||v_c||)
    # [batch_size, 10, 16, 1] => [batch_size, 10, 1, 1]
    self.v_length = tf.sqrt(tf.reduce_sum(tf.square(self.caps2),
axis=2, keep_dims=True) + epsilon)
    self.softmax_v = tf.nn.softmax(self.v_length, dim=1)
    assert self.softmax_v.get_shape() == [cfg.batch_size, 10, 1, 1]
    # b). pick out the index of max softmax val of the 10 caps
    # [batch_size, 10, 1, 1] => [batch_size] (index)
    self.argmax_idx = tf.to_int32(tf.argmax(self.softmax_v, axis=1))
    assert self.argmax_idx.get_shape() == [cfg.batch_size, 1, 1]
    self.argmax_idx = tf.reshape(self.argmax_idx, shape=(cfg.batch_size, )) .  
    # Method 1.
   if not cfg.mask_with_y:
     # c). indexing
     # It's not easy to understand the indexing process with  argmax_idx
     # as we are 3-dim animal
     masked_v = []
     for batch_size in range(cfg.batch_size):
       v = self.caps2[batch_size][self.argmax_idx[batch_size], :]
       masked_v.append(tf.reshape(v, shape=(1, 1, 16, 1)))
       self.masked_v = tf.concat(masked_v, axis=0)
       assert self.masked_v.get_shape() == [cfg.batch_size, 1, 16, 1]

   # Method 2\. masking with true label, default mode
   else:
     self.masked_v = tf.matmul(tf.squeeze(self.caps2), tf.reshape(self.Y, (-1, 10, 1)), transpose_a=True)
     self.v_length = tf.sqrt(tf.reduce_sum(tf.square(self.caps2), axis=2, keep_dims=True) + epsilon)

  # 2\. Reconstruct the MNIST images with 3 FC layers
  # [batch_size, 1, 16, 1] => [batch_size, 16] => [batch_size, 512] 
  with tf.variable_scope('Decoder'):
    vector_j = tf.reshape(self.masked_v, shape=(cfg.batch_size, -1))
    fc1 = tf.contrib.layers.fully_connected(vector_j, num_outputs=512)
    assert fc1.get_shape() == [cfg.batch_size, 512]
    fc2 = tf.contrib.layers.fully_connected(fc1, num_outputs=1024)
    assert fc2.get_shape() == [cfg.batch_size, 1024]
    self.decoded = tf.contrib.layers.fully_connected(fc2, num_outputs=784, activation_fn=tf.sigmoid)

def loss(self):
  # 1\. The margin loss
  # [batch_size, 10, 1, 1]
  # max_l = max(0, m_plus-||v_c||)^2
  max_l = tf.square(tf.maximum(0., cfg.m_plus - self.v_length))
  # max_r = max(0, ||v_c||-m_minus)^2
  max_r = tf.square(tf.maximum(0., self.v_length - cfg.m_minus))
  assert max_l.get_shape() == [cfg.batch_size, 10, 1, 1]

  # reshape: [batch_size, 10, 1, 1] => [batch_size, 10]
  max_l = tf.reshape(max_l, shape=(cfg.batch_size, -1))
  max_r = tf.reshape(max_r, shape=(cfg.batch_size, -1))
  # calc T_c: [batch_size, 10]
  T_c = self.Y
  # [batch_size, 10], element-wise multiply
  L_c = T_c * max_l + cfg.lambda_val * (1 - T_c) * max_r

  self.margin_loss = tf.reduce_mean(tf.reduce_sum(L_c, axis=1))

  # 2\. The reconstruction loss
  orgin = tf.reshape(self.X, shape=(cfg.batch_size, -1))
  squared = tf.square(self.decoded - orgin)
  self.reconstruction_err = tf.reduce_mean(squared)

  # 3\. Total loss
  # The paper uses sum of squared error as reconstruction   error, but we
  # have used reduce_mean in `# 2 The reconstruction loss` to calculate
  # mean squared error. In order to keep in line with the paper,the
  # regularization scale should be 0.0005*784=0.392
  self.total_loss = self.margin_loss + cfg.regularization_scale * self.reconstruction_err

#Summary
def _summary(self):
  train_summary = []
  train_summary.append(tf.summary.scalar('train/margin_loss', self.margin_loss))train_summary.append(tf.summary.scalar('train/reconstruction_loss', self.reconstruction_err))
  train_summary.append(tf.summary.scalar('train/total_loss', self.total_loss))
  recon_img = tf.reshape(self.decoded, shape=(cfg.batch_size, 28, 28, 1))
  train_summary.append(tf.summary.image('reconstruction_img', recon_img))
  correct_prediction = tf.equal(tf.to_int32(self.labels), self.argmax_idx)
  self.batch_accuracy = tf.reduce_sum(tf.cast(correct_prediction, tf.float32))
  self.test_acc = tf.placeholder_with_default(tf.constant(0.), shape=[])
  test_summary = []
  test_summary.append(tf.summary.scalar('test/accuracy', self.test_acc))
  self.train_summary = tf.summary.merge(train_summary)
  self.test_summary = tf.summary.merge(test_summary)