mnist_vae.prototxt

name: "MNISTVariationalAutoencoder"
layer {
  name: "data"
  type: "Data"
  top: "data"
  include {
    phase: TRAIN
  }
  transform_param {
    scale: 0.0039215684
  }
  data_param {
    source: "$CAFFE_PATH/examples/mnist/mnist_train_lmdb"
    batch_size: 100
    backend: LMDB
  }
}
layer {
  name: "flatdata"
  type: "Flatten"
  bottom: "data"
  top: "flatdata"
  include {
    phase: TRAIN
  }
}
layer {
  name: "encode1"
  type: "InnerProduct"
  bottom: "data"
  top: "encode1"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 1
    decay_mult: 0
  }
  inner_product_param {
    num_output: 1000
    weight_filler {
      type: "gaussian"
      std: 0.1
      sparse: 15
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
  include {
    phase: TRAIN
  }
}
layer {
  name: "encode1neuron"
  type: "ReLU"
  bottom: "encode1"
  top: "encode1neuron"
  include {
    phase: TRAIN
  }
}
layer {
  name: "encode2"
  type: "InnerProduct"
  bottom: "encode1neuron"
  top: "encode2"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 1
    decay_mult: 0
  }
  inner_product_param {
    num_output: 500
    weight_filler {
      type: "gaussian"
      std: 0.1
      sparse: 15
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
  include {
    phase: TRAIN
  }
}
layer {
  name: "encode2neuron"
  type: "ReLU"
  bottom: "encode2"
  top: "encode2neuron"
  include {
    phase: TRAIN
  }
}
layer {
  name: "encode3"
  type: "InnerProduct"
  bottom: "encode2neuron"
  top: "encode3"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 1
    decay_mult: 0
  }
  inner_product_param {
    num_output: 250
    weight_filler {
      type: "gaussian"
      std: 0.1
      sparse: 15
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
  include {
    phase: TRAIN
  }
}
layer {
  name: "encode3neuron"
  type: "ReLU"
  bottom: "encode3"
  top: "encode3neuron"
  include {
    phase: TRAIN
  }
}

# end encoder, begin VAE z definition

layer {
  name: "mu"
  type: "InnerProduct"
  bottom: "encode3neuron"
  top: "mu"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 1
    decay_mult: 0
  }
  inner_product_param {
    num_output: 30 # num z's
    weight_filler {
      type: "gaussian"
      std: 0.1
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
  include {
    phase: TRAIN
  }
}
# Predict log sd because sd needs to be
# positive, and the exp ensures that it is.
layer {
  name: "logsd"
  type: "InnerProduct"
  bottom: "encode3"
  top: "logsd"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 1
    decay_mult: 0
  }
  inner_product_param {
    num_output: 30 # num z's
    weight_filler {
      type: "gaussian"
      std: 0.1
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
  include {
    phase: TRAIN
  }
}
layer{
  name: "sd"
  type: "Exp"
  bottom: "logsd"
  top: "sd"
  include {
    phase: TRAIN
  }
}
layer{
  name: "var"
  type: "Eltwise"
  bottom: "sd"
  bottom: "sd"
  top: "var"
  eltwise_param{
    operation: PROD
  }
  include {
    phase: TRAIN
  }
}
layer{
  name: "meansq"
  type: "Eltwise"
  bottom: "mu"
  bottom: "mu"
  top: "meansq"
  eltwise_param{
    operation: PROD
  }
  include {
    phase: TRAIN
  }
}
layer{
  name: "kldiv_plus_half"
  type: "Eltwise"
  bottom: "meansq"
  bottom: "var"
  bottom: "logsd"
  top: "kldiv_plus_half"
  eltwise_param{
    operation: SUM
    coeff: 0.5
    coeff: 0.5
    coeff: -1.0
  }
  include {
    phase: TRAIN
  }
}
layer {
  name: "kldiv"
  type: "Power"
  bottom: "kldiv_plus_half"
  top: "kldiv"
  power_param{
    shift: -0.5
  }
  include {
    phase: TRAIN
  }
}
layer{
  name: "klloss"
  type: "Reduction"
  bottom: "kldiv"
  top: "klloss"
  include {
    phase: TRAIN
  }
  loss_weight: 0.01 # 1 over batch_size, because 
                    # SigmoidCrossEntropyLoss
                    # normalizes by batch_size but
                    # Reduction does not.
}
layer{
  name: "mu_dummy" # can't call this 'mu' or
                   # caffe will try to copy 
                   # mu's parameters into 
                   # this layer at test time
  type: "DummyData"
  top: "mu"
  dummy_data_param{
    shape {
      dim: 100 # test-time batch_size
      dim: 30 # num z's
    }
    data_filler{
      type: "constant"
      value: 0
    }
  }
  include {
    phase: TEST
  }
}
layer{
  name: "sd"
  type: "DummyData"
  top: "sd"
  dummy_data_param{
    shape {
      dim: 100 # test-time batch_size
      dim: 30 # num z's
    }
    data_filler{
      type: "constant"
      value: 1
    }
  }
  include {
    phase: TEST
  }
}
layer{
  name: "noise"
  type: "DummyData"
  top: "noise"
  dummy_data_param{
    shape {
      dim: 100 # train batch_size
      dim: 30 # num z's
    }
    data_filler{
      type: "gaussian"
      std: 1.
    }
  }
  include {
    phase: TRAIN
  }
}
layer{
  name: "noise"
  type: "DummyData"
  top: "noise"
  dummy_data_param{
    shape {
      dim: 100 # test batch_size
      dim: 30 # num z's
    }
    data_filler{
      type: "gaussian"
      std: 1.
    }
  }
  include {
    phase: TEST
  }
}
layer{
  name: "sdnoise"
  type: "Eltwise"
  bottom: "noise"
  bottom: "sd"
  top: "sdnoise"
  eltwise_param{
    operation: PROD
  }
}
layer{
  name: "sample"
  type: "Eltwise"
  bottom: "mu"
  bottom: "sdnoise"
  top: "sample"
  eltwise_param{
    operation: SUM
  }
}

# end VAE z's definition, begin decoder

layer {
  name: "decode4"
  type: "InnerProduct"
  bottom: "sample"
  top: "decode4"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 1
    decay_mult: 0
  }
  inner_product_param {
    num_output: 250
    weight_filler {
      type: "gaussian"
      std: 0.1
      sparse: 15
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}
layer {
  name: "decode4neuron"
  type: "ReLU"
  bottom: "decode4"
  top: "decode4neuron"
}
layer {
  name: "decode3"
  type: "InnerProduct"
  bottom: "decode4neuron"
  top: "decode3"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 1
    decay_mult: 0
  }
  inner_product_param {
    num_output: 500
    weight_filler {
      type: "gaussian"
      std: 0.1
      sparse: 15
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}
layer {
  name: "decode3neuron"
  type: "ReLU"
  bottom: "decode3"
  top: "decode3neuron"
}
layer {
  name: "decode2"
  type: "InnerProduct"
  bottom: "decode3neuron"
  top: "decode2"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 1
    decay_mult: 0
  }
  inner_product_param {
    num_output: 1000
    weight_filler {
      type: "gaussian"
      std: 0.1
      sparse: 15
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}
layer {
  name: "decode2neuron"
  type: "ReLU"
  bottom: "decode2"
  top: "decode2neuron"
}
layer {
  name: "decode1"
  type: "InnerProduct"
  bottom: "decode2neuron"
  top: "decode1"
  param {
    lr_mult: 1
    decay_mult: 1
  }
  param {
    lr_mult: 1
    decay_mult: 0
  }
  inner_product_param {
    num_output: 784
    weight_filler {
      type: "gaussian"
      std: 0.1
      sparse: 15
    }
    bias_filler {
      type: "constant"
      value: 0
    }
  }
}

# We use a SigmoidCrossEntropyLoss because the data is constrained between
# 0 and 1.  See section 4.1 of the tutorial for a probabilistic interpretation.
layer {
  name: "loss"
  type: "SigmoidCrossEntropyLoss"
  bottom: "decode1"
  bottom: "flatdata"
  top: "cross_entropy_loss"
  loss_weight: 1
  include {
    phase: TRAIN
  }
}

# Include a Euclidean loss for reference.  Note that the loss weight is 0,
# so this loss has no effect on training.
layer {
  name: "decode1neuron"
  type: "Sigmoid"
  bottom: "decode1"
  top: "decode1neuron"
}
layer {
  name: "loss"
  type: "EuclideanLoss"
  bottom: "decode1neuron"
  bottom: "flatdata"
  top: "l2_error"
  loss_weight: 0
  include {
    phase: TRAIN
  }
}