How Do I Store And Rebuild And Dictionary Of Weights In Tensorflow

When training I store my weights in a dictionary of tensorflow-variables. I pass that dictionary of weights to a 'model'-function together with some data to get my desired output.

Solution 1:

I edited your code to make it work - One possible way ! Check it.

import tensorflow as tf
import numpy as np

# first train a linear model on random vectors of length 5 and store the trained parameters.# Then load those parameters and try to apply them to a new vector.defrun():
    train_model()
    apply_model()

deftrain_model():
     # create random training data: 100 vectors of length 5 for both input and output.
     train_data  = np.random.random((100,5))
     train_labels = np.random.random((100,5))

     train_data_node = tf.placeholder(tf.float32, shape=(5), name="train_data_node")
     train_labels_node = tf.placeholder(tf.float32, shape=(5), name="train_labels_node")

     weights = defineWeights()

     prediction = model(train_data_node, weights)
     prediction = tf.identity(prediction, name="prediction")

     loss = tf.norm(prediction - train_labels_node)
     train_op = tf.train.AdagradOptimizer(learning_rate=1).minimize(loss)

     saver = tf.train.Saver()

     sess = tf.Session()
     sess.run(tf.global_variables_initializer())
     # train for 50 epochs on all 100 training examples, with a batchsize of 1.for _ inrange(50):
        for i inrange(100):
            batch_data = train_data[i,:]
            batch_labels = train_labels[i,:]

            feed_dict = {train_data_node: batch_data, train_labels_node: batch_labels}
            sess.run([train_op, loss, weights], feed_dict=feed_dict)
            saver.save(sess, 'results/model')
     print("Trained Weights")
     print(sess.run(weights))

defapply_model():
    sess = tf.Session()

    new_saver = tf.train.import_meta_graph('results/model.meta')
    new_saver.restore(sess, tf.train.latest_checkpoint('results'))


    print("Loaded Weights")
    print(sess.run(['a:0','b:0']))

    prediction = tf.get_default_graph().get_tensor_by_name("prediction:0")
    train_data_node = tf.get_default_graph().get_tensor_by_name("train_data_node:0")

    test_data = np.random.random(5).astype(np.float32)
    pred = sess.run([prediction],feed_dict={train_data_node:test_data})
    print("Prediction")
    print(pred)


defmodel(data, weights):
    # multiply the matrix weights['a'] with the vector data
    l1 = tf.matmul(tf.expand_dims(data,0), weights['a'])
    l1 = l1 + weights['b']
    return l1

defdefineWeights():
    weights = {
       'a': tf.Variable(tf.random_normal([5, 5],
                                                    stddev=0.01, 
                                                    dtype =  tf.float32),
                                                    name = 'a'),
       'b': tf.Variable(tf.random_normal([5]), name = 'b'),
    }
return weights

defmain(_):
    run()

if __name__ == '__main__':
    tf.app.run(main=main)

Output:

Trained Weights
{'a': array([[ 0.01243415, -0.42879951,  0.0174435 , -0.24622701,  0.35309449],
   [ 0.03154161, -0.08194152,  0.09223857, -0.15719411, -0.06323836],
   [-0.03263358,  0.05096304,  0.1769278 , -0.17564282,  0.04325204],
   [-0.17412457, -0.00338688,  0.08468977, -0.06877152, -0.02180972],
   [ 0.25160244, -0.19224152,  0.14535131, -0.20594895, -0.03813718]], dtype=float32), 'b': array([ 0.33825615,  0.79861975,  0.30609566,  0.91897982,  0.20577262], dtype=float32)}
I tensorflow/core/common_runtime/gpu/gpu_device.cc:975] Creating TensorFlow device (/gpu:0) -> (device: 0, name: GeForce GTX 10606GB, pci bus id: 0000:01:00.0)
Loaded Weights
[array([[ 0.01243415, -0.42879951,  0.0174435 , -0.24622701,  0.35309449],
   [ 0.03154161, -0.08194152,  0.09223857, -0.15719411, -0.06323836],
   [-0.03263358,  0.05096304,  0.1769278 , -0.17564282,  0.04325204],
   [-0.17412457, -0.00338688,  0.08468977, -0.06877152, -0.02180972],
   [ 0.25160244, -0.19224152,  0.14535131, -0.20594895, -0.03813718]], dtype=float32), array([ 0.33825615,  0.79861975,  0.30609566,  0.91897982,  0.20577262], dtype=float32)]
Prediction
[array([[ 0.3465074 ,  0.42139536,  0.71310139,  0.30854774,  0.32671657]], dtype=float32)]

Explanation:

1. Name the tensors which you want to access after restoring.
2. Restore the graph and restore variables that you named - shown in apply_model()
3. Feed the new test_data into placeholder using feed_dict

Issues:

1. I tried to use sess.run(tf.global_variables_initializer()) but it is re-initializing variables to new random values. (Using TF 1.0)

I hope this helps !