INNER CODE UNIT · Python

cross_entropy

ematvey/hierarchical-attention-networks · bn_lstm_test.py:37

cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))

optimizer = tf.train.AdamOptimizer()
gvs = optimizer.compute_gradients(cross_entropy)
capped_gvs = [(None if grad is None else tf.clip_by_value(grad, -1., 1.), var) for grad, var in gvs]
train_step = optimizer.apply_gradients(capped_gvs)

correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# Summaries
tf.summary.scalar("accuracy", accuracy)
tf.summary.scalar("xe_loss", cross_entropy)
for (grad, var), (capped_grad, _) in zip(gvs, capped_gvs):
    if grad is not None:
        tf.summary.histogram('grad/{}'.format(var.name), capped_grad)
        tf.summary.histogram('capped_fraction/{}'.format(var.name),
            tf.nn.zero_fraction(grad - capped_grad))

View source record →