From d46d0eb6c67bf0531b331c5f47ae535b28520405 Mon Sep 17 00:00:00 2001 From: haozhuolin Date: Sun, 3 Mar 2019 16:46:18 +0800 Subject: [PATCH] fix position_encoding bug, the result of dividing the integer is 0. --- modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules.py b/modules.py index 16df087..a839231 100644 --- a/modules.py +++ b/modules.py @@ -283,7 +283,7 @@ def positional_encoding(inputs, # First part of the PE function: sin and cos argument position_enc = np.array([ - [pos / np.power(10000, (i-i%2)/E) for i in range(E)] + [pos / np.power(10000, 1.0*(i-i%2)/E) for i in range(E)] for pos in range(maxlen)]) # Second part, apply the cosine to even columns and sin to odds. @@ -308,4 +308,4 @@ def noam_scheme(init_lr, global_step, warmup_steps=4000.): until it reaches init_lr. ''' step = tf.cast(global_step + 1, dtype=tf.float32) - return init_lr * warmup_steps ** 0.5 * tf.minimum(step * warmup_steps ** -1.5, step ** -0.5) \ No newline at end of file + return init_lr * warmup_steps ** 0.5 * tf.minimum(step * warmup_steps ** -1.5, step ** -0.5)