[Hotfix] miss parentheses when calcuating the coef of llama3 rope (#2157

)
InternLM · Jul 26, 2024 · 148f629 · 148f629
1 parent 7199b4e
commit 148f629
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/turbomind/models/llama/unified_attention_layer.cc b/src/turbomind/models/llama/unified_attention_layer.cc
@@ -246,7 +246,7 @@ inline void UnifiedAttentionLayer<T>::forward(TensorMap* outputs, const TensorMa
             const double PI                   = 3.14159265358979323846;
             float        inv_diff_freq_factor = 1.0 / (params_.high_freq_factor - params_.low_freq_factor);
             params.llama3_inv_scaling_factor  = 1.0 / params_.rope_scaling_factor;
-            params.llama3_alpha = params_.original_max_position_embeddings / 2 * PI * inv_diff_freq_factor;
+            params.llama3_alpha = params_.original_max_position_embeddings / (2 * PI) * inv_diff_freq_factor;
             params.llama3_beta  = params_.low_freq_factor * inv_diff_freq_factor;
         }