8318127: align_up has potential overflow

Reviewed-by: kbarrett, dlong

src/hotspot/share/utilities/align.hpp

@@ -72,7 +72,9 @@ constexpr T align_down(T size, A alignment) {
 
 template<typename T, typename A, ENABLE_IF(std::is_integral<T>::value)>
 constexpr T align_up(T size, A alignment) {
-  T adjusted = checked_cast<T>(size + alignment_mask(alignment));
+  T mask = checked_cast<T>(alignment_mask(alignment));
+  assert(size <= std::numeric_limits<T>::max() - mask, "overflow");
+  T adjusted = size + mask;
   return align_down(adjusted, alignment);
 }
 

test/hotspot/gtest/runtime/test_os_reserve_between.cpp

@@ -304,12 +304,12 @@ TEST_VM(os, attempt_reserve_memory_randomization_cornercases) {
   // Zero-sized range
   test_attempt_reserve_memory_between(nullptr, nullptr, ps, ag, false, Expect::failure());
   test_attempt_reserve_memory_between((char*)(3 * G), (char*)(3 * G), ps, ag, false, Expect::dontcare(), __LINE__);
-  test_attempt_reserve_memory_between((char*)SIZE_MAX, (char*)SIZE_MAX, ps, ag, false, Expect::failure(), __LINE__);
+  test_attempt_reserve_memory_between((char*)SIZE_MAX-ag, (char*)SIZE_MAX-ag, ps, ag, false, Expect::failure(), __LINE__);
 
   test_attempt_reserve_memory_between(nullptr, nullptr, ps, ag, true, Expect::failure());
   test_attempt_reserve_memory_between((char*)(3 * G), (char*)(3 * G), ps, ag, true, Expect::dontcare(), __LINE__);
   test_attempt_reserve_memory_between((char*)(3 * G), (char*)(3 * G), ps, ag, true, Expect::dontcare(), __LINE__);
-  test_attempt_reserve_memory_between((char*)SIZE_MAX, (char*)SIZE_MAX, ps, ag, true, Expect::failure(), __LINE__);
+  test_attempt_reserve_memory_between((char*)SIZE_MAX-ag, (char*)SIZE_MAX-ag, ps, ag, true, Expect::failure(), __LINE__);
 
   // Full size
   // Note: paradoxically, success is not guaranteed here, since a significant portion of the attach points

test/hotspot/gtest/utilities/test_align.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,14 +23,15 @@
 
 #include "precompiled.hpp"
 #include "utilities/align.hpp"
+#include "utilities/checkedCast.hpp"
 #include "utilities/formatBuffer.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "unittest.hpp"
 
 #include <limits>
 
 // A few arbitrarily chosen values to test the align functions on.
-static constexpr uint64_t values[] = {1, 3, 10, 345, 1023, 1024, 1025, 23909034, INT_MAX, uint64_t(-1) / 2, uint64_t(-1) / 2 + 100, uint64_t(-1)};
+static constexpr uint64_t values[] = {1, 3, 10, 345, 1023, 1024, 1025, 23909034, INT_MAX, uint64_t(-1) / 2, uint64_t(-1) / 2 + 100, ~(uint64_t(1) << 62)};
 
 template <typename T>
 static constexpr T max_alignment() {
@@ -195,3 +196,33 @@ TEST(Align, alignments) {
 
   test_alignments<int8_t, int8_t>();
 }
+
+#ifdef ASSERT
+template <typename T, typename A>
+static void test_fail_alignment() {
+  A alignment = max_alignment<A>();
+  T value = align_down(std::numeric_limits<T>::max(), alignment) + 1;
+  // Aligning up would overflow, as there is not enough room for alignment
+  T aligned = align_up(value, alignment);
+}
+
+TEST_VM_ASSERT(Align, fail_alignments_same_size) {
+  test_fail_alignment<uint64_t, uint64_t>();
+}
+
+TEST_VM_ASSERT(Align, fail_alignments_unsigned_signed) {
+  test_fail_alignment<uint32_t, int32_t>();
+}
+
+TEST_VM_ASSERT(Align, fail_alignments_signed_unsigned) {
+  test_fail_alignment<int64_t, int32_t>();
+}
+
+TEST_VM_ASSERT(Align, fail_alignments_small_large) {
+  test_fail_alignment<uint8_t, uint64_t>();
+}
+
+TEST_VM_ASSERT(Align, fail_alignments_large_small) {
+  test_fail_alignment<uint64_t, uint8_t>();
+}
+#endif // ASSERT