# HG changeset patch
# User zgu
# Date 1409856631 14400
# Node ID ed3d653e4012d472a4c2b6107b7f152b530c3f34
# Parent  9b8bd21b6823ec3cec5743d2d952f93153d09a86
8055289: Internal Error: mallocTracker.cpp:146 fatal error: Should not use malloc for big memory block, use virtual memory instead
Summary: Return NULL if memory allocation size is bigger than MAX_MALLOC_SIZE when NMT is on
Reviewed-by: coleenp, gtriantafill

diff -r 9b8bd21b6823 -r ed3d653e4012 src/share/vm/runtime/os.cpp
--- a/src/share/vm/runtime/os.cpp	Thu Sep 11 14:21:24 2014 +0200
+++ b/src/share/vm/runtime/os.cpp	Thu Sep 04 14:50:31 2014 -0400
@@ -50,6 +50,7 @@
 #include "runtime/thread.inline.hpp"
 #include "services/attachListener.hpp"
 #include "services/nmtCommon.hpp"
+#include "services/mallocTracker.hpp"
 #include "services/memTracker.hpp"
 #include "services/threadService.hpp"
 #include "utilities/defaultStream.hpp"
@@ -570,6 +571,17 @@
   NOT_PRODUCT(inc_stat_counter(&num_mallocs, 1));
   NOT_PRODUCT(inc_stat_counter(&alloc_bytes, size));
 
+#if INCLUDE_NMT
+  // NMT can not track malloc allocation size > MAX_MALLOC_SIZE, which is
+  // (1GB - 1) on 32-bit system. It is not an issue on 64-bit system, where
+  // MAX_MALLOC_SIZE = ((1 << 62) - 1).
+  // VM code does not have such large malloc allocation. However, it can come
+  // Unsafe call.
+  if (MemTracker::tracking_level() >= NMT_summary && size > MAX_MALLOC_SIZE) {
+    return NULL;
+  }
+#endif
+
 #ifdef ASSERT
   // checking for the WatcherThread and crash_protection first
   // since os::malloc can be called when the libjvm.{dll,so} is
@@ -640,6 +652,13 @@
 }
 
 void* os::realloc(void *memblock, size_t size, MEMFLAGS memflags, const NativeCallStack& stack) {
+#if INCLUDE_NMT
+  // See comments in os::malloc() above
+  if (MemTracker::tracking_level() >= NMT_summary && size > MAX_MALLOC_SIZE) {
+    return NULL;
+  }
+#endif
+
 #ifndef ASSERT
   NOT_PRODUCT(inc_stat_counter(&num_mallocs, 1));
   NOT_PRODUCT(inc_stat_counter(&alloc_bytes, size));
diff -r 9b8bd21b6823 -r ed3d653e4012 test/TEST.groups
--- a/test/TEST.groups	Thu Sep 11 14:21:24 2014 +0200
+++ b/test/TEST.groups	Thu Sep 04 14:50:31 2014 -0400
@@ -90,6 +90,7 @@
   runtime/NMT/SummarySanityCheck.java \
   runtime/NMT/ThreadedMallocTestType.java \
   runtime/NMT/ThreadedVirtualAllocTestType.java \
+  runtime/NMT/UnsafeMallocLimit.java \
   runtime/NMT/VirtualAllocCommitUncommitRecommit.java \
   runtime/NMT/VirtualAllocTestType.java \
   runtime/RedefineObject/TestRedefineObject.java \
diff -r 9b8bd21b6823 -r ed3d653e4012 test/runtime/NMT/UnsafeMallocLimit.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/runtime/NMT/UnsafeMallocLimit.java	Thu Sep 04 14:50:31 2014 -0400
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8055289
+ * @library /testlibrary
+ * @build UnsafeMallocLimit
+ * @run main/othervm -Xmx32m -XX:NativeMemoryTracking=summary UnsafeMallocLimit
+ */
+
+import com.oracle.java.testlibrary.*;
+import sun.misc.Unsafe;
+
+public class UnsafeMallocLimit {
+
+    public static void main(String args[]) throws Exception {
+        if (Platform.is32bit()) {
+            Unsafe unsafe = Utils.getUnsafe();
+            try {
+                unsafe.allocateMemory(1 << 30);
+                throw new RuntimeException("Did not get expected OOME");
+            } catch (OutOfMemoryError e) {
+                // Expected exception
+            }
+        } else {
+            System.out.println("Test only valid on 32-bit platforms");
+        }
+    }
+}