# HG changeset patch
# User rbackman
# Date 1360745179 -3600
# Node ID 2394a89e89f42e3c9b1eceab3090050b1072abce
# Parent  7adae9244bc8d01f06dee00272498d79cba76900
8008088: SA can hang the VM
Reviewed-by: mgronlun, sla, dholmes

diff -r 7adae9244bc8 -r 2394a89e89f4 agent/src/os/bsd/libproc_impl.c
--- a/agent/src/os/bsd/libproc_impl.c	Wed Feb 13 11:23:46 2013 +0100
+++ b/agent/src/os/bsd/libproc_impl.c	Wed Feb 13 09:46:19 2013 +0100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -91,6 +91,14 @@
    }
 }
 
+void print_error(const char* format,...) {
+  va_list alist;
+  va_start(alist, format);
+  fputs("ERROR: ", stderr);
+  vfprintf(stderr, format, alist);
+  va_end(alist);
+}
+
 bool is_debug() {
    return _libsaproc_debug;
 }
diff -r 7adae9244bc8 -r 2394a89e89f4 agent/src/os/bsd/libproc_impl.h
--- a/agent/src/os/bsd/libproc_impl.h	Wed Feb 13 11:23:46 2013 +0100
+++ b/agent/src/os/bsd/libproc_impl.h	Wed Feb 13 09:46:19 2013 +0100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -107,6 +107,7 @@
 int pathmap_open(const char* name);
 
 void print_debug(const char* format,...);
+void print_error(const char* format,...);
 bool is_debug();
 
 typedef bool (*thread_info_callback)(struct ps_prochandle* ph, pthread_t pid, lwpid_t lwpid);
diff -r 7adae9244bc8 -r 2394a89e89f4 agent/src/os/bsd/ps_proc.c
--- a/agent/src/os/bsd/ps_proc.c	Wed Feb 13 11:23:46 2013 +0100
+++ b/agent/src/os/bsd/ps_proc.c	Wed Feb 13 09:46:19 2013 +0100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -129,42 +129,66 @@
   return (errno == 0)? true: false;
 }
 
+static bool ptrace_continue(pid_t pid, int signal) {
+  // pass the signal to the process so we don't swallow it
+  if (ptrace(PTRACE_CONT, pid, NULL, signal) < 0) {
+    print_debug("ptrace(PTRACE_CONT, ..) failed for %d\n", pid);
+    return false;
+  }
+  return true;
+}
+
+// waits until the ATTACH has stopped the process
+// by signal SIGSTOP
+static bool ptrace_waitpid(pid_t pid) {
+  int ret;
+  int status;
+  do {
+    // Wait for debuggee to stop.
+    ret = waitpid(pid, &status, 0);
+    if (ret >= 0) {
+      if (WIFSTOPPED(status)) {
+        // Any signal will stop the thread, make sure it is SIGSTOP. Otherwise SIGSTOP
+        // will still be pending and delivered when the process is DETACHED and the process
+        // will go to sleep.
+        if (WSTOPSIG(status) == SIGSTOP) {
+          // Debuggee stopped by SIGSTOP.
+          return true;
+        }
+        if (!ptrace_continue(pid, WSTOPSIG(status))) {
+          print_error("Failed to correctly attach to VM. VM might HANG! [PTRACE_CONT failed, stopped by %d]\n", WSTOPSIG(status));
+          return false;
+        }
+      } else {
+        print_debug("waitpid(): Child process exited/terminated (status = 0x%x)\n", status);
+        return false;
+      }
+    } else {
+      switch (errno) {
+        case EINTR:
+          continue;
+          break;
+        case ECHILD:
+          print_debug("waitpid() failed. Child process pid (%d) does not exist \n", pid);
+          break;
+        case EINVAL:
+          print_debug("waitpid() failed. Invalid options argument.\n");
+          break;
+        default:
+          print_debug("waitpid() failed. Unexpected error %d\n",errno);
+      }
+      return false;
+    }
+  } while(true);
+}
+
 // attach to a process/thread specified by "pid"
 static bool ptrace_attach(pid_t pid) {
   if (ptrace(PT_ATTACH, pid, NULL, 0) < 0) {
     print_debug("ptrace(PTRACE_ATTACH, ..) failed for %d\n", pid);
     return false;
   } else {
-    int ret;
-    int status;
-    do {
-      // Wait for debuggee to stop.
-      ret = waitpid(pid, &status, 0);
-      if (ret >= 0) {
-        if (WIFSTOPPED(status)) {
-          // Debuggee stopped.
-          return true;
-        } else {
-          print_debug("waitpid(): Child process exited/terminated (status = 0x%x)\n", status);
-          return false;
-        }
-      } else {
-        switch (errno) {
-          case EINTR:
-            continue;
-            break;
-          case ECHILD:
-            print_debug("waitpid() failed. Child process pid (%d) does not exist \n", pid);
-            break;
-          case EINVAL:
-            print_debug("waitpid() failed. Invalid options argument.\n");
-            break;
-          default:
-            print_debug("waitpid() failed. Unexpected error %d\n",errno);
-        }
-        return false;
-      }
-    } while(true);
+    return ptrace_waitpid(pid);
   }
 }
 
diff -r 7adae9244bc8 -r 2394a89e89f4 agent/src/os/linux/libproc_impl.c
--- a/agent/src/os/linux/libproc_impl.c	Wed Feb 13 11:23:46 2013 +0100
+++ b/agent/src/os/linux/libproc_impl.c	Wed Feb 13 09:46:19 2013 +0100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -92,6 +92,14 @@
    }
 }
 
+void print_error(const char* format,...) {
+  va_list alist;
+  va_start(alist, format);
+  fputs("ERROR: ", stderr);
+  vfprintf(stderr, format, alist);
+  va_end(alist);
+}
+
 bool is_debug() {
    return _libsaproc_debug;
 }
diff -r 7adae9244bc8 -r 2394a89e89f4 agent/src/os/linux/libproc_impl.h
--- a/agent/src/os/linux/libproc_impl.h	Wed Feb 13 11:23:46 2013 +0100
+++ b/agent/src/os/linux/libproc_impl.h	Wed Feb 13 09:46:19 2013 +0100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -105,6 +105,7 @@
 int pathmap_open(const char* name);
 
 void print_debug(const char* format,...);
+void print_error(const char* format,...);
 bool is_debug();
 
 typedef bool (*thread_info_callback)(struct ps_prochandle* ph, pthread_t pid, lwpid_t lwpid);
diff -r 7adae9244bc8 -r 2394a89e89f4 agent/src/os/linux/ps_proc.c
--- a/agent/src/os/linux/ps_proc.c	Wed Feb 13 11:23:46 2013 +0100
+++ b/agent/src/os/linux/ps_proc.c	Wed Feb 13 09:46:19 2013 +0100
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <signal.h>
 #include <errno.h>
 #include <sys/ptrace.h>
 #include "libproc_impl.h"
@@ -142,46 +143,71 @@
 
 }
 
+static bool ptrace_continue(pid_t pid, int signal) {
+  // pass the signal to the process so we don't swallow it
+  if (ptrace(PTRACE_CONT, pid, NULL, signal) < 0) {
+    print_debug("ptrace(PTRACE_CONT, ..) failed for %d\n", pid);
+    return false;
+  }
+  return true;
+}
+
+// waits until the ATTACH has stopped the process
+// by signal SIGSTOP
+static bool ptrace_waitpid(pid_t pid) {
+  int ret;
+  int status;
+  while (true) {
+    // Wait for debuggee to stop.
+    ret = waitpid(pid, &status, 0);
+    if (ret == -1 && errno == ECHILD) {
+      // try cloned process.
+      ret = waitpid(pid, &status, __WALL);
+    }
+    if (ret >= 0) {
+      if (WIFSTOPPED(status)) {
+        // Any signal will stop the thread, make sure it is SIGSTOP. Otherwise SIGSTOP
+        // will still be pending and delivered when the process is DETACHED and the process
+        // will go to sleep.
+        if (WSTOPSIG(status) == SIGSTOP) {
+          // Debuggee stopped by SIGSTOP.
+          return true;
+        }
+        if (!ptrace_continue(pid, WSTOPSIG(status))) {
+          print_error("Failed to correctly attach to VM. VM might HANG! [PTRACE_CONT failed, stopped by %d]\n", WSTOPSIG(status));
+          return false;
+        }
+      } else {
+        print_debug("waitpid(): Child process exited/terminated (status = 0x%x)\n", status);
+        return false;
+      }
+    } else {
+      switch (errno) {
+        case EINTR:
+          continue;
+          break;
+        case ECHILD:
+          print_debug("waitpid() failed. Child process pid (%d) does not exist \n", pid);
+          break;
+        case EINVAL:
+          print_debug("waitpid() failed. Invalid options argument.\n");
+          break;
+        default:
+          print_debug("waitpid() failed. Unexpected error %d\n",errno);
+          break;
+      }
+      return false;
+    }
+  }
+}
+
 // attach to a process/thread specified by "pid"
 static bool ptrace_attach(pid_t pid) {
   if (ptrace(PTRACE_ATTACH, pid, NULL, NULL) < 0) {
     print_debug("ptrace(PTRACE_ATTACH, ..) failed for %d\n", pid);
     return false;
   } else {
-    int ret;
-    int status;
-    do {
-      // Wait for debuggee to stop.
-      ret = waitpid(pid, &status, 0);
-      if (ret == -1 && errno == ECHILD) {
-        // try cloned process.
-        ret = waitpid(pid, &status, __WALL);
-      }
-      if (ret >= 0) {
-        if (WIFSTOPPED(status)) {
-          // Debuggee stopped.
-          return true;
-        } else {
-          print_debug("waitpid(): Child process exited/terminated (status = 0x%x)\n", status);
-          return false;
-        }
-      } else {
-        switch (errno) {
-          case EINTR:
-            continue;
-            break;
-          case ECHILD:
-            print_debug("waitpid() failed. Child process pid (%d) does not exist \n", pid);
-            break;
-          case EINVAL:
-            print_debug("waitpid() failed. Invalid options argument.\n");
-            break;
-          default:
-            print_debug("waitpid() failed. Unexpected error %d\n",errno);
-        }
-        return false;
-      }
-    } while(true);
+    return ptrace_waitpid(pid);
   }
 }